Whamcloud - gitweb
b=22481
[fs/lustre-release.git] / lustre-iokit / ior-survey / ior-survey
1 #!/bin/bash
2
3 # cluster name (all node names are this followed by the node number)
4 cluster=mdev
5
6 # client node numbers (individual numbers or inclusive ranges)
7 clients=(7-8)
8
9 # numbers of clients to survey
10 clients_lo=1
11 clients_hi=2
12 clients_iterator="+=1"
13
14 # numbers of tasks per client to survey
15 tasks_per_client_lo=1
16 tasks_per_client_hi=8
17 tasks_per_client_iterator="*=2"
18
19 # record sizes to survey
20 rsize_lo=1M
21 rsize_hi=1M
22 rsize_iterator="*=2"
23
24 ## which tests to run (first must be write)
25 # clear_cache)   not really a test; just uncache everything
26 # *write*)   write
27 # *)         read
28 #tests=(write rewrite read reread rewrite_again)
29 tests=(write rewrite clear_cache read reread)
30
31 # total # bytes written/read by any client node
32 min_per_client_size=4G
33 min_total_size=8G
34
35 # should each task do I/O to its own file?
36 file_per_task=1
37
38 # the binaries
39 IOR=/usr/local/sbin/IOR
40 llmount=llmount
41 pdsh=pdsh
42
43 # the result file prefix (date/time + hostname makes unique)
44 rslt=/tmp/ior_survey_`date +%F@%R`_`uname -n`
45
46 # where lustre is mounted on the clients
47 lustre=/mnt/lustre
48
49 # basename of the test file(s)
50 testfile=${lustre}/ior_survey_testfile
51
52 # pdsh args required to instantiate all instances of IOR in parallel
53 # the chosen module must support '-n <procs-per-node>'
54 # -R<module>, -f<fanout> etc
55 pdsh_mpiargs="-Rmqsh"
56
57 #don't spin for MPI completions
58 export LIBELAN_WAITTYPE=0
59
60 ################################################################################
61 # dont change stuff below here unless you know what you're doing...
62
63 count_range() {
64     echo $1 | awk '{ nvals=split($1, vals, "-");\
65                      if (nvals == 1) print 1;\
66                      else if (nvals == 2) printf "%d\n", vals[2] - vals[1] + 1;}'
67 }
68
69 base_range() {
70     echo $1 | awk '{ split($1, vals, "-"); print vals[1]; }'
71 }
72
73 idx2nodenum() {
74     local n=$1; shift
75     while ((1)); do
76         local range=$1; shift
77         if [ -z "$range" ]; then
78             return
79         fi
80         chunk=`count_range $range`
81         if ((chunk > n)); then
82             base=`base_range $range`
83             echo $((base + n))
84             return
85         fi
86         n=$((n-chunk))
87     done
88 }
89
90 n2noderange() {
91     local n=$1; shift
92     sep=""
93     nodes="["
94     while ((n > 0)); do
95         local range=$1; shift
96         if [ -z "$range" ]; then
97             return
98         fi
99         local base=`base_range $range`
100         local chunk=`count_range $range`
101         if ((chunk > n)); then chunk=n; fi
102         local nodes="${nodes}${sep}${base}"; sep=","
103         if ((chunk > 1)); then nodes="${nodes}-$((base+chunk-1))"; fi
104         n=$((n-chunk))
105     done
106     echo "${nodes}]"
107 }
108
109 countnodes() {
110     local radix=16384
111     local n=0
112     while ((radix > 0)); do
113         local nodes=`n2noderange $((n+radix)) $@`
114         if [ -n "$nodes" ]; then
115             n=$((n+radix))
116         fi
117         radix=$((radix/2))
118     done
119     echo $n
120 }
121
122 parse_number() {
123     local str=$1
124     case $str in
125         *G|*g) n=`echo $str | sed 's/[gG]//'`; echo $((n*1024*1024*1024));;
126         *M|*m) n=`echo $str | sed 's/[Mm]//'`; echo $((n*1024*1024));;
127         *K|*k) n=`echo $str | sed 's/[Kk]//'`; echo $((n*1024));;
128         *)     echo $1;;
129     esac
130 }
131
132 pp_number() {
133     local n=$1
134     local G=$((1024*1024*1024))
135     local M=$((1024*1024))
136     local K=$((1024))
137     if ((n%G == 0 && n >= G)); then
138         echo "$((n/G))G"
139     elif ((n%M == 0 && n >= M)); then
140         echo "$((n/M))M"
141     elif ((n%K == 0 && n >= K)); then
142         echo "$((n/K))K"
143     else
144         echo $n
145     fi
146 }
147
148 if [ ${#tests[@]} -eq 0 -o "${tests[0]}" != "write" ]; then
149     echo "First test must be 'write'" 1>&2
150     exit 1
151 fi
152
153 rsltf="${rslt}.summary"
154 workf="${rslt}.detail"
155 echo -n > $rsltf
156 echo -n > $workf
157
158 print_summary () {
159     if [ "$1" = "-n" ]; then
160         minusn=$1; shift
161     else
162         minusn=""
163     fi
164     echo $minusn "$*" >> $rsltf
165     echo $minusn "$*"
166 }
167
168 # convert params to actual numbers
169 min_per_client_size=`parse_number $min_per_client_size`
170 min_total_size=`parse_number $min_total_size`
171
172 rsize_lo=`parse_number $rsize_lo`
173 rsize_hi=`parse_number $rsize_hi`
174
175 # check on actual numbers of client nodes
176 nclients=`countnodes ${clients[@]}`
177 if ((clients_hi > nclients)); then clients_hi=$nclients; fi
178
179 for ((rsize=rsize_lo; rsize<=rsize_hi; rsize$rsize_iterator)); do
180     pp_rsize=`pp_number $rsize`
181
182     for ((nclnt=clients_lo; nclnt<=clients_hi; nclnt$clients_iterator)); do
183         test_clients="${cluster}`n2noderange $nclnt ${clients[@]}`"
184
185         per_client_size=$((min_total_size/nclnt))
186         if ((per_client_size < min_per_client_size)); then
187             per_client_size=$min_per_client_size
188         fi
189
190         for ((ntask=tasks_per_client_lo; ntask <= tasks_per_client_hi; ntask$tasks_per_client_iterator)); do
191             per_task_size=$((per_client_size/ntask))
192             if ((per_task_size%rsize != 0)); then
193                 per_task_size=$(((per_task_size/rsize + 1)*rsize))
194             fi
195             total_size=`pp_number $((per_task_size*nclnt*ntask))`
196             
197             hdrstr=`printf "Total: %5sB rsize: %4sB clients: %4d tasks: %3d: " \
198                 $total_size $pp_rsize $nclnt $ntask`
199             print_summary -n "$hdrstr"
200
201             for ((test_idx=0; test_idx < ${#tests[@]}; test_idx++)); do
202                 test=${tests[$test_idx]}
203                 
204                 print_summary -n "$test "
205                 echo "===========> ${hdrstr} on $test_clients doing $test" >> $workf
206                 tmpf=${workf}_tmp
207                 echo -n > $tmpf
208
209                 if [ "$test" = "clear_cache" ]; then
210                     clear_cache='for LRU in /proc/fs/lustre/ldlm/namespaces/*/lru_size; do; echo clear > $LRU; done'
211                     echo "=> $clear_cache" >> $tmpf
212                     $pdsh -S -b -w "$test_clients" >> $tmpf 2>&1 \
213                         "$clear_cache"
214                     status=$?
215                     echo "Completion Status: $status" >> $tmpf
216
217                     if ((status)); then
218                         result="ERROR"
219                     else
220                         result="OK"
221                     fi
222                 else
223                     # check lustre is mounted everywhere it's needed
224                     cmd="(mount -t lustre; mount -t lustre_lite) | grep $lustre"
225                     echo "=> Mount Check: $cmd" >> $tmpf
226                     $pdsh -S -b -w "$test_clients" >> $tmpf 2>&1 \
227                         "$cmd"
228                     status=$?
229                     echo "Completion Status: $status" >> $tmpf
230                     if ((status)); then
231                         cat $tmpf >> $workf
232                         rm $tmpf
233                         print_summary "Lustre NOT mounted on $lustre somewhere"
234                         exit 1
235                     fi
236
237                     cmdline=(
238                     $IOR                     # the command
239                     -o${testfile}            # test file prefix
240                     -b${per_task_size}       # bytes per task
241                     -t${rsize}               # record size
242                     -e                       # fsync before close
243                     -q                       # quit on error
244                     )
245
246                     idx=${#cmdline[@]}
247
248                     # keep the test file(s) unless this is the last test
249                     ((test_idx < ${#tests[@]}-1)) && cmdline[$((idx++))]="-k"
250
251                     # use the existing test file(s) unless this is the first test
252                     ((test_idx > 0)) && cmdline[$((idx++))]="-E"
253
254                     # file-per-task
255                     ((file_per_task)) && cmdline[$((idx++))]="-F"
256
257                     case "$test" in
258                     *write*) cmdline[$((idx++))]="-w"
259                              awkstr="Max Write";;
260                     *)       cmdline[$((idx++))]="-r"
261                              awkstr="Max Read";;
262                     esac
263
264                     echo "=> ${cmdline[@]}" >> $tmpf
265         
266                     $pdsh -S -b $pdsh_mpiargs -w "$test_clients" -n $ntask >> $tmpf 2>&1 \
267                         "${cmdline[@]}"
268                     status=$?
269
270                     echo "Completion Status: $status" >> $tmpf
271                
272                     if ((status)); then
273                         result="ERROR"
274                     else
275                         result=`awk < $tmpf "/$awkstr/ {print $ 3; found=1; exit}\
276                                              END       {if (!found) print \"ERROR\"}"`
277                     fi
278                 fi
279
280                 cat $tmpf >> $workf
281                 rm $tmpf
282
283                 str=`printf "%8s" "$result"`
284                 print_summary -n "$str "
285             done
286             print_summary ""
287         done
288     done
289 done
290