Whamcloud - gitweb
- ptlrpc_ping_interpret is needless:
[fs/lustre-release.git] / lustre-iokit / ior-survey / ior-survey
1 #!/bin/bash
2
3 # cluster name (all node names are this followed by the node number)
4 cluster=mdev
5
6 # client node numbers (individual numbers or inclusive ranges)
7 clients=(7-8)
8
9 # numbers of clients to survey
10 clients_lo=1
11 clients_hi=2
12 clients_iterator="+=1"
13
14 # numbers of tasks per client to survey
15 tasks_per_client_lo=1
16 tasks_per_client_hi=8
17 tasks_per_client_iterator="*=2"
18
19 # record sizes to survey
20 rsize_lo=1M
21 rsize_hi=1M
22 rsize_iterator="*=2"
23
24 ## which tests to run (first must be write)
25 # remount)   not really a test; just remount to uncache everything
26 # *write*)   write
27 # *)         read
28 #tests=(write rewrite read reread rewrite_again)
29 tests=(write rewrite remount read reread)
30
31 # total # bytes written/read by any client node
32 min_per_client_size=4G
33 min_total_size=8G
34
35 # should each task do I/O to its own file?
36 file_per_task=1
37
38 # the binaries
39 IOR="/home/ericb/ior/src/C/IOR"
40 llmount=/home/ericb/lustre/utils/llmount
41 pdsh=pdsh
42
43 # the result file prefix (date/time + hostname makes unique)
44 #rslt=/home/ericb/ior_survey_`date +%F@%R`_`uname -n`
45 rslt=/home/ericb/ior_survey
46
47 # where lustre is mounted on the clients
48 lustre=/mnt/lustre
49
50 # basename of the test file(s)
51 testfile=${lustre}/ior_survey_testfile
52
53 # how to unmount and remount the F/S on a client (to clear the cache)
54 # change this depending on lustre config (network type, MDS etc)
55 remount="umount $lustre && $llmount -o nettype=elan mdev6:/ll_mds/client $lustre"
56
57 # pdsh args required to instantiate all instances of IOR in parallel
58 # the chosen module must support '-n <procs-per-node>'
59 # -R<module>, -f<fanout> etc
60 pdsh_mpiargs="-Rmqsh"
61
62 #don't spin for MPI completions
63 export LIBELAN_WAITTYPE=0
64
65 ################################################################################
66 # dont change stuff below here unless you know what you're doing...
67
68 count_range() {
69     echo $1 | awk '{ nvals=split($1, vals, "-");\
70                      if (nvals == 1) print 1;\
71                      else if (nvals == 2) printf "%d\n", vals[2] - vals[1] + 1;}'
72 }
73
74 base_range() {
75     echo $1 | awk '{ split($1, vals, "-"); print vals[1]; }'
76 }
77
78 idx2nodenum() {
79     local n=$1; shift
80     while ((1)); do
81         local range=$1; shift
82         if [ -z "$range" ]; then
83             return
84         fi
85         chunk=`count_range $range`
86         if ((chunk > n)); then
87             base=`base_range $range`
88             echo $((base + n))
89             return
90         fi
91         n=$((n-chunk))
92     done
93 }
94
95 n2noderange() {
96     local n=$1; shift
97     sep=""
98     nodes="["
99     while ((n > 0)); do
100         local range=$1; shift
101         if [ -z "$range" ]; then
102             return
103         fi
104         local base=`base_range $range`
105         local chunk=`count_range $range`
106         if ((chunk > n)); then chunk=n; fi
107         local nodes="${nodes}${sep}${base}"; sep=","
108         if ((chunk > 1)); then nodes="${nodes}-$((base+chunk-1))"; fi
109         n=$((n-chunk))
110     done
111     echo "${nodes}]"
112 }
113
114 countnodes() {
115     local radix=16384
116     local n=0
117     while ((radix > 0)); do
118         local nodes=`n2noderange $((n+radix)) $@`
119         if [ -n "$nodes" ]; then
120             n=$((n+radix))
121         fi
122         radix=$((radix/2))
123     done
124     echo $n
125 }
126
127 parse_number() {
128     local str=$1
129     case $str in
130         *G|*g) n=`echo $str | sed 's/[gG]//'`; echo $((n*1024*1024*1024));;
131         *M|*m) n=`echo $str | sed 's/[Mm]//'`; echo $((n*1024*1024));;
132         *K|*k) n=`echo $str | sed 's/[Kk]//'`; echo $((n*1024));;
133         *)     echo $1;;
134     esac
135 }
136
137 pp_number() {
138     local n=$1
139     local G=$((1024*1024*1024))
140     local M=$((1024*1024))
141     local K=$((1024))
142     if ((n%G == 0 && n >= G)); then
143         echo "$((n/G))G"
144     elif ((n%M == 0 && n >= M)); then
145         echo "$((n/M))M"
146     elif ((n%K == 0 && n >= K)); then
147         echo "$((n/K))K"
148     else
149         echo $n
150     fi
151 }
152
153 if [ ${#tests[@]} -eq 0 -o "${tests[0]}" != "write" ]; then
154     echo "First test must be 'write'" 1>&2
155     exit 1
156 fi
157
158 rsltf="${rslt}.summary"
159 workf="${rslt}.detail"
160 echo -n > $rsltf
161 echo -n > $workf
162
163 print_summary () {
164     if [ "$1" = "-n" ]; then
165         minusn=$1; shift
166     else
167         minusn=""
168     fi
169     echo $minusn "$*" >> $rsltf
170     echo $minusn "$*"
171 }
172
173 # convert params to actual numbers
174 min_per_client_size=`parse_number $min_per_client_size`
175 min_total_size=`parse_number $min_total_size`
176
177 rsize_lo=`parse_number $rsize_lo`
178 rsize_hi=`parse_number $rsize_hi`
179
180 # check on actual numbers of client nodes
181 nclients=`countnodes ${clients[@]}`
182 if ((clients_hi > nclients)); then clients_hi=$nclients; fi
183
184 for ((rsize=rsize_lo; rsize<=rsize_hi; rsize$rsize_iterator)); do
185     pp_rsize=`pp_number $rsize`
186
187     for ((nclnt=clients_lo; nclnt<=clients_hi; nclnt$clients_iterator)); do
188         test_clients="${cluster}`n2noderange $nclnt ${clients[@]}`"
189
190         per_client_size=$((min_total_size/nclnt))
191         if ((per_client_size < min_per_client_size)); then
192             per_client_size=$min_per_client_size
193         fi
194
195         for ((ntask=tasks_per_client_lo; ntask <= tasks_per_client_hi; ntask$tasks_per_client_iterator)); do
196             per_task_size=$((per_client_size/ntask))
197             if ((per_task_size%rsize != 0)); then
198                 per_task_size=$(((per_task_size/rsize + 1)*rsize))
199             fi
200             total_size=`pp_number $((per_task_size*nclnt*ntask))`
201             
202             hdrstr=`printf "Total: %5sB rsize: %4sB clients: %4d tasks: %3d: " \
203                 $total_size $pp_rsize $nclnt $ntask`
204             print_summary -n "$hdrstr"
205
206             for ((test_idx=0; test_idx < ${#tests[@]}; test_idx++)); do
207                 test=${tests[$test_idx]}
208                 
209                 print_summary -n "$test "
210                 echo "===========> ${hdrstr} on $test_clients doing $test" >> $workf
211                 tmpf=${workf}_tmp
212                 echo -n > $tmpf
213
214                 if [ "$test" = "remount" ]; then
215                     echo "=> $remount" >> $tmpf
216                     $pdsh -S -b -w "$test_clients" >> $tmpf 2>&1 \
217                         "$remount"
218                     status=$?
219                     echo "Completion Status: $status" >> $tmpf
220
221                     if ((status)); then
222                         result="ERROR"
223                     else
224                         result="OK"
225                     fi
226                 else
227                     # check lustre is mounted everywhere it's needed
228                     cmd="(mount -t lustre; mount -t lustre_lite) | grep $lustre"
229                     echo "=> Mount Check: $cmd" >> $tmpf
230                     $pdsh -S -b -w "$test_clients" >> $tmpf 2>&1 \
231                         "$cmd"
232                     status=$?
233                     echo "Completion Status: $status" >> $tmpf
234                     if ((status)); then
235                         cat $tmpf >> $workf
236                         rm $tmpf
237                         print_summary "Lustre NOT mounted on $lustre somewhere"
238                         exit 1
239                     fi
240
241                     cmdline=(
242                     $IOR                     # the command
243                     -o${testfile}            # test file prefix
244                     -b${per_task_size}       # bytes per task
245                     -t${rsize}               # record size
246                     -e                       # fsync before close
247                     -q                       # quit on error
248                     )
249
250                     idx=${#cmdline[@]}
251
252                     # keep the test file(s) unless this is the last test
253                     ((test_idx < ${#tests[@]}-1)) && cmdline[$((idx++))]="-k"
254
255                     # use the existing test file(s) unless this is the first test
256                     ((test_idx > 0)) && cmdline[$((idx++))]="-E"
257
258                     # file-per-task
259                     ((file_per_task)) && cmdline[$((idx++))]="-F"
260
261                     case "$test" in
262                     *write*) cmdline[$((idx++))]="-w"
263                              awkstr="Max Write";;
264                     *)       cmdline[$((idx++))]="-r"
265                              awkstr="Max Read";;
266                     esac
267
268                     echo "=> ${cmdline[@]}" >> $tmpf
269         
270                     $pdsh -S -b $pdsh_mpiargs -w "$test_clients" -n $ntask >> $tmpf 2>&1 \
271                         "${cmdline[@]}"
272                     status=$?
273
274                     echo "Completion Status: $status" >> $tmpf
275                
276                     if ((status)); then
277                         result="ERROR"
278                     else
279                         result=`awk < $tmpf "/$awkstr/ {print $ 3; found=1; exit}\
280                                              END       {if (!found) print \"ERROR\"}"`
281                     fi
282                 fi
283
284                 cat $tmpf >> $workf
285                 rm $tmpf
286
287                 str=`printf "%8s" "$result"`
288                 print_summary -n "$str "
289             done
290             print_summary ""
291         done
292     done
293 done
294