Whamcloud - gitweb
* Added obdfilter-survey + README
[fs/lustre-release.git] / lustre-iokit / obdfilter-survey / obdfilter-survey
1 #!/bin/bash
2
3 ######################################################################
4 # customize per survey
5
6 # specify either the obdecho client names or the obdfilter names
7 client_names=()
8 ost_names=(ost{1,2,3,4,5,6,7,8})
9
10 # result file prefix
11 rslt=/tmp/obdfilter_survey
12
13 # lustre root (leave blank unless running with own source)
14 lustre_root=
15
16 # what to do (we always do an initial write)
17 #tests="rewrite read reread rewrite_again"
18 tests="rewrite read"
19
20 # total size (MBytes)
21 # large enough to avoid cache effects
22 size=8192
23
24 # record size (KBytes)
25 rszlo=1024
26 rszhi=1024
27
28 # number of objects per OST
29 nobjlo=1
30 nobjhi=32
31
32 # threads per OST (1024 max)
33 thrlo=1
34 thrhi=128
35
36 # restart from here iff all are defined
37 restart_rsz=
38 restart_thr=1
39 restart_nobj=1
40
41 # machine's page size
42 PAGE_SIZE=64
43
44 # max buffer_mem (total_threads * buffer size)
45 # (to avoid lctl ENOMEM problems)
46 max_buffer_mem=$((256*1024))
47
48 #####################################################################
49
50 snap=1
51 verify=1
52
53 check_obdecho() {
54     lsmod | grep obdecho > /dev/null 2>&1
55 }
56
57 check_obdecho
58 load_obdecho=$(($? != 0))
59
60 if [ -z "$lustre_root" ]; then
61     lctl=lctl
62     if ((load_obdecho)); then
63         modprobe obdecho
64     fi
65 else
66     lctl=${lustre_root}/lctl
67     if ((load_obdecho)); then
68         if [ -f ${lustre_root}/obdecho/obdecho.ko ]; then
69             insmod ${lustre_root}/obdecho/obdecho.ko
70         else
71             insmod ${lustre_root}/obdecho/obdecho.o
72         fi
73     fi
74 fi
75
76 check_obdecho || (echo "Can't load obdecho"; exit 1)
77
78 get_devno () {
79     local type=$1
80     local name=$2
81     $lctl device_list | awk "{if (\$2 == \"UP\" && \$3 == \"$type\" && \$4 == \"$name\") {\
82                                  print \$1; exit}}"
83 }
84
85 get_ec_devno () {
86     local idx=$1
87     local client_name=${client_names[idx]}
88     local ost_name=${ost_names[idx]}
89     if [ -z "$client_name" ]; then
90         if [ -z "$ost_name" ]; then
91             echo "client and ost name both null" 1>&2
92             return
93         fi
94         client_name=${ost_name}_echo_client
95     fi
96     ec=`get_devno echo_client $client_name`
97     if [ -n "$ec" ]; then
98         echo $ec $client_name 
99         return
100     fi
101     if [ -z "$ost_name" ]; then
102         echo "no echo client and ost_name not set" 1>&2
103         return
104     fi
105     ost=`get_devno obdfilter $ost_name`
106     if [ -z "$ost" ]; then
107         echo "OST $ost_name not setup" 1>&2
108         return
109     fi
110     $lctl <<EOF
111         attach echo_client $client_name ${client_name}_UUID
112         setup $ost_name
113 EOF
114     ec=`get_devno echo_client $client_name`
115     if [ -z "$ec" ]; then
116         echo "Can't setup echo client" 1>&2
117         return
118     fi
119     echo $ec $client_name 1
120 }
121
122 teardown_ec_devno () {
123     local idx=$1
124     local client_name=${client_names[$idx]}
125     if ((do_teardown_ec[$idx])); then
126         $lctl <<EOF
127         cfg $client_name
128         cleanup
129         detach
130 EOF
131    fi
132 }
133
134 create_objects () {
135     # create a set of objects, check there are 'n' contiguous ones and
136     # return the first or 'ERROR'
137     local devno=$1
138     local nobj=$2
139     local rfile=$3
140     $lctl --device $devno create $nobj > $rfile 2>&1
141     n=(`awk < $rfile \
142         '/is object id/ {obj=strtonum($6);\
143                          first=!not_first; not_first=1;\
144                          if (first) first_obj=obj;
145                          else if (obj != prev + 1) exit;\
146                          prev=obj; n++}\
147             END {printf "%d %d\n", first_obj, n}'`)
148     if ((n[1] != nobj)); then
149         echo "ERROR"
150     else
151         echo ${n[0]}
152     fi
153 }
154
155 destroy_objects () {
156     local devno=$1
157     local obj0=$2
158     local nobj=$3
159     local rfile=$4
160     $lctl --device $devno destroy $obj0 $nobj > $rfile 2>&1
161 }
162
163 get_stats () {
164     local rfile=$1
165     awk < $rfile \
166         '/^Selected device [0-9]+$/ {n = 0; next}\
167         /error/ {n = -1; exit}\
168         /^[0-9]+\/[0-9]+ Total: [0-9]+\.[0-9]+\/second$/ {n++; v=strtonum($3); \
169                                                           if (n == 1 || v < min) min = v;\
170                                                           if (n == 1 || v > max) max = v;\
171                                                           next}\
172         {if (n != 0) {n = -1; exit}}\
173         END {printf "%d %f %f\n", n, min, max}'
174 }
175
176 get_global_stats () {
177     local rfile=$1
178     awk < $rfile 'BEGIN {n = 0;}\
179                   {n++; if (n == 1) {err = $1; min = $2; max = $3} else\
180                                     {if ($1 < err) err = $1;\
181                                      if ($2 < min) min = $2;\
182                                      if ($3 > max) max = $3}}\
183                   END {if (n == 0) err = 0;\
184                        printf "%d %f %f\n", err, min, max}'
185 }
186
187 testname2type () {
188     # 'x' disables data check
189     if ((verify)); then
190         x=""
191     else
192         x="x"
193     fi
194     case $1 in
195         *write*)  echo "w$x";;
196         *)        echo "r$x";;
197     esac
198 }
199
200 start=`date +%F@%R`
201 rsltf="${rslt}_${start}.summary"
202 echo -n > $rsltf
203 workf="${rslt}_${start}.detail"
204 echo -n > $workf
205
206 print_summary () {
207     if [ "$1" = "-n" ]; then
208         minusn=$1; shift
209     else
210         minusn=""
211     fi
212     echo $minusn "$*" >> $rsltf
213     echo $minusn "$*"
214 }
215
216 ndevs=${#client_names[@]}
217 if ((ndevs < ${#ost_names[@]} )); then
218     ndevs=${#ost_names[@]}
219 fi
220
221 for ((idx = 0; idx < ndevs; idx++)); do
222     devno=(`get_ec_devno $idx`)
223     if ((${#devno[@]} < 2)); then
224         exit 1
225     fi
226     devnos[$idx]=${devno[0]}
227     client_names[$idx]=${devno[1]}
228     do_teardown_ec[$idx]=$((${#devno[@]} > 2))
229 done
230
231 echo 0 > /proc/sys/portals/debug
232
233 for ((rsz=$rszlo;rsz<=$rszhi;rsz*=2)); do
234     for ((nobj=$nobjlo;nobj<=$nobjhi;nobj*=2)); do 
235         for ((thr=$thrlo;thr<=$thrhi;thr*=2)); do
236             if ((thr < nobj)); then
237                 continue
238             fi
239             # restart?
240             if [ -n "$restart_rsz" -a\
241                  -n "$restart_nobj" -a\
242                  -n "$restart_thr" ]; then
243                 if ((rsz < restart_rsz ||\
244                      (rsz == restart_rsz &&\
245                       (nobj < restart_nobj ||\
246                        (nobj == restart_nobj &&\
247                         thr < restart_thr))))); then
248                     continue;
249                 fi
250             fi
251             # compute parameters
252             total_thr=$((ndevs*thr))
253             total_nobj=$((ndevs*nobj))
254             pages=$((rsz/PAGE_SIZE))
255             actual_rsz=$((pages*PAGE_SIZE))
256             count=$((size*1024/(actual_rsz*thr)))
257             actual_size=$((actual_rsz*count*thr))
258             total_size=$((actual_size*ndevs))
259             # show computed parameters
260             str=`printf 'ost %2d sz %8dK rsz %4d obj %4d thr %4d ' \
261                      $ndevs $total_size $actual_rsz $total_nobj $total_thr`
262             echo "=======================> $str" >> $workf
263             print_summary -n "$str"
264             if ((total_thr * actual_rsz > max_buffer_mem)); then
265                 print_summary "Too much buffer space"
266                 continue
267             fi
268             # create the objects
269             tmpf="${workf}_tmp"
270             for ((idx=0; idx < ndevs; idx++)); do
271                 devno=${devnos[$idx]}
272                 first_obj=`create_objects $devno $nobj $tmpf`
273                 echo "========> Create [$idx]" >> $workf
274                 cat $tmpf >> $workf
275                 rm $tmpf
276                 if [ $first_obj = "ERROR" ]; then
277                     print_summary "created object #s [$idx] not contiguous"
278                     exit 1
279                 fi
280                 first_objs[$idx]=$first_obj
281             done
282             for test in write $tests; do
283                 print_summary -n "$test "
284                 t0=`date +%s.%N`
285                 for ((idx=0; idx < ndevs; idx++)); do
286                     devno=${devnos[$idx]}
287                     tmpfi="${tmpf}_$idx"
288                     first_obj=${first_objs[$idx]}
289                     $lctl > $tmpfi 2>&1 \
290                         --threads $thr -$snap $devno \
291                         test_brw $count `testname2type $test` q $pages ${thr}t${first_obj} &
292                 done
293                 wait
294                 t1=`date +%s.%N`
295                 str=`awk "BEGIN {printf \"%7.2f \",\
296                          $total_size / (( $t1 - $t0 ) * 1024)}"`
297                 print_summary -n "$str"
298                 echo -n > $tmpf
299                 for ((idx=0; idx < ndevs; idx++)); do
300                     tmpfi="${tmpf}_$idx"
301                     echo "========> $test [$idx]" >> $workf
302                     cat $tmpfi >> $workf
303                     get_stats $tmpfi >> $tmpf
304                     rm $tmpfi
305                 done
306                 echo "========> $test [$idx] global" >> $workf
307                 cat $tmpf >> $workf
308                 stats=(`get_global_stats $tmpf`)
309                 rm $tmpf
310                 if ((stats[0] <= 0)); then
311                     if ((stats[0] < 0)); then
312                         str=`printf "%15s " ERROR`
313                     else
314                         str=`printf "%15s " SHORT`
315                     fi
316                 else
317                     str=`awk "BEGIN {printf \"[%6.2f,%6.2f] \",\
318                              (${stats[1]} * $actual_rsz)/1024,\
319                              (${stats[2]} * $actual_rsz)/1024; exit}"`
320                 fi
321                 print_summary -n "$str"
322             done
323             print_summary ""
324             for ((idx=0; idx < ndevs; idx++)); do
325                 devno=${devnos[$idx]}
326                 first_obj=${first_objs[$idx]}
327                 destroy_objects $devno $first_obj $nobj $tmpf
328                 echo "========> Destroy [$idx]" >> $workf
329                 cat $tmpf >> $workf
330                 rm $tmpf
331             done
332         done
333     done
334 done
335
336 for ((idx=0; idx < ndevs; idx++)); do
337     teardown_ec_devno $idx
338 done
339
340 if ((load_obdecho)); then
341     rmmod obdecho
342 fi