Whamcloud - gitweb
Branch HEAD
[fs/lustre-release.git] / lustre / scripts / bdev-io-survey.sh
1 #!/bin/bash
2
3 # for now all the units are in 'k', but we could introduce some helpers
4 # would be nice to run tests in the background and trap signals and kill
5 #
6 #  todo:
7 #       make sure devices aren't in use before going to town
8 #       really use threads with iozone
9 #       look into what sgp_dd is really doing, update arguments
10 #       rename config/prepare/setup/cleanup/finish/teardown
11 #       do something with sf and fpp iterating
12 #       discard first vmstat line
13 #
14
15 HOSTNAME=`hostname`
16 # a temp dir that is setup and torn down for each script run
17 tmpdir=""
18 # so we can kill background processes as the test cleans up
19 declare -a cleanup_pids
20 # to unmount mounts in our tmpdir before removing it
21 declare -a cleanup_mounts
22 # global for completing the table.  XXX this is a wart that could go
23 cur_y="0"
24 # a global which funcs use to get at the blocks[] array
25 last_block=-1
26 # prefix to run oprofile or readprofile
27 oprofile=""
28 readprofile=""
29
30 # defaults for some options:
31 min_threads=1
32 max_threads=4
33 possible_tests="sgp_dd ext2_iozone echo_filter"
34 run_tests="$possible_tests"
35 echo_module=""
36
37 # optional output directory
38 output_dir=""
39  
40 die() {
41         echo $* 1>&2
42         exit 1
43 }
44 rm_or_die() {
45         for path in $*; do
46                 [ -e $path ] || continue;
47                 [ -f $path ] || die "needed to remove non-file $path"
48                 rm -f $path || die "couldn't remove $path"
49         done
50 }
51 save_output() {
52         [ ! -z "$output_dir" ] && mv -f $1 $output_dir/$2
53 }
54 cleanup() {
55         # only cleanup test runs if we have block devices
56         if [ $last_block != -1 ]; then
57                 for pid in ${cleanup_pids[*]}; do
58                         kill $pid
59                 done
60                 cleanup_echo_filter
61                 for a in ${cleanup_mounts[*]}; do
62                         umount -f $a
63                 done
64         fi
65
66         [ ${#tmpdir} == 18 ] && [ -d $tmpdir ] && rm -rf $tmpdir
67 }
68 trap cleanup EXIT
69
70 pid_now_running() {
71         local pid=$1
72         cleanup_pids[$pid]=$pid
73 }
74 pid_has_stopped() {
75         local pid=$1
76         unset cleanup_pids[$pid]
77 }
78                                                                                 
79 commas() {
80         echo $* | sed -e 's/ /,/g'
81 }
82 do_bc_scale() {
83         local scale=$1
84         shift
85         echo "scale=$scale; $*" | bc
86 }
87 do_bc() {
88         do_bc_scale 10 $*
89 }
90 mean_stddev() {
91         local points=$*
92
93         local avg=0
94         local num=0
95         for p in $points; do
96                 avg=`do_bc $avg + $p`
97                 num=`do_bc $num + 1`
98         done
99         case $num in
100                 0) echo '??' ; return ;;
101                 1) echo "$avg:0" ; return ;;
102         esac
103
104         avg=`do_bc $avg / $num`
105         local tmp=0
106         for p in $points; do
107                 local dev=`do_bc \($p - $avg\) \^ 2`
108                 tmp=`do_bc $tmp + $dev`
109         done
110         tmp=`do_bc_scale 1 sqrt \( $tmp / \($num - 1\) \)`
111         avg=`do_bc_scale 1 $avg / 1`
112         echo "$avg:$tmp"
113 }
114
115 usage() {
116         echo $*
117         echo "       -b <block device to profile>"
118         echo "       -d <summary output directory>"
119         echo "       -l <max io len>"
120         echo "       -t <minimum number of threads per device>"
121         echo "       -T <maximum number of threads per device>"
122         echo "       -r <tests to run>"
123         exit;
124 }
125
126 # some cute code for handling tables whose columns fit
127 set_max() {
128         local target=$1
129         local val=$2
130                                                                                 
131         if [ $val -gt ${!target:-0} ]; then
132                 eval $target=$val
133         fi
134 }
135 table_set() {
136         local name="_table_$1"
137         local col=$2
138         local row=$3
139         local val=$4
140         local num
141                                                                                 
142         eval ${name}_${row}_${col}="'$val'"
143                                                                                 
144         set_max ${name}_${col}_longest ${#val}
145         set_max ${name}_num_col $(($col + 1))
146         set_max ${name}_num_row $(($row + 1))
147 }
148                                                                                 
149 table_get() {
150         local name="_table_$1"
151         local col=$2
152         local row=$3
153         tmp="${name}_${row}_${col}"
154         echo ${!tmp}
155 }
156                                                                                 
157 table_dump() {
158         local name="_table_$1"
159         local num_col;
160         local num_row;
161         local fmt="";
162         local tmp
163         local sep
164                                                                                 
165         tmp="${name}_num_col"
166         num_col="${!tmp:-0}"
167         tmp="${name}_num_row"
168         num_row="${!tmp:-0}"
169                                                                                 
170         # iterate through the columns to find the longest
171                                                                                 
172         sep=" "
173         for x in `seq 0 $num_col`; do
174                 tmp="${name}_${x}_longest"
175                 tmp=${!tmp:-0}
176                 [ $tmp -eq 0 ] && continue
177                                                                                 
178                 [ $x -eq $((num_col - 1)) ] && sep='\n'
179                                                                                 
180                 fmt="$fmt%-${tmp}s$sep"
181         done
182                                                                                 
183         # nothing in the table to print
184         [ -z "$fmt" ] && return
185                                                                                 
186         for y in `seq 0 $num_row`; do
187                 local row=""
188                 for x in `seq 0 $num_col`; do
189                                                                                 
190                         # skip this element if the column is empty
191                         tmp="${name}_${x}_longest"
192                         [ ${!tmp:-0} -eq 0 ] && continue
193                                                                                 
194                         # fill this cell with the value or '' for printf
195                         tmp="${name}_${y}_${x}"
196                         row="$row'${!tmp:-""}' "
197                 done
198                 eval printf "'$fmt'" $row
199         done
200 }
201
202 ######################################################################
203 # the sgp_dd tests
204 sgp_dd_banner() {
205         echo sgp_dd using dio=1 and thr=
206 }
207 sgp_dd_config() {
208         # it could be making sure that the block dev
209         # isn't in use by something else
210         local nothing=0
211 }
212 sgp_dd_prepare() {
213         if ! which sgp_dd; then
214                 echo "can't find sgp_dd binary"
215                 return 1
216         fi
217         return 0
218 }
219 sgp_dd_setup() {
220         # it could be making sure that the block dev
221         # isn't in use by something else
222         local nothing=0
223 }
224 sgp_dd_start() {
225         local threads=$1
226         local iosize=$2
227         local wor=$3
228         local i=$4
229         local ifof;
230         local bdev=${blocks[$i]};
231
232         case "$wor" in
233                 [wo]) ifof="if=/dev/zero of=$bdev" ;;
234                 r) ifof="if=$bdev of=/dev/null" ;;
235                 *) die "asked to do io with $wor?"
236         esac
237         echo sgp_dd $ifof bs=$iosize"k" count=$(($io_len / $iosize)) time=1 \
238                         dio=1 thr=$threads
239 }
240 sgp_dd_result() {
241         local output=$1
242
243         awk '($(NF) == "MB/sec") {print $(NF-1)}' < $output
244 }
245 sgp_dd_cleanup() {
246         # got me
247         local nothing=0
248 }
249 sgp_dd_finish() {
250         # got me
251         local nothing=0
252 }
253 sgp_dd_teardown() {
254         # got me
255         local nothing=0
256 }
257
258 ######################################################################
259 # the iozone tests
260 ext2_iozone_banner() {
261         echo "iozone -I on a clean ext2 fs"
262 }
263 ext2_iozone_config() {
264         local nothing=0
265 }
266 ext2_iozone_prepare() {
267         local index=$1
268         local bdev=${blocks[$index]}
269         local mntpnt=$tmpdir/mount_$index
270
271         if ! which iozone; then
272                 echo "iozone binary not found in PATH"
273                 return 1
274         fi
275         if ! iozone -i 0 -w -+o -s 1k -r 1k -f /dev/null > /dev/null; then
276                 echo "iozone doesn't support -+o"
277                 return 1
278         fi
279         if ! which mke2fs; then
280                 echo "mke2fs binary not found in PATH"
281                 return 1
282         fi
283
284         if ! mkdir -p $mntpnt ; then
285                 echo "$mntpnt isn't a directory?"
286         fi
287
288         echo making ext2 filesystem on $bdev
289         if ! mke2fs -b 4096 $bdev; then
290                 echo "mke2fs failed"
291                 return 1;
292         fi
293
294         if ! mount -t ext2 $bdev $mntpnt; then 
295                 echo "couldn't mount $bdev on $mntpnt"
296                 return 1;
297         fi
298
299         cleanup_mounts[$index]="$mntpnt"
300         return 0
301 }
302 ext2_iozone_setup() {
303         local id=$1
304         local wor=$2
305         local f="$tmpdir/mount_$id/iozone"
306
307         case "$wor" in
308                 w) rm -f $f ;;
309                 [or]) ;;
310                 *) die "asked to do io with $wor?"
311         esac
312 }
313 ext2_iozone_start() {
314         local threads=$1
315         local iosize=$2
316         local wor=$3
317         local id=$4
318         local args;
319         local f="$tmpdir/mount_$id/iozone"
320
321         case "$wor" in
322                 [wo]) args="-i 0 -w" ;;
323                 r) args="-i 1" ;;
324                 *) die "asked to do io with $wor?"
325         esac
326
327         echo iozone "$args -r ${iosize}k -s $(($io_len / $threads))k \
328                         -t $threads -+o -x -I -f $f"
329 }
330 ext2_iozone_result() {
331         local output=$1
332         local wor=$2
333         local string
334         local field
335
336         case "$wor" in
337                 [wo]) string="writers" 
338                    field=7 
339                         ;;
340                 r) string="readers" 
341                    field=6
342                         ;;
343                 *) die "asked to do io with $wor?"
344         esac
345
346         do_bc_scale 1 `awk '($1 == "Parent" && $'$field' == "'$string'") \
347                         {print $'$(($field + 2))'}' $output` / 1024
348 }
349 ext2_iozone_cleanup() {
350         # the final read w/o -w removed the file
351         local nothing=0
352 }
353 ext2_iozone_finish() {
354         local index=$1
355         local mntpnt=$tmpdir/mount_$index
356
357         umount -f $mntpnt
358         unset cleanup_mounts[$index]
359 }
360 ext2_iozone_teardown() {
361         local nothing=0
362 }
363
364 ######################################################################
365 # the lctl test_brw via the echo_client on top of the filter
366
367 # the echo_client setup is nutty enough to warrant its own clenaup
368 running_config=""
369 running_module=""
370 declare -a running_names
371 declare -a running_oids
372
373 cleanup_echo_filter() {
374         local i
375
376         for i in `seq 0 $last_block`; do
377                 [ -z "${running_oids[$i]}" ] && continue
378                 lctl --device "\$"echo_$i destroy ${running_oids[$i]} \
379                         $running_threads
380         done
381         unset running_oids
382
383         for n in ${running_names[*]}; do
384 # I can't believe leading whitespace matters here.
385 lctl << EOF
386 cfg_device $n
387 cleanup
388 detach
389 quit
390 EOF
391         done
392         unset running_names
393
394         for m in $running_module; do
395                 rmmod $m
396         done
397         running_module=""
398
399         [ ! -z "$running_config" ] && lconf --cleanup $running_config
400         running_config=""
401 }
402
403 echo_filter_banner() {
404         echo "test_brw on the echo_client on the filter" 
405 }
406 echo_filter_config() {
407         local index=$1
408         local bdev=${blocks[$index]}
409         local config="$tmpdir/config.xml"
410
411         if ! which lmc; then
412                 echo "lmc binary not found in PATH"
413                 return 1
414         fi
415         if ! which lconf; then
416                 echo "lconf binary not found in PATH"
417                 return 1
418         fi
419         if ! which lctl; then
420                 echo "lctl binary not found in PATH"
421                 return 1
422         fi
423
424         if [ $index = 0 ]; then
425                 if ! lmc -m $config --add net \
426                         --node $HOSTNAME --nid $HOSTNAME --nettype tcp; then
427                         echo "error adding $HOSTNAME net node"
428                         return 1
429                 fi
430         fi
431
432         if ! lmc -m $config --add ost --ost ost_$index --node $HOSTNAME \
433                         --fstype ext3 --dev $bdev --journal_size 400; then
434                 echo "error adding $bdev to config with lmc"
435                 return 1
436         fi
437
438         # it would be nice to be able to ask lmc to setup an echo client
439         # to the filter here.  --add echo_client assumes osc
440 }
441 echo_filter_prepare() {
442         local index=$1
443         local bdev=${blocks[$index]}
444         local config="$tmpdir/config.xml"
445         local name="echo_$index"
446         local uuid="echo_$index_uuid"
447
448         if [ $index = 0 ]; then
449                 if ! lconf --reformat $config; then
450                         echo "error setting up with lconf"
451                         return 1;
452                 fi
453                 running_config="$config"
454
455                 echo 0 > /proc/sys/lnet/debug
456                 echo 0 > /proc/sys/lnet/subsystem_debug
457
458                 if ! grep -q '^obdecho\>' /proc/modules; then
459                         local m
460                         if ! modprobe obdecho; then
461                                 if [ ! -z "$echo_module" ]; then
462                                         if ! insmod $echo_module; then
463                                                 echo "err: insmod $echo_module"
464                                                 return 1;
465                                         else
466                                                 m="$echo_module"
467                                         fi
468                                 else
469                                         echo "err: modprobe $obdecho"
470                                         return 1;
471                                 fi
472                         else
473                                 m=obdecho
474                         fi
475                         running_module=`basename $m | cut -d'.' -f 1`
476                 fi
477         fi
478
479 lctl << EOF
480         newdev
481         attach echo_client $name $uuid
482         setup ost_$index
483         quit
484 EOF
485         if [  $? != 0 ]; then
486                 echo "error setting up echo_client $name against ost_$index"
487                 return 1
488         fi
489         running_names[$index]=$name
490 }
491 echo_filter_setup() {
492         local id=$1
493         local wor=$2
494         local threads=$3
495         local name="echo_$id"
496         local oid
497
498         case "$wor" in
499                 w) ;;
500                 [or]) return ;;
501                 *) die "asked to do io with $wor?"
502         esac
503
504         running_threads=$threads
505         oid=`lctl --device "\$"$name create $threads | \
506                 awk '/ #1 is object id/ { print $6 }'`
507         # XXX need to deal with errors
508         running_oids[$id]=$oid
509 }
510 echo_filter_start() {
511         local threads=$1
512         local iosize=$2
513         local wor=$3
514         local id=$4
515         local rw
516
517         local name="echo_$id"
518         local len_pages=$(($io_len / $(($page_size / 1024)) / $threads ))
519         local size_pages=$(($iosize / $(($page_size / 1024)) ))
520
521         case "$wor" in
522                 [wo]) rw="w" ;;
523                 r) rw="r" ;;
524                 *) die "asked to do io with $wor?"
525         esac
526
527         echo lctl --threads $threads v "\$"$name \
528                 test_brw 1 $rw v $len_pages t${running_oids[$id]} p$size_pages
529 }
530 echo_filter_result() {
531         local output=$1
532         local total=0
533         local mbs
534
535         for mbs in `awk '($8=="MB/s):"){print substr($7,2)}' < $output`; do
536                 total=$(do_bc $total + $mbs)
537         done
538         do_bc_scale 2 $total / 1
539 }
540 echo_filter_cleanup() {
541         local id=$1
542         local wor=$2
543         local threads=$3
544         local name="echo_$id"
545
546         case "$wor" in
547                 [wo]) return ;;
548                 r) ;;
549                 *) die "asked to do io with $wor?"
550         esac
551
552         lctl --device "\$"$name destroy ${running_oids[$id]} $threads
553         unset running_oids[$id]
554 }
555 echo_filter_finish() {
556         local index=$1
557         # leave real work for _teardown
558 }
559 echo_filter_teardown() {
560         cleanup_echo_filter
561 }
562
563 ######################################################################
564 # the iteration that drives the tests
565
566 test_one() {
567         local test=$1
568         local my_x=$2
569         local threads=$3
570         local iosize=$4
571         local wor=$5
572         local vmstat_pid
573         local vmstat_log="$tmpdir/vmstat.log"
574         local opref="$test-$threads-$iosize-$wor"
575         local -a iostat_pids
576         # sigh.  but this makes it easier to dump into the tables
577         local -a read_req_s
578         local -a mb_s
579         local -a write_req_s
580         local -a sects_req
581         local -a queued_reqs
582         local -a service_ms
583
584         for i in `seq 0 $last_block`; do
585                 ${test}_setup $i $wor $threads
586         done
587
588         echo $test with $threads threads
589
590         $oprofile opcontrol --start
591
592         # start up vmstat and record its pid
593         nice -19 vmstat 1 > $vmstat_log 2>&1 &
594         [ $? = 0 ] || die "vmstat failed"
595         vmstat_pid=$!
596         pid_now_running $vmstat_pid
597
598         # start up each block device's iostat
599         for i in `seq 0 $last_block`; do
600                 nice -19 iostat -x ${blocks[$i]} 1 | awk \
601                         '($1 == "'${blocks[$i]}'"){print $0; fflush()}' \
602                         > $tmpdir/iostat.$i &
603                 local pid=$!
604                 pid_now_running $pid
605                 iostat_pids[$i]=$pid
606         done
607
608         $oprofile opcontrol --reset
609         $readprofile -r
610
611         # start all the tests.  each returns a pid to wait on
612         pids=""
613         for i in `seq 0 $last_block`; do
614                 local cmd=`${test}_start $threads $iosize $wor $i`
615                 echo "$cmd" >> $tmpdir/commands
616                 $cmd > $tmpdir/$i 2>&1 &
617                 local pid=$!
618                 pids="$pids $pid"
619                 pid_now_running $pid
620         done
621
622         echo -n waiting on pids $pids:
623         for p in $pids; do
624                 wait $p
625                 echo -n .
626                 pid_has_stopped $p
627         done
628
629         # stop vmstat and all the iostats
630         kill $vmstat_pid
631         pid_has_stopped $vmstat_pid
632         for i in `seq 0 $last_block`; do
633                 local pid=${iostat_pids[$i]}
634                 [ -z "$pid" ] && continue
635
636                 kill $pid
637                 unset iostat_pids[$i]
638                 pid_has_stopped $pid
639         done
640
641         $readprofile | sort -rn > $tmpdir/readprofile
642
643         $oprofile opcontrol --shutdown
644         $oprofile opreport > $tmpdir/oprofile
645         echo >> $tmpdir/oprofile
646         $oprofile opreport -c -l | head -20 >> $tmpdir/oprofile
647
648         save_output $tmpdir/oprofile $opref.oprofile
649         save_output $tmpdir/readprofile $opref.readprofile
650
651         # collect the results of vmstat and iostat
652         cpu=$(mean_stddev $(awk \
653               '(NR > 3 && NF == 16 && $16 != "id" )     \
654                 {print 100 - $16}' < $vmstat_log) )
655         save_output $vmstat_log $opref.vmstat
656
657         for i in `seq 0 $last_block`; do
658                 read_req_s[$i]=$(mean_stddev $(awk \
659                       '(NR > 1) {print $4}' < $tmpdir/iostat.$i) )
660                 write_req_s[$i]=$(mean_stddev $(awk \
661                       '(NR > 1) {print $5}' < $tmpdir/iostat.$i) )
662                 sects_req[$i]=$(mean_stddev $(awk \
663                       '(NR > 1) {print $10}' < $tmpdir/iostat.$i) )
664                 queued_reqs[$i]=$(mean_stddev $(awk \
665                       '(NR > 1) {print $11}' < $tmpdir/iostat.$i) )
666                 service_ms[$i]=$(mean_stddev $(awk \
667                       '(NR > 1) {print $13}' < $tmpdir/iostat.$i) )
668
669                 save_output $tmpdir/iostat.$i $opref.iostat.$i
670         done
671
672         # record each index's test results and sum them
673         thru=0
674         for i in `seq 0 $last_block`; do
675                 local t=`${test}_result $tmpdir/$i $wor`
676                 save_output $tmpdir/$i $opref.$i
677                 echo test returned "$t"
678                 mb_s[$i]="$t"
679                 # some tests return mean:stddev per device, filter out stddev
680                 thru=$(do_bc $thru + $(echo $t | sed -e 's/:.*$//g'))
681         done
682
683         for i in `seq 0 $last_block`; do
684                 ${test}_cleanup $i $wor $threads
685         done
686
687         # tabulate the results
688         echo $test did $thru mb/s with $cpu
689         table_set $test $my_x $cur_y `do_bc_scale 2 $thru / 1`
690         table_set $test $(($my_x + 1)) $cur_y $cpu
691
692         for i in `seq 0 $last_block`; do
693                 cur_y=$(($cur_y + 1))
694                 table_set $test $(($my_x)) $cur_y ${mb_s[$i]}
695                 table_set $test $(($my_x + 1)) $cur_y ${read_req_s[$i]}
696                 table_set $test $(($my_x + 2)) $cur_y ${write_req_s[$i]}
697                 table_set $test $(($my_x + 3)) $cur_y ${sects_req[$i]}
698                 table_set $test $(($my_x + 4)) $cur_y ${queued_reqs[$i]}
699                 table_set $test $(($my_x + 5)) $cur_y ${service_ms[$i]}
700         done
701
702         cur_y=$(($cur_y + 1))
703 }
704
705 test_iterator() {
706         local test=$1
707         local thr=$min_threads
708         local cleanup=""
709         local rc=0
710         local i
711         
712         for i in `seq 0 $last_block`; do
713                 if ! ${test}_config $i; then
714                         echo "couldn't config $test for bdev ${blocks[$i]}"
715                         echo "skipping $test for all block devices"
716                         cleanup=$(($i - 1))
717                         rc=1;
718                         break
719                 fi
720         done
721
722         for i in `seq 0 $last_block`; do
723                 # don't prepare if _config already failed
724                 [ ! -z "$cleanup" ] && break
725                 if ! ${test}_prepare $i; then
726                         echo "couldn't prepare $test for bdev ${blocks[$i]}"
727                         echo "skipping $test for all block devices"
728                         cleanup=$(($i - 1))
729                         rc=1;
730                         break
731                 fi
732         done
733
734         while [ -z "$cleanup" -a $thr -lt $(($max_threads + 1)) ]; do
735                 for iosize in 128 512; do
736                         table_set $test 0 $cur_y $thr
737                         table_set $test 1 $cur_y $iosize
738
739                         for wor in w o r; do
740                                 table_set $test 2 $cur_y $wor
741                                 test_one $test 3 $thr $iosize $wor
742                         done
743                 done
744                 thr=$(($thr + $thr))
745         done
746
747         [ -z "$cleanup" ] && cleanup=$last_block
748
749         if [ "$cleanup" != -1 ]; then
750                 for i in `seq $cleanup 0`; do
751                         ${test}_finish $i
752                 done
753         fi
754
755         ${test}_teardown
756
757         return $rc;
758 }
759
760 while getopts ":d:b:l:t:T:r:e:" opt; do
761         case $opt in
762                 e) echo_module=$OPTARG                 ;;
763                 b) block=$OPTARG                 ;;
764                 d) output_dir=$OPTARG                 ;;
765                 l) io_len=$OPTARG                       ;;
766                 r) run_tests=$OPTARG                    ;;
767                 t) min_threads=$OPTARG                  ;;
768                 T) max_threads=$OPTARG                  ;;
769                 \?) usage
770         esac
771 done
772
773 page_size=`getconf PAGE_SIZE` || die '"getconf PAGE_SIZE" failed'
774
775 [ ! -z "$echo_module" -a ! -f "$echo_module" ] && \
776         die "obdecho module $echo_module is not a file"
777
778 if [ -z "$io_len" ]; then
779         io_len=`awk '($1 == "MemTotal:"){print $2}' < /proc/meminfo`
780         [ -z "$io_len" ] && die "couldn't determine the amount of memory"
781 fi
782
783 if [ ! -z "$output_dir" ]; then
784         if [ ! -e "$output_dir" ]; then
785                  mkdir -p "$output_dir" || die  "error creating $output_dir"
786         fi
787         [ ! -d "$output_dir" ] && die "$output_dir isn't a directory"
788 fi
789
790 block=`echo $block | sed -e 's/,/ /g'`
791 [ -z "$block" ] && usage "need block devices"
792
793 run_tests=`echo $run_tests | sed -e 's/,/ /g'`
794 [ -z "$run_tests" ] && usage "need to specify tests to run with -r"
795 for t in $run_tests; do
796         if ! echo $possible_tests | grep -q $t ; then
797                 die "$t isn't one of the possible tests: $possible_tests"
798         fi
799 done
800
801 if which opcontrol; then
802         echo generating oprofile results
803         oprofile=""
804 else
805         echo not using oprofile
806         oprofile=": "
807 fi
808
809 if which readprofile; then
810         map="/boot/System.map-`uname -r`"
811         if [ -f /proc/profile -a -f "$map" ]; then
812                 echo generating profiles with 'readprofile'
813                 readprofile="readprofile -m $map"
814         fi
815 fi
816 if [ -z "$readprofile" ]; then
817         echo not using readprofile
818         readprofile=": "
819 fi
820
821 [ $min_threads -gt $max_threads ] && \
822         die "min threads $min_threads must be <= min_threads $min_threads"
823
824 for b in $block; do
825         [ ! -e $b ] && die "block device file $b doesn't exist"
826         [ ! -b $b ] && die "$b isn't a block device"
827         dd if=$b of=/dev/null bs=8192 count=1 || \
828                 die "couldn't read 8k from $b, is it alive?"
829         [ ! -b $b ] && die "$b isn't a block device"
830         last_block=$(($last_block + 1))
831         blocks[$last_block]=$b
832 done    
833
834 tmpdir=`mktemp -d /tmp/.surveyXXXXXX` || die "couldn't create tmp dir"
835
836 echo each test will operate on $io_len"k"
837
838 test_results=""
839
840 for t in $run_tests; do
841
842         table_set $t 0 0 "T"
843         table_set $t 1 0 "L"
844         table_set $t 2 0 "m"
845         table_set $t 3 0 "A"
846         table_set $t 4 0 "C"
847         table_set $t 3 1 "MB"
848         table_set $t 4 1 "rR"
849         table_set $t 5 1 "wR"
850         table_set $t 6 1 "SR"
851         table_set $t 7 1 "Q"
852         table_set $t 8 1 "ms"
853         cur_y=2;
854
855         if ! test_iterator $t; then
856                 continue;
857         fi
858         test_results="$test_results $t"
859 done
860
861 save_output $tmpdir/commands commands
862
863 [ ! -z "$test_results" ] && (
864         echo
865         echo "T = number of concurrent threads per device"
866         echo "L = base io operation length, in KB"
867         echo "m = IO method: read, write, or over-write"
868         echo "A = aggregate throughput from all devices"
869         echo "C = percentage CPU used, both user and system"
870         echo "MB/s = per-device throughput"
871         echo "rR = read requests issued to the device per second"
872         echo "wR = write requests issued to the device per second"
873         echo "SR = sectors per request; sectors tend to be 512 bytes"
874         echo "Q = the average number of requests queued on the device"
875         echo "ms = the average ms taken by the device to service a req"
876         echo
877         echo "foo:bar represents a mean of foo with a stddev of bar"
878 )
879
880 for t in $test_results; do
881         ${t}_banner
882         table_dump $t
883 done