3 # for now all the units are in 'k', but we could introduce some helpers
4 # would be nice to run tests in the background and trap signals and kill
7 # make sure devices aren't in use before going to town
8 # really use threads with iozone
9 # look into what sgp_dd is really doing, update arguments
10 # rename config/prepare/setup/cleanup/finish/teardown
11 # do something with sf and fpp iterating
12 # discard first vmstat line
15 # a temp dir that is setup and torn down for each script run
17 # so we can kill background processes as the test cleans up
18 declare -a cleanup_pids
19 # to unmount mounts in our tmpdir before removing it
20 declare -a cleanup_mounts
21 # global for completing the table. XXX this is a wart that could go
24 # defaults for some options:
27 possible_tests="sgp_dd ext2_iozone echo_filter"
28 run_tests="$possible_tests"
30 # optional output directory
39 [ -e $path ] || continue;
40 [ -f $path ] || die "needed to remove non-file $path"
41 rm -f $path || die "couldn't remove $path"
45 [ ! -z "$output_dir" ] && mv -f $1 $output_dir/$2
48 # only cleanup test runs if we have block devices
49 if [ $last_block != -1 ]; then
50 for pid in ${cleanup_pids[*]}; do
54 for a in ${cleanup_mounts[*]}; do
59 [ ${#tmpdir} == 18 ] && [ -d $tmpdir ] && rm -rf $tmpdir
65 cleanup_pids[$pid]=$pid
69 unset cleanup_pids[$pid]
73 echo $* | sed -e 's/ /,/g'
78 echo "scale=$scale; $*" | bc
93 0) echo '??' ; return ;;
94 1) echo "$avg:0" ; return ;;
97 avg=`do_bc $avg / $num`
100 local dev=`do_bc \($p - $avg\) \^ 2`
101 tmp=`do_bc $tmp + $dev`
103 tmp=`do_bc_scale 1 sqrt \( $tmp / \($num - 1\) \)`
104 avg=`do_bc_scale 1 $avg / 1`
110 echo " -b <block device to profile>"
111 echo " -d <summary output directory>"
112 echo " -l <max io len>"
113 echo " -t <minimum number of threads per device>"
114 echo " -T <maximum number of threads per device>"
115 echo " -r <tests to run>"
119 # some cute code for handling tables whose columns fit
124 if [ $val -gt ${!target:-0} ]; then
129 local name="_table_$1"
135 eval ${name}_${row}_${col}="'$val'"
137 set_max ${name}_${col}_longest ${#val}
138 set_max ${name}_num_col $(($col + 1))
139 set_max ${name}_num_row $(($row + 1))
143 local name="_table_$1"
146 tmp="${name}_${row}_${col}"
151 local name="_table_$1"
158 tmp="${name}_num_col"
160 tmp="${name}_num_row"
163 # iterate through the columns to find the longest
166 for x in `seq 0 $num_col`; do
167 tmp="${name}_${x}_longest"
169 [ $tmp -eq 0 ] && continue
171 [ $x -eq $((num_col - 1)) ] && sep='\n'
173 fmt="$fmt%-${tmp}s$sep"
176 # nothing in the table to print
177 [ -z "$fmt" ] && return
179 for y in `seq 0 $num_row`; do
181 for x in `seq 0 $num_col`; do
183 # skip this element if the column is empty
184 tmp="${name}_${x}_longest"
185 [ ${!tmp:-0} -eq 0 ] && continue
187 # fill this cell with the value or '' for printf
188 tmp="${name}_${y}_${x}"
189 row="$row'${!tmp:-""}' "
191 eval printf "'$fmt'" $row
195 ######################################################################
198 echo sgp_dd using dio=1 and thr=
201 # it could be making sure that the block dev
202 # isn't in use by something else
206 if ! which sgp_dd; then
207 echo "can't find sgp_dd binary"
213 # it could be making sure that the block dev
214 # isn't in use by something else
223 local bdev=${blocks[$i]};
226 w) ifof="if=/dev/zero of=$bdev" ;;
227 r) ifof="if=$bdev of=/dev/null" ;;
228 *) die "asked to do io with $wor?"
230 echo sgp_dd $ifof bs=$iosize"k" count=$(($io_len / $iosize)) time=1 \
236 awk '($(NF) == "MB/sec") {print $(NF-1)}' < $output
251 ######################################################################
253 ext2_iozone_banner() {
254 echo "iozone -I on a clean ext2 fs"
256 ext2_iozone_config() {
259 ext2_iozone_prepare() {
261 local bdev=${blocks[$index]}
262 local mntpnt=$tmpdir/mount_$index
264 if ! which iozone; then
265 echo "iozone binary not found in PATH"
268 if ! which mke2fs; then
269 echo "mke2fs binary not found in PATH"
273 if ! mkdir -p $mntpnt ; then
274 echo "$mntpnt isn't a directory?"
277 echo making ext2 filesystem on $bdev
278 if ! mke2fs -b 4096 $bdev; then
283 if ! mount -t ext2 $bdev $mntpnt; then
284 echo "couldn't mount $bdev on $mntpnt"
288 cleanup_mounts[$index]="$mntpnt"
291 ext2_iozone_setup() {
294 local f="$tmpdir/mount_$id/iozone"
299 *) die "asked to do io with $wor?"
302 ext2_iozone_start() {
308 local f="$tmpdir/mount_$id/iozone"
313 *) die "asked to do io with $wor?"
316 echo iozone "$args -r ${iosize}k -s ${io_len}k -I -f $f"
318 ext2_iozone_result() {
321 kps=`awk '($2 == "reclen"){results=NR+1}(results == NR){print $3}' \
323 do_bc_scale 0 "$kps / 1024"
325 ext2_iozone_cleanup() {
328 local f="$tmpdir/mount_$id/iozone"
333 *) die "asked to do io with $wor?"
336 ext2_iozone_finish() {
338 local mntpnt=$tmpdir/mount_$index
341 unset cleanup_mounts[$index]
343 ext2_iozone_teardown() {
347 ######################################################################
348 # the lctl test_brw via the echo_client on top of the filter
350 # the echo_client setup is nutty enough to warrant its own clenaup
353 declare -a running_names
354 declare -a running_oids
356 cleanup_echo_filter() {
359 for i in `seq 0 $last_block`; do
360 [ -z "${running_oids[$i]}" ] && continue
361 lctl --device "\$"echo_$i destroy ${running_oids[$i]} \
366 for n in ${running_names[*]}; do
367 # I can't believe leading whitespace matters here.
377 for m in $running_modules; do
382 [ ! -z "$running_config" ] && lconf --cleanup $running_config
386 echo_filter_banner() {
387 echo "test_brw on the echo_client on the filter"
389 echo_filter_config() {
391 local bdev=${blocks[$index]}
392 local config="$tmpdir/config.xml"
395 echo "lmc binary not found in PATH"
398 if ! which lconf; then
399 echo "lconf binary not found in PATH"
402 if ! which lctl; then
403 echo "lctl binary not found in PATH"
407 if [ $index = 0 ]; then
408 if ! lmc -m $config --add net \
409 --node localhost --nid localhost --nettype tcp; then
410 echo "error adding localhost net node"
415 if ! lmc -m $config --add ost --ost ost_$index --node localhost \
416 --fstype ext3 --dev $bdev --journal_size 400; then
417 echo "error adding $bdev to config with lmc"
421 # it would be nice to be able to ask lmc to setup an echo client
422 # to the filter here. --add echo_client assumes osc
424 echo_filter_prepare() {
426 local bdev=${blocks[$index]}
427 local config="$tmpdir/config.xml"
428 local name="echo_$index"
429 local uuid="echo_$index_uuid"
431 if [ $index = 0 ]; then
432 if ! lconf --reformat $config; then
433 echo "error setting up with lconf"
436 running_config="$config"
437 if ! grep -q '^obdecho\>' /proc/modules; then
438 if ! modprobe obdecho; then
439 echo "error running modprobe obdecho"
442 running_modules="obdecho"
448 attach echo_client $name $uuid
453 echo "error setting up echo_client $name against ost_$index"
456 running_names[$index]=$name
458 echo_filter_setup() {
462 local name="echo_$id"
468 *) die "asked to do io with $wor?"
471 running_threads=$threads
472 oid=`lctl --device "\$"$name create $threads | \
473 awk '/1 is object id/ { print $6 }'`
474 # XXX need to deal with errors
475 running_oids[$id]=$oid
477 echo_filter_start() {
482 local name="echo_$id"
483 local pages=$(($io_len / 4))
488 *) die "asked to do io with $wor?"
491 echo lctl --threads $threads v "\$"$name \
492 test_brw 1 w v $pages ${running_oids[$i]} p$iosize
494 echo_filter_result() {
499 for mbs in `awk '($8=="MB/s):"){print substr($7,2)}' < $output`; do
500 total=$(do_bc $total + $mbs)
502 do_bc_scale $total / 1
504 echo_filter_cleanup() {
508 local name="echo_$id"
513 *) die "asked to do io with $wor?"
516 lctl --device "\$"$name destroy ${running_oids[$i]} $threads
517 unset running_oids[$i]
519 echo_filter_finish() {
521 # leave real work for _teardown
523 echo_filter_teardown() {
527 ######################################################################
528 # the iteration that drives the tests
537 local vmstat_log="$tmpdir/vmstat.log"
538 local opref="$test-$threads-$iosize-$wor"
540 # sigh. but this makes it easier to dump into the tables
548 for i in `seq 0 $last_block`; do
549 ${test}_setup $i $wor $threads
552 echo $test with $threads threads
554 # start up vmstat and record its pid
555 nice -19 vmstat 1 > $vmstat_log 2>&1 &
556 [ $? = 0 ] || die "vmstat failed"
558 pid_now_running $vmstat_pid
560 # start up each block device's iostat
561 for i in `seq 0 $last_block`; do
562 nice -19 iostat -x ${blocks[$i]} 1 | awk \
563 '($1 == "'${blocks[$i]}'"){print $0; fflush()}' \
564 > $tmpdir/iostat.$i &
570 # start all the tests. each returns a pid to wait on
572 for i in `seq 0 $last_block`; do
573 local cmd=`${test}_start $threads $iosize $wor $i`
574 $cmd > $tmpdir/$i 2>&1 &
580 echo -n waiting on pids $pids:
587 # stop vmstat and all the iostats
589 pid_has_stopped $vmstat_pid
590 for i in `seq 0 $last_block`; do
591 local pid=${iostat_pids[$i]}
592 [ -z "$pid" ] && continue
595 unset iostat_pids[$i]
599 # collect the results of vmstat and iostat
600 cpu=$(mean_stddev $(awk \
601 '(NR > 3 && NF == 16 && $16 != "id" ) \
602 {print 100 - $16}' < $vmstat_log) )
603 save_output $vmstat_log $opref.vmstat
605 for i in `seq 0 $last_block`; do
606 read_req_s[$i]=$(mean_stddev $(awk \
607 '(NR > 1) {print $4}' < $tmpdir/iostat.$i) )
608 write_req_s[$i]=$(mean_stddev $(awk \
609 '(NR > 1) {print $5}' < $tmpdir/iostat.$i) )
610 sects_req[$i]=$(mean_stddev $(awk \
611 '(NR > 1) {print $10}' < $tmpdir/iostat.$i) )
612 queued_reqs[$i]=$(mean_stddev $(awk \
613 '(NR > 1) {print $11}' < $tmpdir/iostat.$i) )
614 service_ms[$i]=$(mean_stddev $(awk \
615 '(NR > 1) {print $13}' < $tmpdir/iostat.$i) )
617 save_output $tmpdir/iostat.$i $opref.iostat.$i
620 # record each index's test results and sum them
622 for i in `seq 0 $last_block`; do
623 local t=`${test}_result $tmpdir/$i`
624 save_output $tmpdir/$i $opref.$i
625 echo test returned "$t"
627 # some tests return mean:stddev per device, filter out stddev
628 thru=$(do_bc $thru + $(echo $t | sed -e 's/:.*$//g'))
631 for i in `seq 0 $last_block`; do
632 ${test}_cleanup $i $wor $threads
635 # tabulate the results
636 echo $test did $thru mb/s with $cpu
637 table_set $test $my_x $cur_y `do_bc_scale 2 $thru / 1`
638 table_set $test $(($my_x + 1)) $cur_y $cpu
640 for i in `seq 0 $last_block`; do
641 cur_y=$(($cur_y + 1))
642 table_set $test $(($my_x)) $cur_y ${mb_s[$i]}
643 table_set $test $(($my_x + 1)) $cur_y ${read_req_s[$i]}
644 table_set $test $(($my_x + 2)) $cur_y ${write_req_s[$i]}
645 table_set $test $(($my_x + 3)) $cur_y ${sects_req[$i]}
646 table_set $test $(($my_x + 4)) $cur_y ${queued_reqs[$i]}
647 table_set $test $(($my_x + 5)) $cur_y ${service_ms[$i]}
650 cur_y=$(($cur_y + 1))
655 local thr=$min_threads
660 for i in `seq 0 $last_block`; do
661 if ! ${test}_config $i; then
662 echo "couldn't config $test for bdev ${blocks[$i]}"
663 echo "skipping $test for all block devices"
670 for i in `seq 0 $last_block`; do
671 # don't prepare if _config already failed
672 [ ! -z "$cleanup" ] && break
673 if ! ${test}_prepare $i; then
674 echo "couldn't prepare $test for bdev ${blocks[$i]}"
675 echo "skipping $test for all block devices"
682 while [ -z "$cleanup" -a $thr -lt $(($max_threads + 1)) ]; do
683 for iosize in 64 128; do
684 table_set $test 0 $cur_y $thr
685 table_set $test 1 $cur_y $iosize
688 table_set $test 2 $cur_y $wor
689 test_one $test 3 $thr $iosize $wor
695 [ -z "$cleanup" ] && cleanup=$last_block
697 if [ "$cleanup" != -1 ]; then
698 for i in `seq $cleanup 0`; do
708 while getopts ":d:b:l:t:T:r:" opt; do
711 d) output_dir=$OPTARG ;;
713 r) run_tests=$OPTARG ;;
714 t) min_threads=$OPTARG ;;
715 T) max_threads=$OPTARG ;;
720 if [ -z "$io_len" ]; then
721 io_len=`awk '($1 == "MemTotal:"){print $2}' < /proc/meminfo`
722 [ -z "$io_len" ] && die "couldn't determine the amount of memory"
725 if [ ! -z "$output_dir" ]; then
726 [ ! -e "$output_dir" ] && "output dir $output_dir doesn't exist"
727 [ ! -d "$output_dir" ] && "output dir $output_dir isn't a directory"
730 block=`echo $block | sed -e 's/,/ /g'`
731 [ -z "$block" ] && usage "need block devices"
733 run_tests=`echo $run_tests | sed -e 's/,/ /g'`
734 [ -z "$run_tests" ] && usage "need to specify tests to run with -r"
735 for t in $run_tests; do
736 if ! echo $possible_tests | grep -q $t ; then
737 die "$t isn't one of the possible tests: $possible_tests"
741 [ $min_threads -gt $max_threads ] && \
742 die "min threads $min_threads must be <= min_threads $min_threads"
746 [ ! -e $b ] && die "block device file $b doesn't exist"
747 [ ! -b $b ] && die "$b isn't a block device"
748 dd if=$b of=/dev/null bs=8192 count=1 || \
749 die "couldn't read 8k from $b, is it alive?"
750 [ ! -b $b ] && die "$b isn't a block device"
751 last_block=$(($last_block + 1))
752 blocks[$last_block]=$b
755 tmpdir=`mktemp -d /tmp/.surveyXXXXXX` || die "couldn't create tmp dir"
757 echo each test will operate on $io_len"k"
761 for t in $run_tests; do
768 table_set $t 3 1 "MB"
769 table_set $t 4 1 "rR"
770 table_set $t 5 1 "wR"
771 table_set $t 6 1 "SR"
773 table_set $t 8 1 "ms"
776 if ! test_iterator $t; then
779 test_results="$test_results $t"
782 [ ! -z "$test_results" ] && (
784 echo "T = number of concurrent threads per device"
785 echo "L = base io operation length, in KB"
786 echo "m = IO method: read, write, or over-write"
787 echo "C = percentage CPU used, both user and system"
788 echo "MB/s = per-device throughput"
789 echo "rR = read requests issued to the device per second"
790 echo "wR = write requests issued to the device per second"
791 echo "SR = sectors per request; sectors tend to be 512 bytes"
792 echo "Q = the average number of requests queued on the device"
793 echo "ms = the average ms taken by the device to service a req"
795 echo "foo:bar represents a mean of foo with a stddev of bar"
798 for t in $test_results; do