3 # for now all the units are in 'k', but we could introduce some helpers
4 # would be nice to run tests in the background and trap signals and kill
7 # make sure devices aren't in use before going to town
8 # really use threads with iozone
9 # look into what sgp_dd is really doing, update arguments
10 # rename config/prepare/setup/cleanup/finish/teardown
11 # do something with sf and fpp iterating
12 # discard first vmstat line
15 # a temp dir that is setup and torn down for each script run
17 # so we can kill background processes as the test cleans up
18 declare -a cleanup_pids
19 # to unmount mounts in our tmpdir before removing it
20 declare -a cleanup_mounts
21 # global for completing the table. XXX this is a wart that could go
23 # a global which funcs use to get at the blocks[] array
26 # defaults for some options:
29 possible_tests="sgp_dd ext2_iozone echo_filter"
30 run_tests="$possible_tests"
33 # optional output directory
42 [ -e $path ] || continue;
43 [ -f $path ] || die "needed to remove non-file $path"
44 rm -f $path || die "couldn't remove $path"
48 [ ! -z "$output_dir" ] && mv -f $1 $output_dir/$2
51 # only cleanup test runs if we have block devices
52 if [ $last_block != -1 ]; then
53 for pid in ${cleanup_pids[*]}; do
57 for a in ${cleanup_mounts[*]}; do
62 [ ${#tmpdir} == 18 ] && [ -d $tmpdir ] && rm -rf $tmpdir
68 cleanup_pids[$pid]=$pid
72 unset cleanup_pids[$pid]
76 echo $* | sed -e 's/ /,/g'
81 echo "scale=$scale; $*" | bc
96 0) echo '??' ; return ;;
97 1) echo "$avg:0" ; return ;;
100 avg=`do_bc $avg / $num`
103 local dev=`do_bc \($p - $avg\) \^ 2`
104 tmp=`do_bc $tmp + $dev`
106 tmp=`do_bc_scale 1 sqrt \( $tmp / \($num - 1\) \)`
107 avg=`do_bc_scale 1 $avg / 1`
113 echo " -b <block device to profile>"
114 echo " -d <summary output directory>"
115 echo " -l <max io len>"
116 echo " -t <minimum number of threads per device>"
117 echo " -T <maximum number of threads per device>"
118 echo " -r <tests to run>"
122 # some cute code for handling tables whose columns fit
127 if [ $val -gt ${!target:-0} ]; then
132 local name="_table_$1"
138 eval ${name}_${row}_${col}="'$val'"
140 set_max ${name}_${col}_longest ${#val}
141 set_max ${name}_num_col $(($col + 1))
142 set_max ${name}_num_row $(($row + 1))
146 local name="_table_$1"
149 tmp="${name}_${row}_${col}"
154 local name="_table_$1"
161 tmp="${name}_num_col"
163 tmp="${name}_num_row"
166 # iterate through the columns to find the longest
169 for x in `seq 0 $num_col`; do
170 tmp="${name}_${x}_longest"
172 [ $tmp -eq 0 ] && continue
174 [ $x -eq $((num_col - 1)) ] && sep='\n'
176 fmt="$fmt%-${tmp}s$sep"
179 # nothing in the table to print
180 [ -z "$fmt" ] && return
182 for y in `seq 0 $num_row`; do
184 for x in `seq 0 $num_col`; do
186 # skip this element if the column is empty
187 tmp="${name}_${x}_longest"
188 [ ${!tmp:-0} -eq 0 ] && continue
190 # fill this cell with the value or '' for printf
191 tmp="${name}_${y}_${x}"
192 row="$row'${!tmp:-""}' "
194 eval printf "'$fmt'" $row
198 ######################################################################
201 echo sgp_dd using dio=1 and thr=
204 # it could be making sure that the block dev
205 # isn't in use by something else
209 if ! which sgp_dd; then
210 echo "can't find sgp_dd binary"
216 # it could be making sure that the block dev
217 # isn't in use by something else
226 local bdev=${blocks[$i]};
229 [wo]) ifof="if=/dev/zero of=$bdev" ;;
230 r) ifof="if=$bdev of=/dev/null" ;;
231 *) die "asked to do io with $wor?"
233 echo sgp_dd $ifof bs=$iosize"k" count=$(($io_len / $iosize)) time=1 \
239 awk '($(NF) == "MB/sec") {print $(NF-1)}' < $output
254 ######################################################################
256 ext2_iozone_banner() {
257 echo "iozone -I on a clean ext2 fs"
259 ext2_iozone_config() {
262 ext2_iozone_prepare() {
264 local bdev=${blocks[$index]}
265 local mntpnt=$tmpdir/mount_$index
267 if ! which iozone; then
268 echo "iozone binary not found in PATH"
271 if ! iozone -i 0 -w -+o -s 1k -r 1k -f /dev/null > /dev/null; then
272 echo "iozone doesn't support -+o"
275 if ! which mke2fs; then
276 echo "mke2fs binary not found in PATH"
280 if ! mkdir -p $mntpnt ; then
281 echo "$mntpnt isn't a directory?"
284 echo making ext2 filesystem on $bdev
285 if ! mke2fs -b 4096 $bdev; then
290 if ! mount -t ext2 $bdev $mntpnt; then
291 echo "couldn't mount $bdev on $mntpnt"
295 cleanup_mounts[$index]="$mntpnt"
298 ext2_iozone_setup() {
301 local f="$tmpdir/mount_$id/iozone"
306 *) die "asked to do io with $wor?"
309 ext2_iozone_start() {
315 local f="$tmpdir/mount_$id/iozone"
318 [wo]) args="-i 0 -w" ;;
320 *) die "asked to do io with $wor?"
323 echo iozone "$args -r ${iosize}k -s $(($io_len / $threads))k \
324 -t $threads -+o -x -I -f $f"
326 ext2_iozone_result() {
333 [wo]) string="writers"
339 *) die "asked to do io with $wor?"
342 do_bc_scale 1 `awk '($1 == "Parent" && $'$field' == "'$string'") \
343 {print $'$(($field + 2))'}' $output` / 1024
345 ext2_iozone_cleanup() {
346 # the final read w/o -w removed the file
349 ext2_iozone_finish() {
351 local mntpnt=$tmpdir/mount_$index
354 unset cleanup_mounts[$index]
356 ext2_iozone_teardown() {
360 ######################################################################
361 # the lctl test_brw via the echo_client on top of the filter
363 # the echo_client setup is nutty enough to warrant its own clenaup
366 declare -a running_names
367 declare -a running_oids
369 cleanup_echo_filter() {
372 for i in `seq 0 $last_block`; do
373 [ -z "${running_oids[$i]}" ] && continue
374 lctl --device "\$"echo_$i destroy ${running_oids[$i]} \
379 for n in ${running_names[*]}; do
380 # I can't believe leading whitespace matters here.
390 for m in $running_module; do
395 [ ! -z "$running_config" ] && lconf --cleanup $running_config
399 echo_filter_banner() {
400 echo "test_brw on the echo_client on the filter"
402 echo_filter_config() {
404 local bdev=${blocks[$index]}
405 local config="$tmpdir/config.xml"
408 echo "lmc binary not found in PATH"
411 if ! which lconf; then
412 echo "lconf binary not found in PATH"
415 if ! which lctl; then
416 echo "lctl binary not found in PATH"
420 if [ $index = 0 ]; then
421 if ! lmc -m $config --add net \
422 --node localhost --nid localhost --nettype tcp; then
423 echo "error adding localhost net node"
428 if ! lmc -m $config --add ost --ost ost_$index --node localhost \
429 --fstype ext3 --dev $bdev --journal_size 400; then
430 echo "error adding $bdev to config with lmc"
434 # it would be nice to be able to ask lmc to setup an echo client
435 # to the filter here. --add echo_client assumes osc
437 echo_filter_prepare() {
439 local bdev=${blocks[$index]}
440 local config="$tmpdir/config.xml"
441 local name="echo_$index"
442 local uuid="echo_$index_uuid"
444 if [ $index = 0 ]; then
445 if ! lconf --reformat $config; then
446 echo "error setting up with lconf"
449 running_config="$config"
450 if ! grep -q '^obdecho\>' /proc/modules; then
452 if ! modprobe obdecho; then
453 if [ ! -z "$echo_module" ]; then
454 if ! insmod $echo_module; then
455 echo "err: insmod $echo_module"
461 echo "err: modprobe $obdecho"
467 running_module=`basename $m | cut -d'.' -f 1`
473 attach echo_client $name $uuid
478 echo "error setting up echo_client $name against ost_$index"
481 running_names[$index]=$name
483 echo_filter_setup() {
487 local name="echo_$id"
493 *) die "asked to do io with $wor?"
496 running_threads=$threads
497 oid=`lctl --device "\$"$name create $threads | \
498 awk '/ #1 is object id/ { print $6 }'`
499 # XXX need to deal with errors
500 running_oids[$id]=$oid
502 echo_filter_start() {
509 local name="echo_$id"
510 local len_pages=$(($io_len / $(($page_size / 1024)) / $threads ))
511 local size_pages=$(($iosize / $(($page_size / 1024)) ))
516 *) die "asked to do io with $wor?"
519 echo lctl --threads $threads v "\$"$name \
520 test_brw 1 $rw v $len_pages t${running_oids[$id]} p$size_pages
522 echo_filter_result() {
527 for mbs in `awk '($8=="MB/s):"){print substr($7,2)}' < $output`; do
528 total=$(do_bc $total + $mbs)
530 do_bc_scale 2 $total / 1
532 echo_filter_cleanup() {
536 local name="echo_$id"
541 *) die "asked to do io with $wor?"
544 lctl --device "\$"$name destroy ${running_oids[$id]} $threads
545 unset running_oids[$id]
547 echo_filter_finish() {
549 # leave real work for _teardown
551 echo_filter_teardown() {
555 ######################################################################
556 # the iteration that drives the tests
565 local vmstat_log="$tmpdir/vmstat.log"
566 local opref="$test-$threads-$iosize-$wor"
568 # sigh. but this makes it easier to dump into the tables
576 for i in `seq 0 $last_block`; do
577 ${test}_setup $i $wor $threads
580 echo $test with $threads threads
582 $oprofile opcontrol --start
584 # start up vmstat and record its pid
585 nice -19 vmstat 1 > $vmstat_log 2>&1 &
586 [ $? = 0 ] || die "vmstat failed"
588 pid_now_running $vmstat_pid
590 # start up each block device's iostat
591 for i in `seq 0 $last_block`; do
592 nice -19 iostat -x ${blocks[$i]} 1 | awk \
593 '($1 == "'${blocks[$i]}'"){print $0; fflush()}' \
594 > $tmpdir/iostat.$i &
600 $oprofile opcontrol --reset
602 # start all the tests. each returns a pid to wait on
604 for i in `seq 0 $last_block`; do
605 local cmd=`${test}_start $threads $iosize $wor $i`
606 $cmd > $tmpdir/$i 2>&1 &
612 echo -n waiting on pids $pids:
619 # stop vmstat and all the iostats
621 pid_has_stopped $vmstat_pid
622 for i in `seq 0 $last_block`; do
623 local pid=${iostat_pids[$i]}
624 [ -z "$pid" ] && continue
627 unset iostat_pids[$i]
631 $oprofile opcontrol --shutdown
632 $oprofile opreport > $tmpdir/oprofile
633 echo >> $tmpdir/oprofile
634 $oprofile opreport -c -l | head -20 >> $tmpdir/oprofile
635 save_output $tmpdir/oprofile $opref.oprofile
637 # collect the results of vmstat and iostat
638 cpu=$(mean_stddev $(awk \
639 '(NR > 3 && NF == 16 && $16 != "id" ) \
640 {print 100 - $16}' < $vmstat_log) )
641 save_output $vmstat_log $opref.vmstat
643 for i in `seq 0 $last_block`; do
644 read_req_s[$i]=$(mean_stddev $(awk \
645 '(NR > 1) {print $4}' < $tmpdir/iostat.$i) )
646 write_req_s[$i]=$(mean_stddev $(awk \
647 '(NR > 1) {print $5}' < $tmpdir/iostat.$i) )
648 sects_req[$i]=$(mean_stddev $(awk \
649 '(NR > 1) {print $10}' < $tmpdir/iostat.$i) )
650 queued_reqs[$i]=$(mean_stddev $(awk \
651 '(NR > 1) {print $11}' < $tmpdir/iostat.$i) )
652 service_ms[$i]=$(mean_stddev $(awk \
653 '(NR > 1) {print $13}' < $tmpdir/iostat.$i) )
655 save_output $tmpdir/iostat.$i $opref.iostat.$i
658 # record each index's test results and sum them
660 for i in `seq 0 $last_block`; do
661 local t=`${test}_result $tmpdir/$i $wor`
662 save_output $tmpdir/$i $opref.$i
663 echo test returned "$t"
665 # some tests return mean:stddev per device, filter out stddev
666 thru=$(do_bc $thru + $(echo $t | sed -e 's/:.*$//g'))
669 for i in `seq 0 $last_block`; do
670 ${test}_cleanup $i $wor $threads
673 # tabulate the results
674 echo $test did $thru mb/s with $cpu
675 table_set $test $my_x $cur_y `do_bc_scale 2 $thru / 1`
676 table_set $test $(($my_x + 1)) $cur_y $cpu
678 for i in `seq 0 $last_block`; do
679 cur_y=$(($cur_y + 1))
680 table_set $test $(($my_x)) $cur_y ${mb_s[$i]}
681 table_set $test $(($my_x + 1)) $cur_y ${read_req_s[$i]}
682 table_set $test $(($my_x + 2)) $cur_y ${write_req_s[$i]}
683 table_set $test $(($my_x + 3)) $cur_y ${sects_req[$i]}
684 table_set $test $(($my_x + 4)) $cur_y ${queued_reqs[$i]}
685 table_set $test $(($my_x + 5)) $cur_y ${service_ms[$i]}
688 cur_y=$(($cur_y + 1))
693 local thr=$min_threads
698 for i in `seq 0 $last_block`; do
699 if ! ${test}_config $i; then
700 echo "couldn't config $test for bdev ${blocks[$i]}"
701 echo "skipping $test for all block devices"
708 for i in `seq 0 $last_block`; do
709 # don't prepare if _config already failed
710 [ ! -z "$cleanup" ] && break
711 if ! ${test}_prepare $i; then
712 echo "couldn't prepare $test for bdev ${blocks[$i]}"
713 echo "skipping $test for all block devices"
720 while [ -z "$cleanup" -a $thr -lt $(($max_threads + 1)) ]; do
721 for iosize in 128 512; do
722 table_set $test 0 $cur_y $thr
723 table_set $test 1 $cur_y $iosize
726 table_set $test 2 $cur_y $wor
727 test_one $test 3 $thr $iosize $wor
733 [ -z "$cleanup" ] && cleanup=$last_block
735 if [ "$cleanup" != -1 ]; then
736 for i in `seq $cleanup 0`; do
746 while getopts ":d:b:l:t:T:r:e:" opt; do
748 e) echo_module=$OPTARG ;;
750 d) output_dir=$OPTARG ;;
752 r) run_tests=$OPTARG ;;
753 t) min_threads=$OPTARG ;;
754 T) max_threads=$OPTARG ;;
759 page_size=`getconf PAGE_SIZE` || die '"getconf PAGE_SIZE" failed'
761 [ ! -z "$echo_module" -a ! -f "$echo_module" ] && \
762 die "obdecho module $echo_module is not a file"
764 if [ -z "$io_len" ]; then
765 io_len=`awk '($1 == "MemTotal:"){print $2}' < /proc/meminfo`
766 [ -z "$io_len" ] && die "couldn't determine the amount of memory"
769 if [ ! -z "$output_dir" ]; then
770 if [ ! -e "$output_dir" ]; then
771 mkdir -p "$output_dir" || die "error creating $output_dir"
773 [ ! -d "$output_dir" ] && die "$output_dir isn't a directory"
776 block=`echo $block | sed -e 's/,/ /g'`
777 [ -z "$block" ] && usage "need block devices"
779 run_tests=`echo $run_tests | sed -e 's/,/ /g'`
780 [ -z "$run_tests" ] && usage "need to specify tests to run with -r"
781 for t in $run_tests; do
782 if ! echo $possible_tests | grep -q $t ; then
783 die "$t isn't one of the possible tests: $possible_tests"
787 if which opcontrol; then
788 echo generating oprofile results
791 echo not using oprofile
795 [ $min_threads -gt $max_threads ] && \
796 die "min threads $min_threads must be <= min_threads $min_threads"
799 [ ! -e $b ] && die "block device file $b doesn't exist"
800 [ ! -b $b ] && die "$b isn't a block device"
801 dd if=$b of=/dev/null bs=8192 count=1 || \
802 die "couldn't read 8k from $b, is it alive?"
803 [ ! -b $b ] && die "$b isn't a block device"
804 last_block=$(($last_block + 1))
805 blocks[$last_block]=$b
808 tmpdir=`mktemp -d /tmp/.surveyXXXXXX` || die "couldn't create tmp dir"
810 echo each test will operate on $io_len"k"
814 for t in $run_tests; do
821 table_set $t 3 1 "MB"
822 table_set $t 4 1 "rR"
823 table_set $t 5 1 "wR"
824 table_set $t 6 1 "SR"
826 table_set $t 8 1 "ms"
829 if ! test_iterator $t; then
832 test_results="$test_results $t"
835 [ ! -z "$test_results" ] && (
837 echo "T = number of concurrent threads per device"
838 echo "L = base io operation length, in KB"
839 echo "m = IO method: read, write, or over-write"
840 echo "A = aggregate throughput from all devices"
841 echo "C = percentage CPU used, both user and system"
842 echo "MB/s = per-device throughput"
843 echo "rR = read requests issued to the device per second"
844 echo "wR = write requests issued to the device per second"
845 echo "SR = sectors per request; sectors tend to be 512 bytes"
846 echo "Q = the average number of requests queued on the device"
847 echo "ms = the average ms taken by the device to service a req"
849 echo "foo:bar represents a mean of foo with a stddev of bar"
852 for t in $test_results; do