3 # for now all the units are in 'k', but we could introduce some helpers
4 # would be nice to run tests in the background and trap signals and kill
7 # make sure devices aren't in use before going to town
8 # really use threads with iozone
9 # look into what sgp_dd is really doing, update arguments
10 # rename config/prepare/setup/cleanup/finish/teardown
11 # do something with sf and fpp iterating
12 # discard first vmstat line
16 # a temp dir that is setup and torn down for each script run
18 # so we can kill background processes as the test cleans up
19 declare -a cleanup_pids
20 # to unmount mounts in our tmpdir before removing it
21 declare -a cleanup_mounts
22 # global for completing the table. XXX this is a wart that could go
24 # a global which funcs use to get at the blocks[] array
26 # prefix to run oprofile or readprofile
30 # defaults for some options:
33 possible_tests="sgp_dd ext2_iozone echo_filter"
34 run_tests="$possible_tests"
37 # optional output directory
46 [ -e $path ] || continue;
47 [ -f $path ] || die "needed to remove non-file $path"
48 rm -f $path || die "couldn't remove $path"
52 [ ! -z "$output_dir" ] && mv -f $1 $output_dir/$2
55 # only cleanup test runs if we have block devices
56 if [ $last_block != -1 ]; then
57 for pid in ${cleanup_pids[*]}; do
61 for a in ${cleanup_mounts[*]}; do
66 [ ${#tmpdir} == 18 ] && [ -d $tmpdir ] && rm -rf $tmpdir
72 cleanup_pids[$pid]=$pid
76 unset cleanup_pids[$pid]
80 echo $* | sed -e 's/ /,/g'
85 echo "scale=$scale; $*" | bc
100 0) echo '??' ; return ;;
101 1) echo "$avg:0" ; return ;;
104 avg=`do_bc $avg / $num`
107 local dev=`do_bc \($p - $avg\) \^ 2`
108 tmp=`do_bc $tmp + $dev`
110 tmp=`do_bc_scale 1 sqrt \( $tmp / \($num - 1\) \)`
111 avg=`do_bc_scale 1 $avg / 1`
117 echo " -b <block device to profile>"
118 echo " -d <summary output directory>"
119 echo " -l <max io len>"
120 echo " -t <minimum number of threads per device>"
121 echo " -T <maximum number of threads per device>"
122 echo " -r <tests to run>"
126 # some cute code for handling tables whose columns fit
131 if [ $val -gt ${!target:-0} ]; then
136 local name="_table_$1"
142 eval ${name}_${row}_${col}="'$val'"
144 set_max ${name}_${col}_longest ${#val}
145 set_max ${name}_num_col $(($col + 1))
146 set_max ${name}_num_row $(($row + 1))
150 local name="_table_$1"
153 tmp="${name}_${row}_${col}"
158 local name="_table_$1"
165 tmp="${name}_num_col"
167 tmp="${name}_num_row"
170 # iterate through the columns to find the longest
173 for x in `seq 0 $num_col`; do
174 tmp="${name}_${x}_longest"
176 [ $tmp -eq 0 ] && continue
178 [ $x -eq $((num_col - 1)) ] && sep='\n'
180 fmt="$fmt%-${tmp}s$sep"
183 # nothing in the table to print
184 [ -z "$fmt" ] && return
186 for y in `seq 0 $num_row`; do
188 for x in `seq 0 $num_col`; do
190 # skip this element if the column is empty
191 tmp="${name}_${x}_longest"
192 [ ${!tmp:-0} -eq 0 ] && continue
194 # fill this cell with the value or '' for printf
195 tmp="${name}_${y}_${x}"
196 row="$row'${!tmp:-""}' "
198 eval printf "'$fmt'" $row
202 ######################################################################
205 echo sgp_dd using dio=1 and thr=
208 # it could be making sure that the block dev
209 # isn't in use by something else
213 if ! which sgp_dd; then
214 echo "can't find sgp_dd binary"
220 # it could be making sure that the block dev
221 # isn't in use by something else
230 local bdev=${blocks[$i]};
233 [wo]) ifof="if=/dev/zero of=$bdev" ;;
234 r) ifof="if=$bdev of=/dev/null" ;;
235 *) die "asked to do io with $wor?"
237 echo sgp_dd $ifof bs=$iosize"k" count=$(($io_len / $iosize)) time=1 \
243 awk '($(NF) == "MB/sec") {print $(NF-1)}' < $output
258 ######################################################################
260 ext2_iozone_banner() {
261 echo "iozone -I on a clean ext2 fs"
263 ext2_iozone_config() {
266 ext2_iozone_prepare() {
268 local bdev=${blocks[$index]}
269 local mntpnt=$tmpdir/mount_$index
271 if ! which iozone; then
272 echo "iozone binary not found in PATH"
275 if ! iozone -i 0 -w -+o -s 1k -r 1k -f /dev/null > /dev/null; then
276 echo "iozone doesn't support -+o"
279 if ! which mke2fs; then
280 echo "mke2fs binary not found in PATH"
284 if ! mkdir -p $mntpnt ; then
285 echo "$mntpnt isn't a directory?"
288 echo making ext2 filesystem on $bdev
289 if ! mke2fs -b 4096 $bdev; then
294 if ! mount -t ext2 $bdev $mntpnt; then
295 echo "couldn't mount $bdev on $mntpnt"
299 cleanup_mounts[$index]="$mntpnt"
302 ext2_iozone_setup() {
305 local f="$tmpdir/mount_$id/iozone"
310 *) die "asked to do io with $wor?"
313 ext2_iozone_start() {
319 local f="$tmpdir/mount_$id/iozone"
322 [wo]) args="-i 0 -w" ;;
324 *) die "asked to do io with $wor?"
327 echo iozone "$args -r ${iosize}k -s $(($io_len / $threads))k \
328 -t $threads -+o -x -I -f $f"
330 ext2_iozone_result() {
337 [wo]) string="writers"
343 *) die "asked to do io with $wor?"
346 do_bc_scale 1 `awk '($1 == "Parent" && $'$field' == "'$string'") \
347 {print $'$(($field + 2))'}' $output` / 1024
349 ext2_iozone_cleanup() {
350 # the final read w/o -w removed the file
353 ext2_iozone_finish() {
355 local mntpnt=$tmpdir/mount_$index
358 unset cleanup_mounts[$index]
360 ext2_iozone_teardown() {
364 ######################################################################
365 # the lctl test_brw via the echo_client on top of the filter
367 # the echo_client setup is nutty enough to warrant its own clenaup
370 declare -a running_names
371 declare -a running_oids
373 cleanup_echo_filter() {
376 for i in `seq 0 $last_block`; do
377 [ -z "${running_oids[$i]}" ] && continue
378 lctl --device "\$"echo_$i destroy ${running_oids[$i]} \
383 for n in ${running_names[*]}; do
384 # I can't believe leading whitespace matters here.
394 for m in $running_module; do
399 [ ! -z "$running_config" ] && lconf --cleanup $running_config
403 echo_filter_banner() {
404 echo "test_brw on the echo_client on the filter"
406 echo_filter_config() {
408 local bdev=${blocks[$index]}
409 local config="$tmpdir/config.xml"
412 echo "lmc binary not found in PATH"
415 if ! which lconf; then
416 echo "lconf binary not found in PATH"
419 if ! which lctl; then
420 echo "lctl binary not found in PATH"
424 if [ $index = 0 ]; then
425 if ! lmc -m $config --add net \
426 --node $HOSTNAME --nid $HOSTNAME --nettype tcp; then
427 echo "error adding $HOSTNAME net node"
432 if ! lmc -m $config --add ost --ost ost_$index --node $HOSTNAME \
433 --fstype ext3 --dev $bdev --journal_size 400; then
434 echo "error adding $bdev to config with lmc"
438 # it would be nice to be able to ask lmc to setup an echo client
439 # to the filter here. --add echo_client assumes osc
441 echo_filter_prepare() {
443 local bdev=${blocks[$index]}
444 local config="$tmpdir/config.xml"
445 local name="echo_$index"
446 local uuid="echo_$index_uuid"
448 if [ $index = 0 ]; then
449 if ! lconf --reformat $config; then
450 echo "error setting up with lconf"
453 running_config="$config"
455 echo 0 > /proc/sys/lnet/debug
456 echo 0 > /proc/sys/lnet/subsystem_debug
458 if ! grep -q '^obdecho\>' /proc/modules; then
460 if ! modprobe obdecho; then
461 if [ ! -z "$echo_module" ]; then
462 if ! insmod $echo_module; then
463 echo "err: insmod $echo_module"
469 echo "err: modprobe $obdecho"
475 running_module=`basename $m | cut -d'.' -f 1`
481 attach echo_client $name $uuid
486 echo "error setting up echo_client $name against ost_$index"
489 running_names[$index]=$name
491 echo_filter_setup() {
495 local name="echo_$id"
501 *) die "asked to do io with $wor?"
504 running_threads=$threads
505 oid=`lctl --device "\$"$name create $threads | \
506 awk '/ #1 is object id/ { print $6 }'`
507 # XXX need to deal with errors
508 running_oids[$id]=$oid
510 echo_filter_start() {
517 local name="echo_$id"
518 local len_pages=$(($io_len / $(($page_size / 1024)) / $threads ))
519 local size_pages=$(($iosize / $(($page_size / 1024)) ))
524 *) die "asked to do io with $wor?"
527 echo lctl --threads $threads v "\$"$name \
528 test_brw 1 $rw v $len_pages t${running_oids[$id]} p$size_pages
530 echo_filter_result() {
535 for mbs in `awk '($8=="MB/s):"){print substr($7,2)}' < $output`; do
536 total=$(do_bc $total + $mbs)
538 do_bc_scale 2 $total / 1
540 echo_filter_cleanup() {
544 local name="echo_$id"
549 *) die "asked to do io with $wor?"
552 lctl --device "\$"$name destroy ${running_oids[$id]} $threads
553 unset running_oids[$id]
555 echo_filter_finish() {
557 # leave real work for _teardown
559 echo_filter_teardown() {
563 ######################################################################
564 # the iteration that drives the tests
573 local vmstat_log="$tmpdir/vmstat.log"
574 local opref="$test-$threads-$iosize-$wor"
576 # sigh. but this makes it easier to dump into the tables
584 for i in `seq 0 $last_block`; do
585 ${test}_setup $i $wor $threads
588 echo $test with $threads threads
590 $oprofile opcontrol --start
592 # start up vmstat and record its pid
593 nice -19 vmstat 1 > $vmstat_log 2>&1 &
594 [ $? = 0 ] || die "vmstat failed"
596 pid_now_running $vmstat_pid
598 # start up each block device's iostat
599 for i in `seq 0 $last_block`; do
600 nice -19 iostat -x ${blocks[$i]} 1 | awk \
601 '($1 == "'${blocks[$i]}'"){print $0; fflush()}' \
602 > $tmpdir/iostat.$i &
608 $oprofile opcontrol --reset
611 # start all the tests. each returns a pid to wait on
613 for i in `seq 0 $last_block`; do
614 local cmd=`${test}_start $threads $iosize $wor $i`
615 echo "$cmd" >> $tmpdir/commands
616 $cmd > $tmpdir/$i 2>&1 &
622 echo -n waiting on pids $pids:
629 # stop vmstat and all the iostats
631 pid_has_stopped $vmstat_pid
632 for i in `seq 0 $last_block`; do
633 local pid=${iostat_pids[$i]}
634 [ -z "$pid" ] && continue
637 unset iostat_pids[$i]
641 $readprofile | sort -rn > $tmpdir/readprofile
643 $oprofile opcontrol --shutdown
644 $oprofile opreport > $tmpdir/oprofile
645 echo >> $tmpdir/oprofile
646 $oprofile opreport -c -l | head -20 >> $tmpdir/oprofile
648 save_output $tmpdir/oprofile $opref.oprofile
649 save_output $tmpdir/readprofile $opref.readprofile
651 # collect the results of vmstat and iostat
652 cpu=$(mean_stddev $(awk \
653 '(NR > 3 && NF == 16 && $16 != "id" ) \
654 {print 100 - $16}' < $vmstat_log) )
655 save_output $vmstat_log $opref.vmstat
657 for i in `seq 0 $last_block`; do
658 read_req_s[$i]=$(mean_stddev $(awk \
659 '(NR > 1) {print $4}' < $tmpdir/iostat.$i) )
660 write_req_s[$i]=$(mean_stddev $(awk \
661 '(NR > 1) {print $5}' < $tmpdir/iostat.$i) )
662 sects_req[$i]=$(mean_stddev $(awk \
663 '(NR > 1) {print $10}' < $tmpdir/iostat.$i) )
664 queued_reqs[$i]=$(mean_stddev $(awk \
665 '(NR > 1) {print $11}' < $tmpdir/iostat.$i) )
666 service_ms[$i]=$(mean_stddev $(awk \
667 '(NR > 1) {print $13}' < $tmpdir/iostat.$i) )
669 save_output $tmpdir/iostat.$i $opref.iostat.$i
672 # record each index's test results and sum them
674 for i in `seq 0 $last_block`; do
675 local t=`${test}_result $tmpdir/$i $wor`
676 save_output $tmpdir/$i $opref.$i
677 echo test returned "$t"
679 # some tests return mean:stddev per device, filter out stddev
680 thru=$(do_bc $thru + $(echo $t | sed -e 's/:.*$//g'))
683 for i in `seq 0 $last_block`; do
684 ${test}_cleanup $i $wor $threads
687 # tabulate the results
688 echo $test did $thru mb/s with $cpu
689 table_set $test $my_x $cur_y `do_bc_scale 2 $thru / 1`
690 table_set $test $(($my_x + 1)) $cur_y $cpu
692 for i in `seq 0 $last_block`; do
693 cur_y=$(($cur_y + 1))
694 table_set $test $(($my_x)) $cur_y ${mb_s[$i]}
695 table_set $test $(($my_x + 1)) $cur_y ${read_req_s[$i]}
696 table_set $test $(($my_x + 2)) $cur_y ${write_req_s[$i]}
697 table_set $test $(($my_x + 3)) $cur_y ${sects_req[$i]}
698 table_set $test $(($my_x + 4)) $cur_y ${queued_reqs[$i]}
699 table_set $test $(($my_x + 5)) $cur_y ${service_ms[$i]}
702 cur_y=$(($cur_y + 1))
707 local thr=$min_threads
712 for i in `seq 0 $last_block`; do
713 if ! ${test}_config $i; then
714 echo "couldn't config $test for bdev ${blocks[$i]}"
715 echo "skipping $test for all block devices"
722 for i in `seq 0 $last_block`; do
723 # don't prepare if _config already failed
724 [ ! -z "$cleanup" ] && break
725 if ! ${test}_prepare $i; then
726 echo "couldn't prepare $test for bdev ${blocks[$i]}"
727 echo "skipping $test for all block devices"
734 while [ -z "$cleanup" -a $thr -lt $(($max_threads + 1)) ]; do
735 for iosize in 128 512; do
736 table_set $test 0 $cur_y $thr
737 table_set $test 1 $cur_y $iosize
740 table_set $test 2 $cur_y $wor
741 test_one $test 3 $thr $iosize $wor
747 [ -z "$cleanup" ] && cleanup=$last_block
749 if [ "$cleanup" != -1 ]; then
750 for i in `seq $cleanup 0`; do
760 while getopts ":d:b:l:t:T:r:e:" opt; do
762 e) echo_module=$OPTARG ;;
764 d) output_dir=$OPTARG ;;
766 r) run_tests=$OPTARG ;;
767 t) min_threads=$OPTARG ;;
768 T) max_threads=$OPTARG ;;
773 page_size=`getconf PAGE_SIZE` || die '"getconf PAGE_SIZE" failed'
775 [ ! -z "$echo_module" -a ! -f "$echo_module" ] && \
776 die "obdecho module $echo_module is not a file"
778 if [ -z "$io_len" ]; then
779 io_len=`awk '($1 == "MemTotal:"){print $2}' < /proc/meminfo`
780 [ -z "$io_len" ] && die "couldn't determine the amount of memory"
783 if [ ! -z "$output_dir" ]; then
784 if [ ! -e "$output_dir" ]; then
785 mkdir -p "$output_dir" || die "error creating $output_dir"
787 [ ! -d "$output_dir" ] && die "$output_dir isn't a directory"
790 block=`echo $block | sed -e 's/,/ /g'`
791 [ -z "$block" ] && usage "need block devices"
793 run_tests=`echo $run_tests | sed -e 's/,/ /g'`
794 [ -z "$run_tests" ] && usage "need to specify tests to run with -r"
795 for t in $run_tests; do
796 if ! echo $possible_tests | grep -q $t ; then
797 die "$t isn't one of the possible tests: $possible_tests"
801 if which opcontrol; then
802 echo generating oprofile results
805 echo not using oprofile
809 if which readprofile; then
810 map="/boot/System.map-`uname -r`"
811 if [ -f /proc/profile -a -f "$map" ]; then
812 echo generating profiles with 'readprofile'
813 readprofile="readprofile -m $map"
816 if [ -z "$readprofile" ]; then
817 echo not using readprofile
821 [ $min_threads -gt $max_threads ] && \
822 die "min threads $min_threads must be <= min_threads $min_threads"
825 [ ! -e $b ] && die "block device file $b doesn't exist"
826 [ ! -b $b ] && die "$b isn't a block device"
827 dd if=$b of=/dev/null bs=8192 count=1 || \
828 die "couldn't read 8k from $b, is it alive?"
829 [ ! -b $b ] && die "$b isn't a block device"
830 last_block=$(($last_block + 1))
831 blocks[$last_block]=$b
834 tmpdir=`mktemp -d /tmp/.surveyXXXXXX` || die "couldn't create tmp dir"
836 echo each test will operate on $io_len"k"
840 for t in $run_tests; do
847 table_set $t 3 1 "MB"
848 table_set $t 4 1 "rR"
849 table_set $t 5 1 "wR"
850 table_set $t 6 1 "SR"
852 table_set $t 8 1 "ms"
855 if ! test_iterator $t; then
858 test_results="$test_results $t"
861 save_output $tmpdir/commands commands
863 [ ! -z "$test_results" ] && (
865 echo "T = number of concurrent threads per device"
866 echo "L = base io operation length, in KB"
867 echo "m = IO method: read, write, or over-write"
868 echo "A = aggregate throughput from all devices"
869 echo "C = percentage CPU used, both user and system"
870 echo "MB/s = per-device throughput"
871 echo "rR = read requests issued to the device per second"
872 echo "wR = write requests issued to the device per second"
873 echo "SR = sectors per request; sectors tend to be 512 bytes"
874 echo "Q = the average number of requests queued on the device"
875 echo "ms = the average ms taken by the device to service a req"
877 echo "foo:bar represents a mean of foo with a stddev of bar"
880 for t in $test_results; do