3 # for now all the units are in 'k', but we could introduce some helpers
4 # would be nice to run tests in the background and trap signals and kill
7 # make sure devices aren't in use before going to town
8 # really use threads with iozone
9 # look into what sgp_dd is really doing, update arguments
10 # rename config/prepare/setup/cleanup/finish/teardown
11 # do something with sf and fpp iterating
12 # discard first vmstat line
15 # a temp dir that is setup and torn down for each script run
17 # so we can kill background processes as the test cleans up
18 declare -a cleanup_pids
19 # to unmount mounts in our tmpdir before removing it
20 declare -a cleanup_mounts
21 # global for completing the table. XXX this is a wart that could go
23 # a global which funcs use to get at the blocks[] array
25 # prefix to run oprofile or readprofile
29 # defaults for some options:
32 possible_tests="sgp_dd ext2_iozone echo_filter"
33 run_tests="$possible_tests"
36 # optional output directory
45 [ -e $path ] || continue;
46 [ -f $path ] || die "needed to remove non-file $path"
47 rm -f $path || die "couldn't remove $path"
51 [ ! -z "$output_dir" ] && mv -f $1 $output_dir/$2
54 # only cleanup test runs if we have block devices
55 if [ $last_block != -1 ]; then
56 for pid in ${cleanup_pids[*]}; do
60 for a in ${cleanup_mounts[*]}; do
65 [ ${#tmpdir} == 18 ] && [ -d $tmpdir ] && rm -rf $tmpdir
71 cleanup_pids[$pid]=$pid
75 unset cleanup_pids[$pid]
79 echo $* | sed -e 's/ /,/g'
84 echo "scale=$scale; $*" | bc
99 0) echo '??' ; return ;;
100 1) echo "$avg:0" ; return ;;
103 avg=`do_bc $avg / $num`
106 local dev=`do_bc \($p - $avg\) \^ 2`
107 tmp=`do_bc $tmp + $dev`
109 tmp=`do_bc_scale 1 sqrt \( $tmp / \($num - 1\) \)`
110 avg=`do_bc_scale 1 $avg / 1`
116 echo " -b <block device to profile>"
117 echo " -d <summary output directory>"
118 echo " -l <max io len>"
119 echo " -t <minimum number of threads per device>"
120 echo " -T <maximum number of threads per device>"
121 echo " -r <tests to run>"
125 # some cute code for handling tables whose columns fit
130 if [ $val -gt ${!target:-0} ]; then
135 local name="_table_$1"
141 eval ${name}_${row}_${col}="'$val'"
143 set_max ${name}_${col}_longest ${#val}
144 set_max ${name}_num_col $(($col + 1))
145 set_max ${name}_num_row $(($row + 1))
149 local name="_table_$1"
152 tmp="${name}_${row}_${col}"
157 local name="_table_$1"
164 tmp="${name}_num_col"
166 tmp="${name}_num_row"
169 # iterate through the columns to find the longest
172 for x in `seq 0 $num_col`; do
173 tmp="${name}_${x}_longest"
175 [ $tmp -eq 0 ] && continue
177 [ $x -eq $((num_col - 1)) ] && sep='\n'
179 fmt="$fmt%-${tmp}s$sep"
182 # nothing in the table to print
183 [ -z "$fmt" ] && return
185 for y in `seq 0 $num_row`; do
187 for x in `seq 0 $num_col`; do
189 # skip this element if the column is empty
190 tmp="${name}_${x}_longest"
191 [ ${!tmp:-0} -eq 0 ] && continue
193 # fill this cell with the value or '' for printf
194 tmp="${name}_${y}_${x}"
195 row="$row'${!tmp:-""}' "
197 eval printf "'$fmt'" $row
201 ######################################################################
204 echo sgp_dd using dio=1 and thr=
207 # it could be making sure that the block dev
208 # isn't in use by something else
212 if ! which sgp_dd; then
213 echo "can't find sgp_dd binary"
219 # it could be making sure that the block dev
220 # isn't in use by something else
229 local bdev=${blocks[$i]};
232 [wo]) ifof="if=/dev/zero of=$bdev" ;;
233 r) ifof="if=$bdev of=/dev/null" ;;
234 *) die "asked to do io with $wor?"
236 echo sgp_dd $ifof bs=$iosize"k" count=$(($io_len / $iosize)) time=1 \
242 awk '($(NF) == "MB/sec") {print $(NF-1)}' < $output
257 ######################################################################
259 ext2_iozone_banner() {
260 echo "iozone -I on a clean ext2 fs"
262 ext2_iozone_config() {
265 ext2_iozone_prepare() {
267 local bdev=${blocks[$index]}
268 local mntpnt=$tmpdir/mount_$index
270 if ! which iozone; then
271 echo "iozone binary not found in PATH"
274 if ! iozone -i 0 -w -+o -s 1k -r 1k -f /dev/null > /dev/null; then
275 echo "iozone doesn't support -+o"
278 if ! which mke2fs; then
279 echo "mke2fs binary not found in PATH"
283 if ! mkdir -p $mntpnt ; then
284 echo "$mntpnt isn't a directory?"
287 echo making ext2 filesystem on $bdev
288 if ! mke2fs -b 4096 $bdev; then
293 if ! mount -t ext2 $bdev $mntpnt; then
294 echo "couldn't mount $bdev on $mntpnt"
298 cleanup_mounts[$index]="$mntpnt"
301 ext2_iozone_setup() {
304 local f="$tmpdir/mount_$id/iozone"
309 *) die "asked to do io with $wor?"
312 ext2_iozone_start() {
318 local f="$tmpdir/mount_$id/iozone"
321 [wo]) args="-i 0 -w" ;;
323 *) die "asked to do io with $wor?"
326 echo iozone "$args -r ${iosize}k -s $(($io_len / $threads))k \
327 -t $threads -+o -x -I -f $f"
329 ext2_iozone_result() {
336 [wo]) string="writers"
342 *) die "asked to do io with $wor?"
345 do_bc_scale 1 `awk '($1 == "Parent" && $'$field' == "'$string'") \
346 {print $'$(($field + 2))'}' $output` / 1024
348 ext2_iozone_cleanup() {
349 # the final read w/o -w removed the file
352 ext2_iozone_finish() {
354 local mntpnt=$tmpdir/mount_$index
357 unset cleanup_mounts[$index]
359 ext2_iozone_teardown() {
363 ######################################################################
364 # the lctl test_brw via the echo_client on top of the filter
366 # the echo_client setup is nutty enough to warrant its own clenaup
369 declare -a running_names
370 declare -a running_oids
372 cleanup_echo_filter() {
375 for i in `seq 0 $last_block`; do
376 [ -z "${running_oids[$i]}" ] && continue
377 lctl --device "\$"echo_$i destroy ${running_oids[$i]} \
382 for n in ${running_names[*]}; do
383 # I can't believe leading whitespace matters here.
393 for m in $running_module; do
398 [ ! -z "$running_config" ] && lconf --cleanup $running_config
402 echo_filter_banner() {
403 echo "test_brw on the echo_client on the filter"
405 echo_filter_config() {
407 local bdev=${blocks[$index]}
408 local config="$tmpdir/config.xml"
411 echo "lmc binary not found in PATH"
414 if ! which lconf; then
415 echo "lconf binary not found in PATH"
418 if ! which lctl; then
419 echo "lctl binary not found in PATH"
423 if [ $index = 0 ]; then
424 if ! lmc -m $config --add net \
425 --node localhost --nid localhost --nettype tcp; then
426 echo "error adding localhost net node"
431 if ! lmc -m $config --add ost --ost ost_$index --node localhost \
432 --fstype ext3 --dev $bdev --journal_size 400; then
433 echo "error adding $bdev to config with lmc"
437 # it would be nice to be able to ask lmc to setup an echo client
438 # to the filter here. --add echo_client assumes osc
440 echo_filter_prepare() {
442 local bdev=${blocks[$index]}
443 local config="$tmpdir/config.xml"
444 local name="echo_$index"
445 local uuid="echo_$index_uuid"
447 if [ $index = 0 ]; then
448 if ! lconf --reformat $config; then
449 echo "error setting up with lconf"
452 running_config="$config"
454 echo 0 > /proc/sys/portals/debug
455 echo 0 > /proc/sys/portals/subsystem_debug
457 if ! grep -q '^obdecho\>' /proc/modules; then
459 if ! modprobe obdecho; then
460 if [ ! -z "$echo_module" ]; then
461 if ! insmod $echo_module; then
462 echo "err: insmod $echo_module"
468 echo "err: modprobe $obdecho"
474 running_module=`basename $m | cut -d'.' -f 1`
480 attach echo_client $name $uuid
485 echo "error setting up echo_client $name against ost_$index"
488 running_names[$index]=$name
490 echo_filter_setup() {
494 local name="echo_$id"
500 *) die "asked to do io with $wor?"
503 running_threads=$threads
504 oid=`lctl --device "\$"$name create $threads | \
505 awk '/ #1 is object id/ { print $6 }'`
506 # XXX need to deal with errors
507 running_oids[$id]=$oid
509 echo_filter_start() {
516 local name="echo_$id"
517 local len_pages=$(($io_len / $(($page_size / 1024)) / $threads ))
518 local size_pages=$(($iosize / $(($page_size / 1024)) ))
523 *) die "asked to do io with $wor?"
526 echo lctl --threads $threads v "\$"$name \
527 test_brw 1 $rw v $len_pages t${running_oids[$id]} p$size_pages
529 echo_filter_result() {
534 for mbs in `awk '($8=="MB/s):"){print substr($7,2)}' < $output`; do
535 total=$(do_bc $total + $mbs)
537 do_bc_scale 2 $total / 1
539 echo_filter_cleanup() {
543 local name="echo_$id"
548 *) die "asked to do io with $wor?"
551 lctl --device "\$"$name destroy ${running_oids[$id]} $threads
552 unset running_oids[$id]
554 echo_filter_finish() {
556 # leave real work for _teardown
558 echo_filter_teardown() {
562 ######################################################################
563 # the iteration that drives the tests
572 local vmstat_log="$tmpdir/vmstat.log"
573 local opref="$test-$threads-$iosize-$wor"
575 # sigh. but this makes it easier to dump into the tables
583 for i in `seq 0 $last_block`; do
584 ${test}_setup $i $wor $threads
587 echo $test with $threads threads
589 $oprofile opcontrol --start
591 # start up vmstat and record its pid
592 nice -19 vmstat 1 > $vmstat_log 2>&1 &
593 [ $? = 0 ] || die "vmstat failed"
595 pid_now_running $vmstat_pid
597 # start up each block device's iostat
598 for i in `seq 0 $last_block`; do
599 nice -19 iostat -x ${blocks[$i]} 1 | awk \
600 '($1 == "'${blocks[$i]}'"){print $0; fflush()}' \
601 > $tmpdir/iostat.$i &
607 $oprofile opcontrol --reset
610 # start all the tests. each returns a pid to wait on
612 for i in `seq 0 $last_block`; do
613 local cmd=`${test}_start $threads $iosize $wor $i`
614 echo "$cmd" >> $tmpdir/commands
615 $cmd > $tmpdir/$i 2>&1 &
621 echo -n waiting on pids $pids:
628 # stop vmstat and all the iostats
630 pid_has_stopped $vmstat_pid
631 for i in `seq 0 $last_block`; do
632 local pid=${iostat_pids[$i]}
633 [ -z "$pid" ] && continue
636 unset iostat_pids[$i]
640 $readprofile | sort -rn > $tmpdir/readprofile
642 $oprofile opcontrol --shutdown
643 $oprofile opreport > $tmpdir/oprofile
644 echo >> $tmpdir/oprofile
645 $oprofile opreport -c -l | head -20 >> $tmpdir/oprofile
647 save_output $tmpdir/oprofile $opref.oprofile
648 save_output $tmpdir/readprofile $opref.readprofile
650 # collect the results of vmstat and iostat
651 cpu=$(mean_stddev $(awk \
652 '(NR > 3 && NF == 16 && $16 != "id" ) \
653 {print 100 - $16}' < $vmstat_log) )
654 save_output $vmstat_log $opref.vmstat
656 for i in `seq 0 $last_block`; do
657 read_req_s[$i]=$(mean_stddev $(awk \
658 '(NR > 1) {print $4}' < $tmpdir/iostat.$i) )
659 write_req_s[$i]=$(mean_stddev $(awk \
660 '(NR > 1) {print $5}' < $tmpdir/iostat.$i) )
661 sects_req[$i]=$(mean_stddev $(awk \
662 '(NR > 1) {print $10}' < $tmpdir/iostat.$i) )
663 queued_reqs[$i]=$(mean_stddev $(awk \
664 '(NR > 1) {print $11}' < $tmpdir/iostat.$i) )
665 service_ms[$i]=$(mean_stddev $(awk \
666 '(NR > 1) {print $13}' < $tmpdir/iostat.$i) )
668 save_output $tmpdir/iostat.$i $opref.iostat.$i
671 # record each index's test results and sum them
673 for i in `seq 0 $last_block`; do
674 local t=`${test}_result $tmpdir/$i $wor`
675 save_output $tmpdir/$i $opref.$i
676 echo test returned "$t"
678 # some tests return mean:stddev per device, filter out stddev
679 thru=$(do_bc $thru + $(echo $t | sed -e 's/:.*$//g'))
682 for i in `seq 0 $last_block`; do
683 ${test}_cleanup $i $wor $threads
686 # tabulate the results
687 echo $test did $thru mb/s with $cpu
688 table_set $test $my_x $cur_y `do_bc_scale 2 $thru / 1`
689 table_set $test $(($my_x + 1)) $cur_y $cpu
691 for i in `seq 0 $last_block`; do
692 cur_y=$(($cur_y + 1))
693 table_set $test $(($my_x)) $cur_y ${mb_s[$i]}
694 table_set $test $(($my_x + 1)) $cur_y ${read_req_s[$i]}
695 table_set $test $(($my_x + 2)) $cur_y ${write_req_s[$i]}
696 table_set $test $(($my_x + 3)) $cur_y ${sects_req[$i]}
697 table_set $test $(($my_x + 4)) $cur_y ${queued_reqs[$i]}
698 table_set $test $(($my_x + 5)) $cur_y ${service_ms[$i]}
701 cur_y=$(($cur_y + 1))
706 local thr=$min_threads
711 for i in `seq 0 $last_block`; do
712 if ! ${test}_config $i; then
713 echo "couldn't config $test for bdev ${blocks[$i]}"
714 echo "skipping $test for all block devices"
721 for i in `seq 0 $last_block`; do
722 # don't prepare if _config already failed
723 [ ! -z "$cleanup" ] && break
724 if ! ${test}_prepare $i; then
725 echo "couldn't prepare $test for bdev ${blocks[$i]}"
726 echo "skipping $test for all block devices"
733 while [ -z "$cleanup" -a $thr -lt $(($max_threads + 1)) ]; do
734 for iosize in 128 512; do
735 table_set $test 0 $cur_y $thr
736 table_set $test 1 $cur_y $iosize
739 table_set $test 2 $cur_y $wor
740 test_one $test 3 $thr $iosize $wor
746 [ -z "$cleanup" ] && cleanup=$last_block
748 if [ "$cleanup" != -1 ]; then
749 for i in `seq $cleanup 0`; do
759 while getopts ":d:b:l:t:T:r:e:" opt; do
761 e) echo_module=$OPTARG ;;
763 d) output_dir=$OPTARG ;;
765 r) run_tests=$OPTARG ;;
766 t) min_threads=$OPTARG ;;
767 T) max_threads=$OPTARG ;;
772 page_size=`getconf PAGE_SIZE` || die '"getconf PAGE_SIZE" failed'
774 [ ! -z "$echo_module" -a ! -f "$echo_module" ] && \
775 die "obdecho module $echo_module is not a file"
777 if [ -z "$io_len" ]; then
778 io_len=`awk '($1 == "MemTotal:"){print $2}' < /proc/meminfo`
779 [ -z "$io_len" ] && die "couldn't determine the amount of memory"
782 if [ ! -z "$output_dir" ]; then
783 if [ ! -e "$output_dir" ]; then
784 mkdir -p "$output_dir" || die "error creating $output_dir"
786 [ ! -d "$output_dir" ] && die "$output_dir isn't a directory"
789 block=`echo $block | sed -e 's/,/ /g'`
790 [ -z "$block" ] && usage "need block devices"
792 run_tests=`echo $run_tests | sed -e 's/,/ /g'`
793 [ -z "$run_tests" ] && usage "need to specify tests to run with -r"
794 for t in $run_tests; do
795 if ! echo $possible_tests | grep -q $t ; then
796 die "$t isn't one of the possible tests: $possible_tests"
800 if which opcontrol; then
801 echo generating oprofile results
804 echo not using oprofile
808 if which readprofile; then
809 map="/boot/System.map-`uname -r`"
810 if [ -f /proc/profile -a -f "$map" ]; then
811 echo generating profiles with 'readprofile'
812 readprofile="readprofile -m $map"
815 if [ -z "$readprofile" ]; then
816 echo not using readprofile
820 [ $min_threads -gt $max_threads ] && \
821 die "min threads $min_threads must be <= min_threads $min_threads"
824 [ ! -e $b ] && die "block device file $b doesn't exist"
825 [ ! -b $b ] && die "$b isn't a block device"
826 dd if=$b of=/dev/null bs=8192 count=1 || \
827 die "couldn't read 8k from $b, is it alive?"
828 [ ! -b $b ] && die "$b isn't a block device"
829 last_block=$(($last_block + 1))
830 blocks[$last_block]=$b
833 tmpdir=`mktemp -d /tmp/.surveyXXXXXX` || die "couldn't create tmp dir"
835 echo each test will operate on $io_len"k"
839 for t in $run_tests; do
846 table_set $t 3 1 "MB"
847 table_set $t 4 1 "rR"
848 table_set $t 5 1 "wR"
849 table_set $t 6 1 "SR"
851 table_set $t 8 1 "ms"
854 if ! test_iterator $t; then
857 test_results="$test_results $t"
860 save_output $tmpdir/commands commands
862 [ ! -z "$test_results" ] && (
864 echo "T = number of concurrent threads per device"
865 echo "L = base io operation length, in KB"
866 echo "m = IO method: read, write, or over-write"
867 echo "A = aggregate throughput from all devices"
868 echo "C = percentage CPU used, both user and system"
869 echo "MB/s = per-device throughput"
870 echo "rR = read requests issued to the device per second"
871 echo "wR = write requests issued to the device per second"
872 echo "SR = sectors per request; sectors tend to be 512 bytes"
873 echo "Q = the average number of requests queued on the device"
874 echo "ms = the average ms taken by the device to service a req"
876 echo "foo:bar represents a mean of foo with a stddev of bar"
879 for t in $test_results; do