3 # for now all the units are in 'k', but we could introduce some helpers
4 # would be nice to run tests in the background and trap signals and kill
7 # make sure devices aren't in use before going to town
8 # really use threads with iozone
9 # look into what sgp_dd is really doing, update arguments
10 # rename config/prepare/setup/cleanup/finish/teardown
11 # do something with sf and fpp iterating
12 # discard first vmstat line
15 # a temp dir that is setup and torn down for each script run
17 # so we can kill background processes as the test cleans up
18 declare -a cleanup_pids
19 # to unmount mounts in our tmpdir before removing it
20 declare -a cleanup_mounts
21 # global for completing the table. XXX this is a wart that could go
24 # defaults for some options:
27 possible_tests="sgp_dd ext2_iozone echo_filter"
28 run_tests="$possible_tests"
30 # optional output directory
39 [ -e $path ] || continue;
40 [ -f $path ] || die "needed to remove non-file $path"
41 rm -f $path || die "couldn't remove $path"
45 [ ! -z "$output_dir" ] && mv -f $1 $output_dir/$2
48 for pid in ${cleanup_pids[*]}; do
52 for a in ${cleanup_mounts[*]}; do
55 [ ${#tmpdir} == 18 ] && [ -d $tmpdir ] && rm -rf $tmpdir
61 cleanup_pids[$pid]=$pid
65 unset cleanup_pids[$pid]
69 echo $* | sed -e 's/ /,/g'
72 echo "scale=2; $*" | bc
84 0) echo '??' ; return ;;
85 1) echo "$avg:0" ; return ;;
88 avg=`do_bc $avg / $num`
91 local dev=`do_bc \($p - $avg\) \^ 2`
92 tmp=`do_bc $tmp + $dev`
94 tmp=`do_bc sqrt \( $tmp / \($num - 1\) \)`
100 echo " -b <block device to profile>"
101 echo " -d <summary output directory>"
102 echo " -l <max io len>"
103 echo " -t <minimum number of threads per device>"
104 echo " -T <maximum number of threads per device>"
105 echo " -r <tests to run>"
109 # some cute code for handling tables whose columns fit
114 if [ $val -gt ${!target:-0} ]; then
119 local name="_table_$1"
125 eval ${name}_${row}_${col}="'$val'"
127 set_max ${name}_${col}_longest ${#val}
128 set_max ${name}_num_col $(($col + 1))
129 set_max ${name}_num_row $(($row + 1))
133 local name="_table_$1"
136 tmp="${name}_${row}_${col}"
141 local name="_table_$1"
148 tmp="${name}_num_col"
150 tmp="${name}_num_row"
153 # iterate through the columns to find the longest
156 for x in `seq 0 $num_col`; do
157 tmp="${name}_${x}_longest"
159 [ $tmp -eq 0 ] && continue
161 [ $x -eq $((num_col - 1)) ] && sep='\n'
163 fmt="$fmt%-${tmp}s$sep"
166 # nothing in the table to print
167 [ -z "$fmt" ] && return
169 for y in `seq 0 $num_row`; do
171 for x in `seq 0 $num_col`; do
173 # skip this element if the column is empty
174 tmp="${name}_${x}_longest"
175 [ ${!tmp:-0} -eq 0 ] && continue
177 # fill this cell with the value or '' for printf
178 tmp="${name}_${y}_${x}"
179 row="$row'${!tmp:-""}' "
181 eval printf "'$fmt'" $row
185 ######################################################################
188 echo sgp_dd using dio=1 and thr=
191 # it could be making sure that the block dev
192 # isn't in use by something else
196 if ! which sgp_dd; then
197 echo "can't find sgp_dd binary"
203 # it could be making sure that the block dev
204 # isn't in use by something else
213 local bdev=${blocks[$i]};
216 w) ifof="if=/dev/zero of=$bdev" ;;
217 r) ifof="if=$bdev of=/dev/null" ;;
218 *) die "asked to do io with $wor?"
220 echo sgp_dd $ifof bs=$iosize"k" count=$(($io_len / $iosize)) time=1 \
226 awk '($(NF) == "MB/sec") {print $(NF-1)}' < $output
241 ######################################################################
243 ext2_iozone_banner() {
244 echo "iozone -I on a clean ext2 fs"
246 ext2_iozone_config() {
249 ext2_iozone_prepare() {
251 local bdev=${blocks[$index]}
252 local mntpnt=$tmpdir/mount_$index
254 if ! which iozone; then
255 echo "iozone binary not found in PATH"
258 if ! which mke2fs; then
259 echo "mke2fs binary not found in PATH"
263 if ! mkdir -p $mntpnt ; then
264 echo "$mntpnt isn't a directory?"
267 echo making ext2 filesystem on $bdev
268 if ! mke2fs -b 4096 $bdev; then
273 if ! mount -t ext2 $bdev $mntpnt; then
274 echo "couldn't mount $bdev on $mntpnt"
278 cleanup_mounts[$index]="$mntpnt"
281 ext2_iozone_setup() {
284 local f="$tmpdir/mount_$id/iozone"
289 *) die "asked to do io with $wor?"
292 ext2_iozone_start() {
298 local f="$tmpdir/mount_$id/iozone"
303 *) die "asked to do io with $wor?"
306 echo iozone "$args -r ${iosize}k -s ${io_len}k -I -f $f"
308 ext2_iozone_result() {
311 kps=`awk '($2 == "reclen"){results=NR+1}(results == NR){print $3}' \
315 ext2_iozone_cleanup() {
318 local f="$tmpdir/mount_$id/iozone"
323 *) die "asked to do io with $wor?"
326 ext2_iozone_finish() {
328 local mntpnt=$tmpdir/mount_$index
331 unset cleanup_mounts[$index]
333 ext2_iozone_teardown() {
337 ######################################################################
338 # the lctl test_brw via the echo_client on top of the filter
340 # the echo_client setup is nutty enough to warrant its own clenaup
343 declare -a running_names
345 cleanup_echo_filter() {
348 for i in `seq 0 $last_block`; do
349 [ -z "${running_oids[$i]}" ] && continue
350 lctl --device "\$"echo_$i destroy ${running_oids[$i]} \
355 for n in ${running_names[*]}; do
356 # I can't believe leading whitespace matters here.
366 for m in $running_modules; do
371 [ ! -z "$running_config" ] && lconf --cleanup $running_config
375 echo_filter_banner() {
376 echo "test_brw on the echo_client on the filter"
378 echo_filter_config() {
380 local bdev=${blocks[$index]}
381 local config="$tmpdir/config.xml"
384 echo "lmc binary not found in PATH"
387 if ! which lconf; then
388 echo "lconf binary not found in PATH"
391 if ! which lctl; then
392 echo "lctl binary not found in PATH"
396 if [ $index = 0 ]; then
397 if ! lmc -m $config --add net \
398 --node localhost --nid localhost --nettype tcp; then
399 echo "error adding localhost net node"
404 if ! lmc -m $config --add ost --ost ost_$index --node localhost \
405 --fstype ext3 --dev $bdev --journal_size 400; then
406 echo "error adding $bdev to config with lmc"
410 # it would be nice to be able to ask lmc to setup an echo client
411 # to the filter here. --add echo_client assumes osc
413 echo_filter_prepare() {
415 local bdev=${blocks[$index]}
416 local config="$tmpdir/config.xml"
417 local name="echo_$index"
418 local uuid="echo_$index_uuid"
420 if [ $index = 0 ]; then
421 if ! lconf --reformat $config; then
422 echo "error setting up with lconf"
425 running_config="$config"
426 if ! grep -q '^obdecho\>' /proc/modules; then
427 if ! modprobe obdecho; then
428 echo "error running modprobe obdecho"
431 running_modules="obdecho"
437 attach echo_client $name $uuid
442 echo "error setting up echo_client $name against ost_$index"
445 running_names[$index]=$name
447 echo_filter_setup() {
451 local name="echo_$id"
457 *) die "asked to do io with $wor?"
460 running_threads=$threads
461 oid=`lctl --device "\$"$name create $threads | \
462 awk '/1 is object id/ { print $6 }'`
463 # XXX need to deal with errors
464 running_oids[$id]=$oid
466 echo_filter_start() {
471 local name="echo_$id"
472 local pages=$(($io_len / 4))
477 *) die "asked to do io with $wor?"
480 echo lctl --threads $threads v "\$"$name \
481 test_brw 1 w v $pages ${running_oids[$i]} p$iosize
483 echo_filter_result() {
488 for mbs in `awk '($8=="MB/s):"){print substr($7,2)}' < $output`; do
489 total=$(do_bc $total + $mbs)
493 echo_filter_cleanup() {
497 local name="echo_$id"
502 *) die "asked to do io with $wor?"
505 lctl --device "\$"$name destroy ${running_oids[$i]} $threads
506 unset running_oids[$i]
508 echo_filter_finish() {
510 # leave real work for _teardown
512 echo_filter_teardown() {
516 ######################################################################
517 # the iteration that drives the tests
527 local vmstat_log="$tmpdir/vmstat.log"
528 local opref="$test-$threads-$iosize-$wor"
530 for i in `seq 0 $last_block`; do
531 ${test}_setup $i $wor $threads
534 echo $test with $threads threads
536 # start up vmstat and record its pid
538 nice -19 vmstat 1 > $vmstat_log 2>&1 &
539 [ $? = 0 ] || die "vmstat failed"
541 pid_now_running $vmstat_pid
543 # start all the tests. each returns a pid to wait on
545 for i in `seq 0 $last_block`; do
546 cmd=`${test}_start $threads $iosize $wor $i`
547 $cmd > $tmpdir/$i 2>&1 &
553 echo -n waiting on pids $pids:
561 # stop vmstat and get cpu use from it
564 pid_has_stopped $vmstat_pid
565 cpu=$(mean_stddev $(awk \
566 '(NR > 3 && NF == 16 && $16 != "id" ) \
567 {print 100 - $16}' < $vmstat_log) )
568 save_output $vmstat_log $opref.vmstat
570 # record each index's test results and sum them
573 for i in `seq 0 $last_block`; do
574 local t=`${test}_result $tmpdir/$i`
575 save_output $tmpdir/$i $opref.$i
576 echo test returned "$t"
578 # some tests return mean:stddev per thread, filter out stddev
579 thru=$(do_bc $thru + $(echo $t | sed -e 's/:.*$//g'))
581 line="("`commas $line`")"
583 for i in `seq 0 $last_block`; do
584 ${test}_cleanup $i $wor $threads
587 # tabulate the results
588 echo $test did $thru mb/s with $cpu
589 table_set $test $my_x $my_y $thru
590 table_set $test $(($my_x + 1)) $my_y $cpu
591 table_set $test $(($my_x + 2)) $my_y $line
596 local thr=$min_threads
601 for i in `seq 0 $last_block`; do
602 if ! ${test}_config $i; then
603 echo "couldn't config $test for bdev ${blocks[$i]}"
604 echo "skipping $test for all block devices"
611 for i in `seq 0 $last_block`; do
612 # don't prepare if _config already failed
613 [ ! -z "$cleanup" ] && break
614 if ! ${test}_prepare $i; then
615 echo "couldn't prepare $test for bdev ${blocks[$i]}"
616 echo "skipping $test for all block devices"
623 while [ -z "$cleanup" -a $thr -lt $(($max_threads + 1)) ]; do
624 for iosize in 64 128; do
625 table_set $test 0 $cur_y $thr
626 table_set $test 1 $cur_y $iosize
627 table_set $test 2 $cur_y "|"
630 table_set $test 3 $cur_y $wor
631 test_one $test 4 $cur_y $thr $iosize $wor
632 cur_y=$(($cur_y + 1))
638 [ -z "$cleanup" ] && cleanup=$last_block
640 if [ "$cleanup" != -1 ]; then
641 for i in `seq $cleanup 0`; do
651 while getopts ":d:b:l:t:T:r:" opt; do
654 d) output_dir=$OPTARG ;;
656 r) run_tests=$OPTARG ;;
657 t) min_threads=$OPTARG ;;
658 T) max_threads=$OPTARG ;;
663 if [ -z "$io_len" ]; then
664 io_len=`awk '($1 == "MemTotal:"){print $2}' < /proc/meminfo`
665 [ -z "$io_len" ] && die "couldn't determine the amount of memory"
668 if [ ! -z "$output_dir" ]; then
669 [ ! -e "$output_dir" ] && "output dir $output_dir doesn't exist"
670 [ ! -d "$output_dir" ] && "output dir $output_dir isn't a directory"
673 block=`echo $block | sed -e 's/,/ /g'`
674 [ -z "$block" ] && usage "need block devices"
676 run_tests=`echo $run_tests | sed -e 's/,/ /g'`
677 [ -z "$run_tests" ] && usage "need to specify tests to run with -r"
678 for t in $run_tests; do
679 if ! echo $possible_tests | grep -q $t ; then
680 die "$t isn't one of the possible tests: $possible_tests"
684 [ $min_threads -gt $max_threads ] && \
685 die "min threads $min_threads must be <= min_threads $min_threads"
689 [ ! -e $b ] && die "block device file $b doesn't exist"
690 [ ! -b $b ] && die "$b isn't a block device"
691 last_block=$(($last_block + 1))
692 blocks[$last_block]=$b
695 tmpdir=`mktemp -d /tmp/.surveyXXXXXX` || die "couldn't create tmp dir"
697 echo each test will operate on $io_len"k"
701 for t in $run_tests; do
707 table_set $t 5 0 "C:S"
711 if ! test_iterator $t; then
714 test_results="$test_results $t"
717 [ ! -z "$test_results" ] && (
719 echo "T = number of concurrent threads per device"
720 echo "L = base io operation length, in KB"
721 echo "W/O/R = write/overwrite/read throughput, in MB/s"
722 echo "C = percentage CPU used, both user and system"
723 echo "S = standard deviation in cpu use"
724 echo "B = per-block results: ("`echo ${blocks[*]} | sed -e 's/ /,/g'`")"
728 for t in $test_results; do