lustre/tests/filter_survey.sh

   1 #!/bin/sh
   2 #
   3 # this runs prep/commit against filters and generates a
   4 # table of their write timings.  it needs the names
   5 # of the filters it will run against and needs the
   6 # obdecho module loaded.  it spews a lot of junk
   7 # as it goes, only the last bit is really interesting;
   8 # tee it to a log file.
   9 #
  10 # ex: FILTER_NAMES="ost1 ost2" sh ./filter_survey.sh
  11 #
  12 SRCDIR="`dirname $0`/"
  13 export PATH=$SRCDIR/../utils:/sbin:/usr/sbin::$PATH
  14
  15 tmp_dir=""
  16 echo_base="f_s_$$"
  17 last_filter="-1"
  18
  19 die() {
  20         echo $* 1>&2
  21         exit 1
  22 }
  23
  24 cleanup() {
  25         [ ! -z "$tmp_dir" ] && [ -d $tmp_dir ] && rm -rf $tmp_dir
  26         [ "$last_filter" = "-1" ] && exit 0
  27         for i in `seq 0 $last_filter`; do
  28                 local name="${echo_names[$i]}"
  29                 echo cleaning up $name
  30 # I can't believe leading whitespace matters here.
  31 lctl << EOF
  32 cfg_device $name
  33 cleanup
  34 detach
  35 quit
  36 EOF
  37         done
  38 }
  39 trap cleanup EXIT
  40
  41 not_a_filter() {
  42         lctl device_list | awk "/obdfilter $1/ {exit 1}"
  43         return $?
  44 }
  45
  46 # holy crap are these confusing
  47 sep1="||||"
  48 sep2="||"
  49 sep3=""
  50 sep4="||||"
  51
  52 #
  53 # build up echo_clients attached to the given filters and record
  54 # their names and obj numbers for later use and teardown
  55 #
  56 [ -z "$FILTER_NAMES" ] && die "please specify filter names to run against"
  57 for fn in $FILTER_NAMES; do
  58         if not_a_filter $fn; then
  59                 die "'$fn' isn't the name of an obdfilter device"
  60         fi
  61         en="${echo_base}_$fn"
  62 lctl << EOF
  63         newdev
  64         attach echo_client $en ${en}_uuid
  65         setup $fn
  66         quit
  67 EOF
  68         [ $? -eq 0 ] || die "error setting up echo_client (is obdecho loaded?)"
  69
  70         obj=`lctl device_list | awk "/echo_client $en/ {print "'$1}'`
  71         [ -z "$obj" ] && die "couldn't find my echo_client's object number"
  72         echo setup echo_client name $en as object $obj
  73         echo_objs="$echo_objs $obj"
  74
  75         last_filter=$(($last_filter + 1))
  76         echo_names[$last_filter]=$en
  77
  78         # build up the seperators we'll use in generating the wiki
  79         sep1="$sep1||||"
  80         sep2="$sep2||"
  81         sep3="$sep3||"
  82         sep4="$sep4||"
  83 done
  84
  85
  86 doit() {
  87         $*
  88 }
  89 nop() {
  90         local nothing;
  91 }
  92 if which opcontrol; then
  93         echo generating oprofile results
  94         oprofile=doit
  95 else
  96         echo not using oprofile
  97         oprofile=nop
  98 fi
  99
 100 tmp_dir=`mktemp -d /tmp/echo_client_survey_XXXXXX` || die "mktemp failed"
 101
 102 TOT_PAGES=${TOT_PAGES:-524288}
 103
 104 throughput() {
 105         local threads="$1"
 106         local pages="$2"
 107         local time="$3"
 108         local tp=`echo 'scale=2; '$threads' * '$pages' * 4096 / ('$time' * 1024 * 1024)' | bc`
 109         echo $tp
 110 }
 111
 112 wait_for_idle_io() {
 113         echo "waiting idle io via vmstat"
 114         vmstat 1 | awk '
 115         ($10 == 0 && $11 == 0) {
 116                 idle++;
 117                 if (idle == 3) {
 118                         print "idle for 3 seconds, must be done"
 119                         exit
 120                 }
 121         }
 122         (NR == 13) {
 123                 "deletion took longer than 10s, bailing";
 124                 exit
 125         } '
 126 }
 127
 128 #
 129 # sorry for the wild indenting.  get a wide terminal, its 2003.
 130 #
 131 num_rows="0"
 132 num_summary_rows="0"
 133 for order_threads in `seq 0 3`; do
 134         nthreads=$(echo "2^$order_threads" | bc)
 135
 136         for stride in 16 64 128; do
 137                 span="<|$(($nthreads +1))>"
 138                 row="||$span $nthreads||$span $stride||"
 139                 sum_row="||$nthreads||$stride||"
 140
 141                 for t in `seq 1 $nthreads`; do
 142                         thread_row[$t]="||"
 143                 done
 144
 145                 for obj_per_thread in y n; do
 146                         if [ $obj_per_thread == "y" ]; then
 147                                 offset_prefix=""
 148                                 objid_prefix="t";
 149                         else
 150                                 offset_prefix="t"
 151                                 objid_prefix="";
 152                         fi
 153
 154                         # create the objects that this write/rewrite run
 155                         # will be using
 156                         for i in `seq 0 $last_filter`; do
 157                                 oid=`lctl --device "\$"${echo_names[$i]} create $nthreads | \
 158                                         awk '/1 is object id/ { print $6 }'`
 159                                 [ -z "$oid" ] && die "error creating object"
 160                                 oids[$i]=$oid
 161                         done
 162
 163                         # iterate through write and rewrite
 164                         for a in 1 2; do
 165                                 total_maxtime="0.0"
 166                                 pids=""
 167
 168                                 $oprofile opcontrol --start
 169
 170                                 echo 'nice -19 vmstat 5' > $tmp_dir/vmstat-log
 171                                 nice -19 vmstat 5 >> $tmp_dir/vmstat-log &
 172                                 vmstat_pid="$!"
 173
 174                                 $oprofile opcontrol --reset
 175
 176                                 # start a test_brw thread in the background
 177                                 # for each given filter
 178                                 for i in `seq 0 $last_filter`; do
 179                                         lctl --threads $nthreads v "\$"${echo_names[$i]} \
 180                                                 test_brw ${offset_prefix}1 w v \
 181                                                 $TOT_PAGES ${objid_prefix}${oids[$i]} p$stride | \
 182                                                         tee $tmp_dir/$i &
 183                                         pids="$pids $!"
 184                                 done
 185                                 echo ------ waiting for $nthreads obj per thread $obj_per_thread rw: $a ----
 186                                 for p in $pids; do
 187                                         wait $p
 188                                 done
 189                                 $oprofile opcontrol --shutdown
 190                                 echo ------ finished $nthreads obj per thread $obj_per_thread rw: $a ----
 191                                 kill $vmstat_pid
 192                                 cat $tmp_dir/vmstat-log
 193                                 rm $tmp_dir/vmstat-log
 194
 195                                 $oprofile opreport
 196                                 $oprofile opreport -c -l
 197
 198                                 for t in `seq 1 $nthreads`; do
 199                                         thread_row[$t]="${thread_row[$t]} ||"
 200                                 done
 201                                 row_tmp=""
 202                                 for i in `seq 0 $last_filter`; do
 203                                         maxtime="0.0"
 204                                         for t in `seq 1 $nthreads`; do
 205                                                 f="$tmp_dir/$i"
 206                                                 MS=`grep "test_brw-$t" $f | \
 207                                                    awk '($8=="MB/s):"){print $6, substr($7,2);}'`
 208                                                 thread_row[$t]="${thread_row[$t]}$MS||"
 209                                                 time=`echo $MS | cut -d s -f 1`
 210                                                 if [ `echo "$time > $maxtime" | bc` -eq "1" ]; then
 211                                                         maxtime=$time;
 212                                                 fi
 213                                         done
 214                                         tp=`throughput $nthreads $TOT_PAGES $maxtime`
 215                                         row_tmp="${row_tmp}<#ffffe0>$tp $maxtime||"
 216                                         sum_row="${sum_row}$tp||"
 217
 218                                         if [ `echo "$maxtime > $total_maxtime" | bc` -eq "1" ]; then
 219                                                 total_maxtime=$maxtime;
 220                                         fi
 221                                 done
 222                                 tp=`throughput $(($nthreads * $(($last_filter +1)))) $TOT_PAGES $total_maxtime`
 223                                 row="${row}<#ffffe0>${tp} $total_maxtime||${row_tmp}"
 224                         done
 225
 226                         # destroy the objects from this run and wait for
 227                         # their destruction to complete
 228                         for i in `seq 0 $last_filter`; do
 229                                 lctl --device "\$"${echo_names[$i]} destroy ${oids[$i]} $nthreads
 230                         done
 231                         wait_for_idle_io
 232                 done
 233
 234                 num_rows=$(($num_rows + 1))
 235                 rows[$num_rows]="$row"
 236
 237                 num_summary_rows=$(($num_summary_rows + 1))
 238                 summary[$num_summary_rows]="$sum_row"
 239
 240                 for t in `seq 1 $nthreads`; do
 241                         num_rows=$(($num_rows + 1))
 242                         rows[$num_rows]="${thread_row[$t]}"
 243                 done
 244         done
 245 done
 246
 247 echo done.
 248
 249 bg='<rowbgcolor="#eoeoff"'
 250 echo "||$bg|2>threads writing $TOT_PAGES pages||<|2>pages per prep/commit${sep1}oid per thread${sep1}shared oid||"
 251 echo "$sep2$bg>write${sep2}re-write${sep2}write${sep2}re-write||"
 252 for r in `seq 1 $num_rows`; do
 253         echo ${rows[$r]}
 254 done
 255
 256 echo summary table
 257
 258 echo "||$bg|2>threads||<|2>pages ${sep4}oid/thread${sep4}shared oid||"
 259 echo "$sep3$bg>write${sep3}re-write${sep3}write${sep3}re-write||"
 260 for r in `seq 1 $num_summary_rows`; do
 261         echo ${summary[$r]}
 262 done