lustre/tests/filter_survey.sh

   1 #!/bin/sh
   2 #
   3 # this runs prep/commit against filters and generates a
   4 # table of their write timings.  it needs the names
   5 # of the filters it will run against and needs the
   6 # obdecho module loaded.  it spews a lot of junk
   7 # as it goes, only the last bit is really interesting;
   8 # tee it to a log file.
   9 #
  10 # ex: FILTER_NAMES="ost1 ost2" sh ./filter_survey.sh
  11 #
  12 SRCDIR="`dirname $0`/"
  13 export PATH=$SRCDIR/../utils:/sbin:/usr/sbin::$PATH
  14
  15 tmp_dir=""
  16 echo_base="f_s_$$"
  17 echo_objs=""
  18
  19 die() {
  20         echo $* 1>&2
  21         exit 1
  22 }
  23
  24 cleanup() {
  25         [ ! -z "$tmp_dir" ] && [ -d $tmp_dir ] && rm -rf $tmp_dir
  26         [ -z "$echo_objs" ] && exit 0
  27         for obj in $echo_objs; do
  28                 echo cleaning up $obj
  29 # I can't believe leading whitespace matters here.
  30 lctl << EOF
  31 device $obj
  32 cleanup
  33 detach
  34 quit
  35 EOF
  36         done
  37 }
  38 trap cleanup EXIT
  39
  40 not_a_filter() {
  41         lctl device_list | awk "/obdfilter $1/ {exit 1}"
  42         return $?
  43 }
  44
  45 # holy crap are these confusing
  46 sep1="||||"
  47 sep2="||"
  48 sep3=""
  49 sep4="||||"
  50
  51 #
  52 # build up echo_clients attached to the given filters and record
  53 # their names and obj numbers for later use and teardown
  54 #
  55 last_filter="-1"
  56 [ -z "$FILTER_NAMES" ] && die "please specify filter names to run against"
  57 for fn in $FILTER_NAMES; do
  58         if not_a_filter $fn; then
  59                 die "'$fn' isn't the name of an obdfilter device"
  60         fi
  61         en="${echo_base}_$fn"
  62 lctl << EOF
  63         newdev
  64         attach echo_client $en ${en}_uuid
  65         setup $fn
  66         probe
  67         quit
  68 EOF
  69         [ $? -eq 0 ] || die "error setting up echo_client (is obdecho loaded?)"
  70
  71         obj=`lctl device_list | awk "/echo_client $en/ {print "'$1}'`
  72         [ -z "$obj" ] && die "couldn't find my echo_client's object number"
  73         echo setup echo_client name $en as object $obj
  74         echo_objs="$echo_objs $obj"
  75
  76         last_filter=$(($last_filter + 1))
  77         echo_names[$last_filter]=$en
  78
  79         # build up the seperators we'll use in generating the wiki
  80         sep1="$sep1||||"
  81         sep2="$sep2||"
  82         sep3="$sep3||"
  83         sep4="$sep4||"
  84 done
  85
  86
  87 doit() {
  88         $*
  89 }
  90 nop() {
  91         local nothing;
  92 }
  93 if which opcontrol; then
  94         echo generating oprofile results
  95         oprofile=doit
  96 else
  97         echo not using oprofile
  98         oprofile=nop
  99 fi
 100
 101 tmp_dir=`mktemp -d /tmp/echo_client_survey_XXXXXX` || die "mktemp failed"
 102
 103 TOT_PAGES=${TOT_PAGES:-524288}
 104
 105 throughput() {
 106         local threads="$1"
 107         local pages="$2"
 108         local time="$3"
 109         local tp=`echo 'scale=2; '$threads' * '$pages' * 4096 / ('$time' * 1024 * 1024)' | bc`
 110         echo $tp
 111 }
 112
 113 wait_for_idle_io() {
 114         echo "waiting idle io via vmstat"
 115         vmstat 1 | awk '
 116         ($10 == 0 && $11 == 0) {
 117                 idle++;
 118                 if (idle == 3) {
 119                         print "idle for 3 seconds, must be done"
 120                         exit
 121                 }
 122         }
 123         (NR == 13) {
 124                 "deletion took longer than 10s, bailing";
 125                 exit
 126         } '
 127 }
 128
 129 #
 130 # sorry for the wild indenting.  get a wide terminal, its 2003.
 131 #
 132 num_rows="0"
 133 num_summary_rows="0"
 134 for order_threads in `seq 0 3`; do
 135         nthreads=$(echo "2^$order_threads" | bc)
 136
 137         for stride in 16 64 128; do
 138                 span="<|$(($nthreads +1))>"
 139                 row="||$span $nthreads||$span $stride||"
 140                 sum_row="||$nthreads||$stride||"
 141
 142                 for t in `seq 1 $nthreads`; do
 143                         thread_row[$t]="||"
 144                 done
 145
 146                 for obj_per_thread in y n; do
 147                         if [ $obj_per_thread == "y" ]; then
 148                                 offset_prefix=""
 149                                 objid_prefix="t";
 150                         else
 151                                 offset_prefix="t"
 152                                 objid_prefix="";
 153                         fi
 154
 155                         # create the objects that this write/rewrite run
 156                         # will be using
 157                         for i in `seq 0 $last_filter`; do
 158                                 oid=`lctl --device "\$"${echo_names[$i]} create $nthreads | \
 159                                         awk '/1 is object id/ { print $6 }'`
 160                                 [ -z "$oid" ] && die "error creating object"
 161                                 oids[$i]=$oid
 162                         done
 163
 164                         # iterate through write and rewrite
 165                         for a in 1 2; do
 166                                 total_maxtime="0.0"
 167                                 pids=""
 168
 169                                 $oprofile opcontrol --start
 170
 171                                 echo 'nice -19 vmstat 5' > $tmp_dir/vmstat-log
 172                                 nice -19 vmstat 5 >> $tmp_dir/vmstat-log &
 173                                 vmstat_pid="$!"
 174
 175                                 $oprofile opcontrol --reset
 176
 177                                 # start a test_brw thread in the background
 178                                 # for each given filter
 179                                 for i in `seq 0 $last_filter`; do
 180                                         lctl --threads $nthreads v "\$"${echo_names[$i]} \
 181                                                 test_brw ${offset_prefix}1 w v \
 182                                                 $TOT_PAGES ${objid_prefix}${oids[$i]} p$stride | \
 183                                                         tee $tmp_dir/$i &
 184                                         pids="$pids $!"
 185                                 done
 186                                 echo ------ waiting for $nthreads obj per thread $obj_per_thread rw: $a ----
 187                                 for p in $pids; do
 188                                         wait $p
 189                                 done
 190                                 $oprofile opcontrol --shutdown
 191                                 echo ------ finished $nthreads obj per thread $obj_per_thread rw: $a ----
 192                                 kill $vmstat_pid
 193                                 cat $tmp_dir/vmstat-log
 194                                 rm $tmp_dir/vmstat-log
 195
 196                                 $oprofile opreport
 197                                 $oprofile opreport -c -l
 198
 199                                 for t in `seq 1 $nthreads`; do
 200                                         thread_row[$t]="${thread_row[$t]} ||"
 201                                 done
 202                                 row_tmp=""
 203                                 for i in `seq 0 $last_filter`; do
 204                                         maxtime="0.0"
 205                                         for t in `seq 1 $nthreads`; do
 206                                                 f="$tmp_dir/$i"
 207                                                 MS=`grep "test_brw-$t" $f | \
 208                                                    awk '($8=="MB/s):"){print $6, substr($7,2);}'`
 209                                                 thread_row[$t]="${thread_row[$t]}$MS||"
 210                                                 time=`echo $MS | cut -d s -f 1`
 211                                                 if [ `echo "$time > $maxtime" | bc` -eq "1" ]; then
 212                                                         maxtime=$time;
 213                                                 fi
 214                                         done
 215                                         tp=`throughput $nthreads $TOT_PAGES $maxtime`
 216                                         row_tmp="${row_tmp}<#ffffe0>$tp $maxtime||"
 217                                         sum_row="${sum_row}$tp||"
 218
 219                                         if [ `echo "$maxtime > $total_maxtime" | bc` -eq "1" ]; then
 220                                                 total_maxtime=$maxtime;
 221                                         fi
 222                                 done
 223                                 tp=`throughput $(($nthreads * $(($last_filter +1)))) $TOT_PAGES $total_maxtime`
 224                                 row="${row}<#ffffe0>${tp} $total_maxtime||${row_tmp}"
 225                         done
 226
 227                         # destroy the objects from this run and wait for
 228                         # their destruction to complete
 229                         for i in `seq 0 $last_filter`; do
 230                                 lctl --device "\$"${echo_names[$i]} destroy ${oids[$i]} $nthreads
 231                         done
 232                         wait_for_idle_io
 233                 done
 234
 235                 num_rows=$(($num_rows + 1))
 236                 rows[$num_rows]="$row"
 237
 238                 num_summary_rows=$(($num_summary_rows + 1))
 239                 summary[$num_summary_rows]="$sum_row"
 240
 241                 for t in `seq 1 $nthreads`; do
 242                         num_rows=$(($num_rows + 1))
 243                         rows[$num_rows]="${thread_row[$t]}"
 244                 done
 245         done
 246 done
 247
 248 echo done.
 249
 250 bg='<rowbgcolor="#eoeoff"'
 251 echo "||$bg|2>threads writing $TOT_PAGES pages||<|2>pages per prep/commit${sep1}oid per thread${sep1}shared oid||"
 252 echo "$sep2$bg>write${sep2}re-write${sep2}write${sep2}re-write||"
 253 for r in `seq 1 $num_rows`; do
 254         echo ${rows[$r]}
 255 done
 256
 257 echo summary table
 258
 259 echo "||$bg|2>threads||<|2>pages ${sep4}oid/thread${sep4}shared oid||"
 260 echo "$sep3$bg>write${sep3}re-write${sep3}write${sep3}re-write||"
 261 for r in `seq 1 $num_summary_rows`; do
 262         echo ${summary[$r]}
 263 done