lustre-iokit/sgpdd-survey/sgpdd-survey

   1 #!/bin/bash
   2
   3 ######################################################################
   4 # customize per survey
   5
   6 # CHOOSE EITHER scsidevs or rawdevs
   7 # the SCSI devices to measure - WARNING: will be erased.
   8 # The raw devices to use
   9 # rawdevs=${rawdevs:-"/dev/raw/raw1"}
  10 # scsidevs=`ls /dev/sd[a-z] /dev/sd[a-z][a-z]` # all devices, if you use udev
  11
  12 # result file prefix.  date/time+hostname makes unique
  13 # NB ensure the path exists if it includes subdirs
  14 rslt_loc=${rslt_loc:-"/tmp"}
  15 rslt=${rslt:-"$rslt_loc/sgpdd_survey_`date +%F@%R`_`uname -n`"}
  16
  17 # what to do (read or write)
  18 actions=${actions:-"write read"}
  19
  20 # total size per device (MBytes)
  21 # NB bigger than device cache is good
  22 size=${size:-8192}
  23
  24 # record size (KBytes)
  25 rszlo=${rszlo:-1024}
  26 rszhi=${rszhi:-1024}
  27
  28 # Concurrent regions per device
  29 crglo=${crglo:-1}
  30 crghi=${crghi:-256}
  31
  32 # boundary blocks between concurrent regions per device
  33 boundary=${boundary:-1024}
  34
  35 # threads to share between concurrent regions per device
  36 # multiple threads per region simulates a deeper request queue
  37 # NB survey skips over #thr < #regions and #thr/#regions > SG_MAX_QUEUE
  38 thrlo=${thrlo:-1}
  39 thrhi=${thrhi:-4096}
  40
  41 #####################################################################
  42 # leave the rest of this alone unless you know what you're doing...
  43
  44 # and max # threads one instance will spawn
  45 SG_MAX_QUEUE=16
  46
  47 # is the sg module loaded?
  48 sg_is_loaded=$(grep -q "^sg " /proc/modules && echo true || echo false)
  49
  50 # did we load it?
  51 sg_was_loaded=false
  52
  53 # map given device names into SG device names
  54 i=0
  55 devs=()
  56 if [ "$scsidevs" ]; then
  57         # we will test for a LUN, the test for a partition
  58         # if the partition number is > 9 this will fail
  59
  60     # make sure sg kernel module is loaded
  61     if ! $sg_is_loaded; then
  62         echo "loading the sg kernel module"
  63         modprobe sg && sg_was_loaded=true
  64         sg_is_loaded=true
  65     fi
  66
  67     for d in $scsidevs; do
  68         if [[ -L "$d" ]]; then
  69             echo "Device $d specified by alias. Will 'readlink' for device name"
  70             d=$(readlink -f $d)
  71         fi
  72         devs[$i]=`sg_map | awk "{if (\\\$2 == \"$d\") print \\\$1}"`
  73         if [ -z "${devs[i]}" ]; then
  74             echo "Can't find SG device for $d, testing for partition"
  75             pt=`echo $d | sed 's/[0-9]*$//'`
  76             # Try again
  77             devs[$i]=`sg_map | awk "{if (\\\$2 == \"$pt\") print \\\$1}"`
  78             if [ -z "${devs[i]}" ]; then
  79                 echo -e "Can't find SG device $pt.\nDo you have the sg module configured for your kernel?"
  80                 exit 1
  81            fi
  82         fi
  83         i=$((i+1))
  84     done
  85 elif [ "$rawdevs" ]; then
  86     for r in $rawdevs; do
  87         RES=`raw -q $r`
  88         if [ $? -eq 0 ];then
  89             devs[$i]=$r
  90             i=$((i+1))
  91         else
  92             echo "Raw device $r not set up"
  93             exit 1
  94         fi
  95     done
  96 else
  97     echo "Must specify scsidevs or rawdevs"
  98     exit 1
  99 fi
 100
 101 ndevs=${#devs[@]}
 102
 103 # determine block size. This should also work for raw devices
 104 # If it fails, set to 512
 105 bs=$((`sg_readcap -lb ${devs[0]} | awk '{print $2}'`))
 106 if [ $bs == 0  ];then
 107         echo "sg_readcap failed, setting block size to 512"
 108         bs=512
 109 fi
 110 rsltf=${rslt}.summary
 111 workf=${rslt}.detail
 112 echo -n > $rsltf
 113 echo -n > $workf
 114
 115 print_summary () {
 116     if [ "$1" = "-n" ]; then
 117         minusn=$1; shift
 118     else
 119         minusn=""
 120     fi
 121     echo $minusn "$*" >> $rsltf
 122     echo $minusn "$*"
 123 }
 124
 125 print_summary "$(date) sgpdd-survey on $rawdevs$scsidevs from $(hostname)"
 126
 127 for ((rsz=$rszlo;rsz<=$rszhi;rsz*=2)); do
 128     for ((crg=$crglo;crg<=$crghi;crg*=2)); do
 129         for ((thr=$thrlo;thr<=$thrhi;thr*=2)); do
 130             if ((thr < crg || thr/crg > SG_MAX_QUEUE)); then
 131                 continue
 132             fi
 133             # compute parameters
 134             bpt=$((rsz*1024/bs))
 135             blocks=$((size*((1024*1024)/bs)/crg))
 136             count=$blocks
 137             # show computed parameters
 138             actual_rsz=$((bpt*bs/1024))
 139             actual_size=$((bs*count*crg/1024))
 140             str=`printf 'total_size %8dK rsz %4d crg %5d thr %5d ' \
 141                          $((actual_size*ndevs)) $actual_rsz $((crg*ndevs)) $((thr*ndevs))`
 142             echo "==============> $str" >> $workf
 143             print_summary -n "$str"
 144             freemem=`awk < /proc/meminfo '/^MemTotal:/ {printf "%d\n", $2}'`
 145             if (((actual_rsz*thr/crg + 64)*crg*ndevs > freemem)); then
 146                 print_summary "ENOMEM"
 147                 continue
 148             fi
 149             # run tests
 150             for action in $actions; do
 151                 print_summary -n "$action "
 152                 echo "=====> $action" >> $workf
 153                 tmpf=${workf}_tmp
 154                 # start test
 155                 t0=`date +%s.%N`
 156                 for ((i=0;i<ndevs;i++)); do
 157                     dev=${devs[i]}
 158                     devsize=$((bs*`sg_readcap -lb ${dev} | awk '{print $1}'`/1024))
 159                     if [ $devsize -lt $actual_size ]; then
 160                         _dev=$(sg_map | grep $dev | awk '{ print $2; }')
 161                         echo -e "device $_dev not big enough: $devsize <" \
 162                                 "$actual_size.\nConsider reducing \$size"
 163                         exit 1
 164                     fi
 165                     if [ $action = read ]; then
 166                         inf="if=$dev"
 167                         outf="of=/dev/null"
 168                         skip=skip
 169                     else
 170                         inf="if=/dev/zero"
 171                         outf="of=$dev"
 172                         skip=seek
 173                     fi
 174                     for ((j=0;j<crg;j++)); do
 175                         sgp_dd 2> ${tmpf}_${i}_${j} \
 176                             $inf $outf ${skip}=$((boundary+j*blocks)) \
 177                             thr=$((thr/crg)) count=$count bs=$bs bpt=$bpt time=1&
 178                     done
 179                 done
 180                 wait
 181                 t1=`date +%s.%N`
 182                 # collect/check individual stats
 183                 echo > $tmpf
 184                 ok=0
 185                 for ((i=0;i<ndevs;i++)); do
 186                     for ((j=0;j<crg;j++)); do
 187                         rtmp=${tmpf}_${i}_${j}
 188                         if grep 'error' $rtmp > /dev/null 2>&1; then
 189                                 echo "Error found in $rtmp"
 190                         elif grep 'time to transfer data' $rtmp > /dev/null 2>&1; then
 191                             ok=$((ok + 1))
 192                         fi
 193                         cat ${rtmp} >> $tmpf
 194                         cat ${rtmp} >> $workf
 195                         rm  ${rtmp}
 196                     done
 197                 done
 198                 if ((ok != ndevs*crg)); then
 199                     print_summary -n "$((ndevs*crg - ok)) failed "
 200                 else
 201                     # compute MB/sec from elapsed
 202                     bw=`awk "BEGIN {printf \"%7.2f MB/s\", $actual_size * $ndevs / (( $t1 - $t0 ) * 1024); exit}"`
 203                     # compute MB/sec from nregions*slowest
 204                     check=`awk < $tmpf \
 205                         '/time to transfer data/ {mb=$8/1.048576; if (n == 0 || mb < min) min = mb; n++}\
 206                         END {printf "%5d x %6.2f = %7.2f MB/s", n, min, min * n}'`
 207                     print_summary -n "$bw $check "
 208                 fi
 209                 rm $tmpf
 210             done
 211             print_summary ""
 212         done
 213     done
 214 done
 215
 216 if $sg_was_loaded; then
 217     echo "unloading sg module"
 218     rmmod sg
 219 fi