lustre-iokit/sgpdd-survey/sgpdd-survey

   1 #!/bin/bash
   2
   3 ######################################################################
   4 # customize per survey
   5
   6 # CHOOSE EITHER scsidevs or rawdevs
   7 # the SCSI devices to measure - WARNING: will be erased.
   8 # The raw devices to use
   9 # rawdevs=${rawdevs:-"/dev/raw/raw1"}
  10 # scsidevs=`ls /dev/sd[a-z] /dev/sd[a-z][a-z]` # all devices, if you use udev
  11
  12 # result file prefix.
  13 # NB ensure the path exists on all servers if it includes subdirs
  14 rslt_loc=${rslt_loc:-"/tmp"}
  15 rslt=${rslt:-"$rslt_loc/sgpdd_survey_`date +%F@%R`"}
  16
  17 # what to do (read or write)
  18 actions=${actions:-"write read"}
  19
  20 # total size per device (MBytes)
  21 # NB bigger than device cache is good
  22 size=${size:-8192}
  23
  24 # record size (KBytes)
  25 rszlo=${rszlo:-1024}
  26 rszhi=${rszhi:-1024}
  27
  28 # Concurrent regions per device
  29 crglo=${crglo:-1}
  30 crghi=${crghi:-256}
  31
  32 # boundary blocks between concurrent regions per device
  33 boundary=${boundary:-1024}
  34
  35 # threads to share between concurrent regions per device
  36 # multiple threads per region simulates a deeper request queue
  37 # NB survey skips over #thr < #regions and #thr/#regions > SG_MAX_QUEUE
  38 thrlo=${thrlo:-1}
  39 thrhi=${thrhi:-4096}
  40
  41 # NUMA support
  42 # User provided script that returns a cpu list from a specified device.
  43 # Implementation depends on the type of device (scsi/raw, with/without
  44 # multipath, technology fc/sas/ib)
  45 # For example:
  46 #   $ cat bin/dev2cpus
  47 #   #!/bin/bash
  48 #   dev=$(basename $1)
  49 #   pci=$(readlink -f /sys/class/block/$dev | cut -d/ -f1-5)
  50 #   cat ${pci}/local_cpulist
  51 dev2cpus=${dev2cpus:-""}
  52
  53 #####################################################################
  54 # leave the rest of this alone unless you know what you're doing...
  55
  56 # and max # threads one instance will spawn
  57 SG_MAX_QUEUE=16
  58
  59 # numactl command
  60 NUMACTL=${NUMACTL:-"/usr/bin/numactl"}
  61
  62 unique () {
  63     echo "$@" | xargs -n1 echo | sort -u
  64 }
  65
  66 split_hostname () {
  67     local name=$1
  68     case $name in
  69     *:*) host=`echo $name | sed 's/:.*$//'`
  70          name=`echo $name | sed 's/[^:]*://'`
  71          ;;
  72     *)   host=localhost
  73          ;;
  74     esac
  75     echo "$host $name"
  76 }
  77
  78 DSH=${DSH:-"ssh"}
  79
  80 dsh () {
  81     local node="$1"
  82     local user="$2"
  83     shift 2
  84     local command="$@"
  85
  86     command="export PATH=/sbin:/usr/sbin:\$PATH; $command"
  87
  88     case $DSH in
  89         ssh)
  90             if [ -n "$user" ]; then
  91                 user="$user@"
  92             fi
  93             $DSH $user$node "$command"
  94             ;;
  95         rsh)
  96             if [ -n "$user" ]; then
  97                 user="-l $user"
  98             fi
  99             $DSH $user $node "$command"
 100             ;;
 101     esac
 102 }
 103
 104 # how to run commands on other nodes
 105 remote_shell () {
 106     local host=$1
 107     shift
 108     local cmds="$@"
 109     if [ "$host" = "localhost" -o "$host" = `uname -n` ]; then
 110         eval "$cmds"
 111     else
 112         # split $host into $host and $user
 113         local user=""
 114         if [[ $host == *@* ]]; then
 115             user=${host%@*}
 116             host=${host#*@}
 117         fi
 118         dsh $host "$user" "$cmds"
 119     fi
 120 }
 121
 122
 123 # check either scsidevs or rawdevs is specified
 124 # but only one of them
 125 if [ -n "$scsidevs" -a -n "$rawdevs" -o -z "$scsidevs$rawdevs" ]; then
 126     echo "Must either specify scsidevs or rawdevs"
 127     exit 1
 128 fi
 129
 130 # retrieve host and device if specified as "hostname:device"
 131 ndevs=0
 132 devs=()
 133 for d in $scsidevs $rawdevs; do
 134     str=(`split_hostname $d`)
 135     hosts[$ndevs]=${str[0]}
 136     devs[$ndevs]=${str[1]}
 137     ndevs=$((ndevs+1))
 138 done
 139 unique_hosts=(`unique ${hosts[@]}`)
 140
 141 # get device cpu list
 142 devcpus=()
 143 if [ -n "$dev2cpus" ]; then
 144     for ((i=0; i < $ndevs; i++)); do
 145         devcpus[$i]=$(remote_shell ${hosts[$i]} $dev2cpus ${devs[$i]})
 146     done
 147 fi
 148
 149 # map given device names into SG device names
 150 if [ "$scsidevs" ]; then
 151     # make sure sg kernel module is loaded
 152     for host in ${unique_hosts[@]}; do
 153         sg_is_loaded=$(remote_shell $host grep -q "^sg " /proc/modules \
 154                        && echo true || echo false)
 155         if ! $sg_is_loaded; then
 156             echo "loading the sg kernel module on $host"
 157             remote_shell $host modprobe sg
 158             sg_was_loaded_on="$sg_was_loaded_on $host"
 159         fi
 160     done
 161
 162     for ((i=0; i < $ndevs; i++)); do
 163         # resolve symbolic link if any
 164         devs[$i]=$(remote_shell ${hosts[$i]} readlink -f ${devs[$i]})
 165
 166         # retrieve associated sg device
 167         # we will test for a LUN, the test for a partition
 168         # if the partition number is > 9 this will fail
 169         tmp=$(remote_shell ${hosts[$i]} sg_map | \
 170               awk -v dev=${devs[$i]} '{if ($2 == dev) print $1}')
 171         if [ -z "$tmp" ]; then
 172             echo "Can't find SG device for ${hosts[$i]}:${devs[$i]}, " \
 173                  "testing for partition"
 174             pt=`echo ${devs[$i]} | sed 's/[0-9]*$//'`
 175             # Try again
 176             tmp=$(remote_shell ${hosts[$i]} sg_map | \
 177                   awk -v dev=$pt '{if ($2 == dev) print $1}')
 178             if [ -z "$tmp" ]; then
 179                 echo -e "Can't find SG device ${hosts[$i]}:$pt.\n" \
 180                         "Do you have the sg module configured for your kernel?"
 181                 exit 1
 182            fi
 183         fi
 184         devs[$i]=$tmp
 185     done
 186 elif [ "$rawdevs" ]; then
 187     for ((i=0; i < $ndevs; i++)); do
 188         RES=$(remote_shell ${hosts[$i]} raw -q ${devs[$i]})
 189         if [ $? -ne 0 ];then
 190             echo "Raw device ${hosts[$i]}:${devs[$i]} not set up"
 191             exit 1
 192         fi
 193     done
 194 fi
 195
 196 # determine block size of each device. This should also work for raw devices
 197 # If it fails, set to 512
 198 for ((i=0; i < $ndevs; i++)); do
 199     # retrieve device size (in kbytes) and block size (in bytes)
 200     tmp=( `remote_shell ${hosts[$i]} sg_readcap -lb ${devs[$i]}` )
 201     bs[$i]=$((tmp[1]))
 202     if [ ${bs[$i]} == 0  ]; then
 203         echo "sg_readcap on device ${hosts[$i]}:${devs[$i]} failed, " \
 204              "setting block size to 512"
 205         bs[$i]=512
 206     fi
 207     devsize=$((tmp[0]*bs[$i]/1024))
 208
 209     # check record size is a multiple of block size
 210     if [ $((rszlo*1024%bs[$i])) -ne 0 ]; then
 211         echo "Record size is not a multiple of block size (${bs[$i]} bytes) " \
 212              "for device ${hosts[$i]}:${devs[$i]}"
 213         exit 1
 214     fi
 215
 216     # check device size
 217     if [ $devsize -lt $((size*1024)) ]; then
 218         echo -e "device ${hosts[$i]}:${devs[$i]} not big enough: " \
 219                 "$devsize < $((size*1024)).\nConsider reducing \$size"
 220         exit 1
 221     fi
 222 done
 223
 224 rsltf=${rslt}.summary
 225 workf=${rslt}.detail
 226 cmdsf=${rslt}.script
 227 echo -n > $rsltf
 228 echo -n > $workf
 229
 230 print_summary () {
 231     if [ "$1" = "-n" ]; then
 232         minusn=$1; shift
 233     else
 234         minusn=""
 235     fi
 236     echo $minusn "$*" >> $rsltf
 237     echo $minusn "$*"
 238 }
 239
 240 print_summary "$(date) sgpdd-survey on $rawdevs$scsidevs from $(hostname)"
 241
 242 for ((rsz=$rszlo;rsz<=$rszhi;rsz*=2)); do
 243     for ((crg=$crglo;crg<=$crghi;crg*=2)); do
 244         for ((thr=$thrlo;thr<=$thrhi;thr*=2)); do
 245             if ((thr < crg || thr/crg > SG_MAX_QUEUE)); then
 246                 continue
 247             fi
 248             # compute total size (in kbytes)
 249             total_size=0
 250             for ((i=0; i < $ndevs; i++)); do
 251                 tsize=$((size*1024*1024/bs[$i]/crg*crg*bs[$i]/1024))
 252                 total_size=$((total_size+tsize))
 253             done
 254             # show test parameters
 255             str=`printf 'dev %2d sz %8dK rsz %4dK crg %5d thr %5d ' \
 256                          $ndevs $total_size $rsz $((crg*ndevs)) $((thr*ndevs))`
 257             echo "==============> $str" >> $workf
 258             print_summary -n "$str"
 259
 260             # check memory for each host
 261             for host in ${unique_hosts[@]}; do
 262                 numdevs=0
 263                 for ((i=0; i < $ndevs; i++)); do
 264                     if [ ${hosts[$i]} == $host ]; then
 265                         numdevs=$((numdevs+1))
 266                     fi
 267                 done
 268                 freemem=$(remote_shell $host cat /proc/meminfo | \
 269                           awk '/^MemTotal:/ {printf "%d\n", $2}')
 270                 if (((rsz*thr/crg + 64)*crg*numdevs > freemem)); then
 271                     echo "ENOMEM on $host" >> $workf
 272                     print_summary "ENOMEM"
 273                     continue 2
 274                 fi
 275             done
 276
 277             # run tests
 278             for action in $actions; do
 279                 declare -a pidarray
 280                 print_summary -n "$action "
 281                 echo "=====> $action" >> $workf
 282                 tmpf=${workf}_tmp
 283
 284                 # create per-host script files
 285                 for host in ${unique_hosts[@]}; do
 286                     echo -n > ${cmdsf}_${host}
 287                 done
 288                 for ((i=0; i < $ndevs; i++)); do
 289                     bpt=$((rsz*1024/bs[$i]))
 290                     blocks=$((size*((1024*1024)/bs[$i])/crg))
 291                     count=$blocks
 292                     host=${hosts[$i]}
 293                     dev=${devs[$i]}
 294                     if [ $action = read ]; then
 295                         inf="if=$dev"
 296                         outf="of=/dev/null"
 297                         skip=skip
 298                     else
 299                         inf="if=/dev/zero"
 300                         outf="of=$dev"
 301                         skip=seek
 302                     fi
 303                     if [ -n "${devcpus[$i]}" -a -x "$NUMACTL" ]; then
 304                         numacmd="$NUMACTL --physcpubind=${devcpus[$i]} --localalloc"
 305                     else
 306                         numacmd=""
 307                     fi
 308                     for ((j=0;j<crg;j++)); do
 309                         echo >> ${cmdsf}_${host} \
 310                                 "$numacmd " \
 311                                 "sgp_dd 2> ${tmpf}_${i}_${j} $inf $outf " \
 312                                 "${skip}=$((boundary+j*blocks)) " \
 313                                 "thr=$((thr/crg)) count=$count bs=${bs[$i]} " \
 314                                 "bpt=$bpt time=1&"
 315                     done
 316                 done
 317                 for host in ${unique_hosts[@]}; do
 318                     echo "wait" >> ${cmdsf}_${host}
 319                 done
 320
 321                 # run of all the per-host script files
 322                 t0=`date +%s.%N`
 323                 pidcount=0
 324                 for host in ${unique_hosts[@]}; do
 325                     remote_shell $host bash < ${cmdsf}_${host} &
 326                     pidarray[$pidcount]=$!
 327                     pidcount=$((pidcount+1))
 328                 done
 329                 pidcount=0
 330                 for host in ${unique_hosts[@]}; do
 331                     wait ${pidarray[$pidcount]}
 332                     pidcount=$((pidcount+1))
 333                 done
 334                 t1=`date +%s.%N`
 335
 336                 # clean up per-host script files
 337                 for host in ${unique_hosts[@]}; do
 338                     rm ${cmdsf}_${host}
 339                 done
 340
 341                 # collect/check individual stats
 342                 echo > $tmpf
 343                 ok=0
 344                 for ((i=0;i<ndevs;i++)); do
 345                     for ((j=0;j<crg;j++)); do
 346                         rtmp=${tmpf}_${i}_${j}_local
 347                         remote_shell ${hosts[$i]} cat ${tmpf}_${i}_${j} > $rtmp
 348                         if grep 'error' $rtmp > /dev/null 2>&1; then
 349                             echo "Error found in $rtmp"
 350                         elif grep 'time to transfer data' $rtmp > /dev/null 2>&1; then
 351                             ok=$((ok + 1))
 352                         fi
 353                         cat ${rtmp} >> $tmpf
 354                         cat ${rtmp} >> $workf
 355                         rm  ${rtmp}
 356                         remote_shell ${hosts[$i]} rm ${tmpf}_${i}_${j}
 357                     done
 358                 done
 359                 if ((ok != ndevs*crg)); then
 360                     print_summary -n "$((ndevs*crg - ok)) failed "
 361                 else
 362                     # compute bandwidth in MiB/s from total data / elapsed time
 363                     bw=`awk "BEGIN {printf \"%7.2f \", \
 364                                     $total_size / (( $t1 - $t0 ) * 1024); exit}"`
 365                     # compute global min/max stats
 366                     minmax=`awk < $tmpf \
 367                         '/time to transfer data/ {mb=$8/1.048576; \
 368                                                   if (n == 0 || mb < min) min = mb; \
 369                                                   if (n == 0 || mb > max) max = mb; \
 370                                                   n++} \
 371                         END {printf "[ %7.2f, %7.2f] ",min,max;}'`
 372                     print_summary -n "$bw $minmax "
 373                 fi
 374                 rm $tmpf
 375             done
 376             print_summary ""
 377         done
 378     done
 379 done
 380
 381 for host in $sg_was_loaded_on; do
 382     echo "unloading sg module on $host"
 383     remote_shell $host rmmod sg
 384 done