lustre-iokit/ior-survey/ior-survey

   1 #!/bin/bash
   2
   3 # cluster name (all node names are this followed by the node number)
   4 cluster=nid000
   5
   6 # client node numbers (individual numbers or inclusive ranges)
   7 clients=(1-64)
   8
   9 # numbers of clients to survey
  10 clients_lo=1
  11 clients_hi=64
  12 clients_iterator="*=2"
  13
  14 # numbers of tasks per client to survey
  15 tasks_per_client_lo=1
  16 tasks_per_client_hi=1
  17 tasks_per_client_iterator="*=2"
  18
  19 # record sizes to survey
  20 rsize_lo=1M
  21 rsize_hi=1M
  22 rsize_iterator="*=2"
  23
  24 # This line contains all of the possible tests.
  25 # IMPORTANT:::if you want to remount, put it in your tests array
  26 # in the order you want to remount to clear the cache.  For ex:
  27 # tests=(write rewrite remount read reread rewrite_again) has a
  28 # remount between the rewrite and remount tests
  29 tests=(write read)
  30
  31 # total # bytes written/read by any client node
  32 min_per_client_size=1G
  33 min_total_size=1G
  34
  35 # should each task do I/O to its own file?
  36 file_per_task=1
  37
  38 # the binaries
  39 IOR="/spin/home/henken/IOR-2.8.4/src/C/IOR"
  40 llmount=llmount
  41
  42 #Command to run IOR (pdsh,mpirun,yod)
  43 runior="yod"
  44 #Path to binary for program specified in runior
  45 pathtobin="$(which yod)"
  46 #location of machines file for mpirun, this file
  47 #will be built from the cluster and client ranges
  48 #above
  49 machines=machines
  50
  51 # the result file prefix (date/time + hostname makes unique)
  52 rslt=/spin/home/henken/ior_survey/ior_survey_`date +%F@%R`_`uname -n`
  53 #rslt=/home/larry/ior_survey
  54
  55 # where lustre is mounted on the clients
  56 lustre=/lustre/fs1/nic/
  57
  58 # basename of the test file(s)
  59 testfile=${lustre}/ior_survey_testfile
  60
  61
  62 # how to unmount and remount the F/S on a client (to clear the cache)
  63 # change this depending on lustre config (network type, MDS etc)
  64 unmount="umount $lustre"
  65 remount="llmount pegasus:/mds1/client $lustre"
  66
  67 # pdsh args required to instantiate all instances of IOR in parallel
  68 # the chosen module must support '-n <procs-per-node>'
  69 # -R<module>, -f<fanout> etc
  70 pdsh_mpiargs="-Rmqsh"
  71
  72 #don't spin for MPI completions
  73 export LIBELAN_WAITTYPE=0
  74
  75 ################################################################################
  76 # dont change stuff below here unless you know what you're doing...
  77
  78 count_range() {
  79     echo $1 | awk '{ nvals=split($1, vals, "-");\
  80                      if (nvals == 1) print 1;\
  81                      else if (nvals == 2) printf "%d\n", vals[2] - vals[1] + 1;}'
  82 }
  83
  84 base_range() {
  85     echo $1 | awk '{ split($1, vals, "-"); print vals[1]; }'
  86 }
  87
  88 idx2nodenum() {
  89     local n=$1; shift
  90     while ((1)); do
  91         local range=$1; shift
  92         if [ -z "$range" ]; then
  93             return
  94         fi
  95         chunk=`count_range $range`
  96         if ((chunk > n)); then
  97             base=`base_range $range`
  98             echo $((base + n))
  99             return
 100         fi
 101         n=$((n-chunk))
 102     done
 103 }
 104
 105 n2noderange() {
 106     local n=$1; shift
 107     sep=""
 108     nodes="["
 109     while ((n > 0)); do
 110         local range=$1; shift
 111         if [ -z "$range" ]; then
 112             return
 113         fi
 114         local base=`base_range $range`
 115         local chunk=`count_range $range`
 116         if ((chunk > n)); then chunk=n; fi
 117         local nodes="${nodes}${sep}${base}"; sep=","
 118         if ((chunk > 1)); then nodes="${nodes}-$((base+chunk-1))"; fi
 119         n=$((n-chunk))
 120     done
 121     echo "${nodes}]"
 122 }
 123
 124 countnodes() {
 125     local radix=16384
 126     local n=0
 127     while ((radix > 0)); do
 128         local nodes=`n2noderange $((n+radix)) $@`
 129         if [ -n "$nodes" ]; then
 130             n=$((n+radix))
 131         fi
 132         radix=$((radix/2))
 133     done
 134     echo $n
 135 }
 136
 137 parse_number() {
 138     local str=$1
 139     case $str in
 140         *G|*g) n=`echo $str | sed 's/[gG]//'`; echo $((n*1024*1024*1024));;
 141         *M|*m) n=`echo $str | sed 's/[Mm]//'`; echo $((n*1024*1024));;
 142         *K|*k) n=`echo $str | sed 's/[Kk]//'`; echo $((n*1024));;
 143         *)     echo $1;;
 144     esac
 145 }
 146
 147 pp_number() {
 148     local n=$1
 149     local G=$((1024*1024*1024))
 150     local M=$((1024*1024))
 151     local K=$((1024))
 152     if ((n%G == 0 && n >= G)); then
 153         echo "$((n/G))G"
 154     elif ((n%M == 0 && n >= M)); then
 155         echo "$((n/M))M"
 156     elif ((n%K == 0 && n >= K)); then
 157         echo "$((n/K))K"
 158     else
 159         echo $n
 160     fi
 161 }
 162
 163 if [ ${#tests[@]} -eq 0 -o "${tests[0]}" != "write" ]; then
 164     echo "First test must be 'write'" 1>&2
 165     exit 1
 166 fi
 167
 168 rsltf="${rslt}.summary"
 169 workf="${rslt}.detail"
 170 echo -n > $rsltf
 171 echo -n > $workf
 172
 173 print_summary () {
 174     if [ "$1" = "-n" ]; then
 175         minusn=$1; shift
 176     else
 177         minusn=""
 178     fi
 179     echo $minusn "$*" >> $rsltf
 180     echo $minusn "$*"
 181 }
 182
 183 mpi_client_file() {
 184         echo -n > $machines
 185         local base=`base_range $1`
 186         echo $base
 187         local chunk=`count_range $1`
 188         echo $chunk
 189         local high=$((base+chunk-1))
 190         echo $high
 191         for ((nmpi=$base; nmpi<=$high; nmpi++)); do
 192                 echo $cluster$nmpi >> $machines
 193         done
 194 }
 195
 196 parse_cmdline() {
 197     case $runior in
 198                 'mpirun')
 199                 #echo "this"
 200                 $pathtobin -np $((ntask*nclnt)) -machinefile $machines >> $tmpf 2>1 \
 201                         "${cmdline[@]}";;
 202                 'pdsh')
 203                 $pathtobin -S -b $pdsh_mpiargs -w "$test_clients" -n $ntask \
 204                         >> $tmpf 2>&1 "${cmdline[@]}";;
 205                 'yod')
 206                         $pathtobin -np $((ntask*nclnt)) >> $tmpf 2>&1 "${cmdline[@]}";;
 207         esac
 208 }
 209
 210 if [ $runior = "mpirun" ]; then
 211         mpi_client_file ${clients[@]}
 212 fi
 213
 214 # convert params to actual numbers
 215 min_per_client_size=`parse_number $min_per_client_size`
 216 min_total_size=`parse_number $min_total_size`
 217
 218 rsize_lo=`parse_number $rsize_lo`
 219 rsize_hi=`parse_number $rsize_hi`
 220
 221 # check on actual numbers of client nodes
 222 nclients=`countnodes ${clients[@]}`
 223 if ((clients_hi > nclients)); then clients_hi=$nclients; fi
 224
 225 for ((rsize=rsize_lo; rsize<=rsize_hi; rsize$rsize_iterator)); do
 226     pp_rsize=`pp_number $rsize`
 227
 228     for ((nclnt=clients_lo; nclnt<=clients_hi; nclnt$clients_iterator)); do
 229         test_clients="${cluster}`n2noderange $nclnt ${clients[@]}`"
 230
 231         per_client_size=$((min_total_size/nclnt))
 232         if ((per_client_size < min_per_client_size)); then
 233             per_client_size=$min_per_client_size
 234         fi
 235
 236         for ((ntask=tasks_per_client_lo; ntask <= tasks_per_client_hi; ntask$tasks_per_client_iterator)); do
 237             per_task_size=$((per_client_size/ntask))
 238             if ((per_task_size%rsize != 0)); then
 239                 per_task_size=$(((per_task_size/rsize + 1)*rsize))
 240             fi
 241             total_size=`pp_number $((per_task_size*nclnt*ntask))`
 242
 243             hdrstr=`printf "Total: %5sB rsize: %4sB clients: %4d tasks: %3d: " \
 244                 $total_size $pp_rsize $nclnt $ntask`
 245             print_summary -n "$hdrstr"
 246
 247             for ((test_idx=0; test_idx < ${#tests[@]}; test_idx++)); do
 248                 test=${tests[$test_idx]}
 249
 250                 print_summary -n "$test "
 251                 echo "===========> ${hdrstr} on $test_clients doing $test" >> $workf
 252                 tmpf=${workf}_tmp
 253                 echo -n > $tmpf
 254
 255                 if [ "$test" = "remount" ]; then
 256                     echo "=> $remount" >> $tmpf
 257                     if [ "$runior" = "pdsh" ]; then
 258                                 $pdsh -S -b -w "$test_clients" >> $tmpf 2>&1 \
 259                                         "$unmount"
 260                                 $pdsh -S -b -w "$test_clients" >> $tmpf 2>&1 \
 261                                         "$remount"
 262                     else
 263                                 $unmount
 264                                 $remount
 265                     fi
 266                     status=$?
 267                     echo "Completion Status: $status" >> $tmpf
 268
 269                     if ((status)); then
 270                         result="ERROR"
 271                     else
 272                         result="OK"
 273                     fi
 274                 else
 275                     # check lustre is mounted everywhere it's needed
 276                     cmd="(mount -t lustre; mount -t lustre_lite) | grep $lustre"
 277                     echo "=> Mount Check: $cmd" >> $tmpf
 278                     if [ "$runior" = "pdsh" ]; then
 279                                 $pdsh -S -b -w "$test_clients" >> $tmpf 2>&1 \
 280                                         "$cmd"
 281                     fi
 282                     status=$?
 283                     echo "Completion Status: $status" >> $tmpf
 284                     if ((status)); then
 285                         cat $tmpf >> $workf
 286                         rm $tmpf
 287                         print_summary "Lustre NOT mounted on $lustre somewhere"
 288                         exit 1
 289                     fi
 290
 291                     cmdline=(
 292                     $IOR                     # the command
 293                     -o${testfile}            # test file prefix
 294                     -b${per_task_size}       # bytes per task
 295                     -t${rsize}               # record size
 296                     -e                       # fsync before close
 297                     -q                       # quit on error
 298                     )
 299
 300                     idx=${#cmdline[@]}
 301
 302                     # keep the test file(s) unless this is the last test
 303                     ((test_idx < ${#tests[@]}-1)) && cmdline[$((idx++))]="-k"
 304
 305                     # use the existing test file(s) unless this is the first test
 306                     ((test_idx > 0)) && cmdline[$((idx++))]="-E"
 307
 308                     # file-per-task
 309                     ((file_per_task)) && cmdline[$((idx++))]="-F"
 310
 311                     case "$test" in
 312                     *write*) cmdline[$((idx++))]="-w"
 313                              awkstr="Max Write";;
 314                     *)       cmdline[$((idx++))]="-r"
 315                              awkstr="Max Read";;
 316                     esac
 317
 318                     echo "=> ${cmdline[@]}" >> $tmpf
 319
 320                     parse_cmdline
 321                     status=$?
 322
 323                     echo "Completion Status: $status" >> $tmpf
 324
 325                     if ((status)); then
 326                         result="ERROR"
 327                     else
 328                         result=`awk < $tmpf "/$awkstr/ {print $ 3; found=1; exit}\
 329                                              END       {if (!found) print \"ERROR\"}"`
 330                     fi
 331                 fi
 332
 333                 cat $tmpf >> $workf
 334                 rm $tmpf
 335
 336                 str=`printf "%8s" "$result"`
 337                 print_summary -n "$str "
 338             done
 339             print_summary ""
 340         done
 341     done
 342 done
 343