lustre-iokit/ior-survey/ior-survey

   1 #!/bin/bash
   2
   3 # cluster name (all node names are this followed by the node number)
   4 cluster=mdev
   5
   6 # client node numbers (individual numbers or inclusive ranges)
   7 clients=(7-8)
   8
   9 # numbers of clients to survey
  10 clients_lo=1
  11 clients_hi=2
  12 clients_iterator="+=1"
  13
  14 # numbers of tasks per client to survey
  15 tasks_per_client_lo=1
  16 tasks_per_client_hi=8
  17 tasks_per_client_iterator="*=2"
  18
  19 # record sizes to survey
  20 rsize_lo=1M
  21 rsize_hi=1M
  22 rsize_iterator="*=2"
  23
  24 ## which tests to run (first must be write)
  25 # remount)   not really a test; just remount to uncache everything
  26 # *write*)   write
  27 # *)         read
  28 #tests=(write rewrite read reread rewrite_again)
  29 tests=(write rewrite remount read reread)
  30
  31 # total # bytes written/read by any client node
  32 min_per_client_size=4G
  33 min_total_size=8G
  34
  35 # should each task do I/O to its own file?
  36 file_per_task=1
  37
  38 # the binaries
  39 IOR="/home/ericb/ior/src/C/IOR"
  40 llmount=/home/ericb/lustre/utils/llmount
  41 pdsh=pdsh
  42
  43 # the result file prefix (date/time + hostname makes unique)
  44 #rslt=/home/ericb/ior_survey_`date +%F@%R`_`uname -n`
  45 rslt=/home/ericb/ior_survey
  46
  47 # where lustre is mounted on the clients
  48 lustre=/mnt/lustre
  49
  50 # basename of the test file(s)
  51 testfile=${lustre}/ior_survey_testfile
  52
  53 # how to unmount and remount the F/S on a client (to clear the cache)
  54 # change this depending on lustre config (network type, MDS etc)
  55 remount="umount $lustre && $llmount -o nettype=elan mdev6:/ll_mds/client $lustre"
  56
  57 # pdsh args required to instantiate all instances of IOR in parallel
  58 # the chosen module must support '-n <procs-per-node>'
  59 # -R<module>, -f<fanout> etc
  60 pdsh_mpiargs="-Rmqsh"
  61
  62 #don't spin for MPI completions
  63 export LIBELAN_WAITTYPE=0
  64
  65 ################################################################################
  66 # dont change stuff below here unless you know what you're doing...
  67
  68 count_range() {
  69     echo $1 | awk '{ nvals=split($1, vals, "-");\
  70                      if (nvals == 1) print 1;\
  71                      else if (nvals == 2) printf "%d\n", vals[2] - vals[1] + 1;}'
  72 }
  73
  74 base_range() {
  75     echo $1 | awk '{ split($1, vals, "-"); print vals[1]; }'
  76 }
  77
  78 idx2nodenum() {
  79     local n=$1; shift
  80     while ((1)); do
  81         local range=$1; shift
  82         if [ -z "$range" ]; then
  83             return
  84         fi
  85         chunk=`count_range $range`
  86         if ((chunk > n)); then
  87             base=`base_range $range`
  88             echo $((base + n))
  89             return
  90         fi
  91         n=$((n-chunk))
  92     done
  93 }
  94
  95 n2noderange() {
  96     local n=$1; shift
  97     sep=""
  98     nodes="["
  99     while ((n > 0)); do
 100         local range=$1; shift
 101         if [ -z "$range" ]; then
 102             return
 103         fi
 104         local base=`base_range $range`
 105         local chunk=`count_range $range`
 106         if ((chunk > n)); then chunk=n; fi
 107         local nodes="${nodes}${sep}${base}"; sep=","
 108         if ((chunk > 1)); then nodes="${nodes}-$((base+chunk-1))"; fi
 109         n=$((n-chunk))
 110     done
 111     echo "${nodes}]"
 112 }
 113
 114 countnodes() {
 115     local radix=16384
 116     local n=0
 117     while ((radix > 0)); do
 118         local nodes=`n2noderange $((n+radix)) $@`
 119         if [ -n "$nodes" ]; then
 120             n=$((n+radix))
 121         fi
 122         radix=$((radix/2))
 123     done
 124     echo $n
 125 }
 126
 127 parse_number() {
 128     local str=$1
 129     case $str in
 130         *G|*g) n=`echo $str | sed 's/[gG]//'`; echo $((n*1024*1024*1024));;
 131         *M|*m) n=`echo $str | sed 's/[Mm]//'`; echo $((n*1024*1024));;
 132         *K|*k) n=`echo $str | sed 's/[Kk]//'`; echo $((n*1024));;
 133         *)     echo $1;;
 134     esac
 135 }
 136
 137 pp_number() {
 138     local n=$1
 139     local G=$((1024*1024*1024))
 140     local M=$((1024*1024))
 141     local K=$((1024))
 142     if ((n%G == 0 && n >= G)); then
 143         echo "$((n/G))G"
 144     elif ((n%M == 0 && n >= M)); then
 145         echo "$((n/M))M"
 146     elif ((n%K == 0 && n >= K)); then
 147         echo "$((n/K))K"
 148     else
 149         echo $n
 150     fi
 151 }
 152
 153 if [ ${#tests[@]} -eq 0 -o "${tests[0]}" != "write" ]; then
 154     echo "First test must be 'write'" 1>&2
 155     exit 1
 156 fi
 157
 158 rsltf="${rslt}.summary"
 159 workf="${rslt}.detail"
 160 echo -n > $rsltf
 161 echo -n > $workf
 162
 163 print_summary () {
 164     if [ "$1" = "-n" ]; then
 165         minusn=$1; shift
 166     else
 167         minusn=""
 168     fi
 169     echo $minusn "$*" >> $rsltf
 170     echo $minusn "$*"
 171 }
 172
 173 # convert params to actual numbers
 174 min_per_client_size=`parse_number $min_per_client_size`
 175 min_total_size=`parse_number $min_total_size`
 176
 177 rsize_lo=`parse_number $rsize_lo`
 178 rsize_hi=`parse_number $rsize_hi`
 179
 180 # check on actual numbers of client nodes
 181 nclients=`countnodes ${clients[@]}`
 182 if ((clients_hi > nclients)); then clients_hi=$nclients; fi
 183
 184 for ((rsize=rsize_lo; rsize<=rsize_hi; rsize$rsize_iterator)); do
 185     pp_rsize=`pp_number $rsize`
 186
 187     for ((nclnt=clients_lo; nclnt<=clients_hi; nclnt$clients_iterator)); do
 188         test_clients="${cluster}`n2noderange $nclnt ${clients[@]}`"
 189
 190         per_client_size=$((min_total_size/nclnt))
 191         if ((per_client_size < min_per_client_size)); then
 192             per_client_size=$min_per_client_size
 193         fi
 194
 195         for ((ntask=tasks_per_client_lo; ntask <= tasks_per_client_hi; ntask$tasks_per_client_iterator)); do
 196             per_task_size=$((per_client_size/ntask))
 197             if ((per_task_size%rsize != 0)); then
 198                 per_task_size=$(((per_task_size/rsize + 1)*rsize))
 199             fi
 200             total_size=`pp_number $((per_task_size*nclnt*ntask))`
 201
 202             hdrstr=`printf "Total: %5sB rsize: %4sB clients: %4d tasks: %3d: " \
 203                 $total_size $pp_rsize $nclnt $ntask`
 204             print_summary -n "$hdrstr"
 205
 206             for ((test_idx=0; test_idx < ${#tests[@]}; test_idx++)); do
 207                 test=${tests[$test_idx]}
 208
 209                 print_summary -n "$test "
 210                 echo "===========> ${hdrstr} on $test_clients doing $test" >> $workf
 211                 tmpf=${workf}_tmp
 212                 echo -n > $tmpf
 213
 214                 if [ "$test" = "remount" ]; then
 215                     echo "=> $remount" >> $tmpf
 216                     $pdsh -S -b -w "$test_clients" >> $tmpf 2>&1 \
 217                         "$remount"
 218                     status=$?
 219                     echo "Completion Status: $status" >> $tmpf
 220
 221                     if ((status)); then
 222                         result="ERROR"
 223                     else
 224                         result="OK"
 225                     fi
 226                 else
 227                     # check lustre is mounted everywhere it's needed
 228                     cmd="(mount -t lustre; mount -t lustre_lite) | grep $lustre"
 229                     echo "=> Mount Check: $cmd" >> $tmpf
 230                     $pdsh -S -b -w "$test_clients" >> $tmpf 2>&1 \
 231                         "$cmd"
 232                     status=$?
 233                     echo "Completion Status: $status" >> $tmpf
 234                     if ((status)); then
 235                         cat $tmpf >> $workf
 236                         rm $tmpf
 237                         print_summary "Lustre NOT mounted on $lustre somewhere"
 238                         exit 1
 239                     fi
 240
 241                     cmdline=(
 242                     $IOR                     # the command
 243                     -o${testfile}            # test file prefix
 244                     -b${per_task_size}       # bytes per task
 245                     -t${rsize}               # record size
 246                     -e                       # fsync before close
 247                     -q                       # quit on error
 248                     )
 249
 250                     idx=${#cmdline[@]}
 251
 252                     # keep the test file(s) unless this is the last test
 253                     ((test_idx < ${#tests[@]}-1)) && cmdline[$((idx++))]="-k"
 254
 255                     # use the existing test file(s) unless this is the first test
 256                     ((test_idx > 0)) && cmdline[$((idx++))]="-E"
 257
 258                     # file-per-task
 259                     ((file_per_task)) && cmdline[$((idx++))]="-F"
 260
 261                     case "$test" in
 262                     *write*) cmdline[$((idx++))]="-w"
 263                              awkstr="Max Write";;
 264                     *)       cmdline[$((idx++))]="-r"
 265                              awkstr="Max Read";;
 266                     esac
 267
 268                     echo "=> ${cmdline[@]}" >> $tmpf
 269
 270                     $pdsh -S -b $pdsh_mpiargs -w "$test_clients" -n $ntask >> $tmpf 2>&1 \
 271                         "${cmdline[@]}"
 272                     status=$?
 273
 274                     echo "Completion Status: $status" >> $tmpf
 275
 276                     if ((status)); then
 277                         result="ERROR"
 278                     else
 279                         result=`awk < $tmpf "/$awkstr/ {print $ 3; found=1; exit}\
 280                                              END       {if (!found) print \"ERROR\"}"`
 281                     fi
 282                 fi
 283
 284                 cat $tmpf >> $workf
 285                 rm $tmpf
 286
 287                 str=`printf "%8s" "$result"`
 288                 print_summary -n "$str "
 289             done
 290             print_summary ""
 291         done
 292     done
 293 done
 294