lustre-iokit/ior-survey/ior-survey

   1 #!/bin/bash
   2
   3 # cluster name (expect all node names to be this followed by a number)
   4 cluster=mdev
   5
   6 # client node numbers (individual numbers or inclusive ranges)
   7 clients=(7-8)
   8
   9 # numbers of clients to survey
  10 clients_lo=1
  11 clients_hi=2
  12 clients_iterator="+=1"
  13
  14 # numbers of tasks per client to survey
  15 tasks_per_client_lo=1
  16 tasks_per_client_hi=16
  17 tasks_per_client_iterator="*=4"
  18
  19 # record sizes to survey
  20 rsize_lo=1M
  21 rsize_hi=1M
  22 rsize_iterator="*=2"
  23
  24 ## which tests to run (first must be write)
  25 # remount)   not really a test; just remount to uncache everything
  26 # *write*)   write
  27 # *)         read
  28 #tests=(write rewrite read reread rewrite_again)
  29 tests=(write rewrite remount read)
  30
  31 # total # bytes written/read by any client node
  32 min_per_client_size=75M
  33 min_total_size=100M
  34
  35 # should each task do I/O to its own file?
  36 file_per_task=1
  37
  38 # the IOR binary
  39 IOR="/home/ericb/ior/src/C/IOR"
  40
  41 # the pdsh binary
  42 pdsh=pdsh
  43
  44 # the llmount binary
  45 llmount=/home/ericb/lustre/utils/llmount
  46
  47 # where lustre is mounted on the clients
  48 lustre=/mnt/lustre
  49
  50 # basename of the test file(s)
  51 testfile=${lustre}/ior_survey_testfile
  52
  53 # how to unmount and remount the F/S on a client (to clear the cache)
  54 remount="umount $lustre && $llmount -o nettype=elan mdev6:/ll_mds/client $lustre"
  55
  56 # the result file prefix (date/time + hostname makes unique)
  57 #rslt=/home/ericb/ior_survey_`date +%F@%R`_`uname -n`
  58 rslt=/home/ericb/ior_survey
  59
  60 #don't spin for MPI completions
  61 export LIBELAN_WAITTYPE=0
  62
  63 ################################################################################
  64 # dont change stuff below here
  65
  66 count_range() {
  67     echo $1 | awk '{ nvals=split($1, vals, "-");\
  68                      if (nvals == 1) print 1;\
  69                      else if (nvals == 2) printf "%d\n", vals[2] - vals[1] + 1;}'
  70 }
  71
  72 base_range() {
  73     echo $1 | awk '{ split($1, vals, "-"); print vals[1]; }'
  74 }
  75
  76 idx2nodenum() {
  77     n=$1; shift
  78     while ((1)); do
  79         range=$1; shift
  80         if [ -z "$range" ]; then
  81             return
  82         fi
  83         chunk=`count_range $range`
  84         if ((chunk > n)); then
  85             base=`base_range $range`
  86             echo $((base + n))
  87             return
  88         fi
  89         n=$((n-chunk))
  90     done
  91 }
  92
  93 n2noderange() {
  94     n=$1; shift
  95     sep=""
  96     nodes="["
  97     while ((n > 0)); do
  98         range=$1; shift
  99         if [ -z "$range" ]; then
 100             return
 101         fi
 102         base=`base_range $range`
 103         chunk=`count_range $range`
 104         if ((chunk > n)); then chunk=n; fi
 105         nodes="${nodes}${sep}${base}"; sep=","
 106         if ((chunk > 1)); then nodes="${nodes}-$((base+chunk-1))"; fi
 107         n=$((n-chunk))
 108     done
 109     echo "${nodes}]"
 110 }
 111
 112 countnodes() {
 113     radix=16384
 114     n=0
 115     while ((radix > 0)); do
 116         nodes=`n2noderange $((n+radix)) $@`
 117         if [ -n "$nodes" ]; then
 118             n=$((n+radix))
 119         fi
 120         radix=$((radix/2))
 121     done
 122     echo $n
 123 }
 124
 125 parse_number() {
 126     str=$1
 127     case $str in
 128         *G|*g) n=`echo $str | sed 's/[gG]//'`; echo $((n*1024*1024*1024));;
 129         *M|*m) n=`echo $str | sed 's/[Mm]//'`; echo $((n*1024*1024));;
 130         *K|*k) n=`echo $str | sed 's/[Kk]//'`; echo $((n*1024));;
 131         *)     echo $1;;
 132     esac
 133 }
 134
 135 pp_number() {
 136     n=$1
 137     G=$((1024*1024*1024))
 138     M=$((1024*1024))
 139     K=$((1024))
 140     if ((n%G == 0 && n >= G)); then
 141         echo "$((n/G))G"
 142     elif ((n%M == 0 && n >= M)); then
 143         echo "$((n/M))M"
 144     elif ((n%K == 0 && n >= K)); then
 145         echo "$((n/K))K"
 146     else
 147         echo $n
 148     fi
 149 }
 150
 151 if [ ${#tests[@]} -eq 0 -o "${tests[0]}" != "write" ]; then
 152     echo "First test must be 'write'" 1>&2
 153     exit 1
 154 fi
 155
 156 rsltf="${rslt}.summary"
 157 iorcf="${rslt}.script"
 158 workf="${rslt}.detail"
 159 tmpf="${workf}_tmp"
 160
 161 echo -n > $rsltf
 162 echo -n > $workf
 163
 164 print_summary () {
 165     if [ "$1" = "-n" ]; then
 166         minusn=$1; shift
 167     else
 168         minusn=""
 169     fi
 170     echo $minusn "$*" >> $rsltf
 171     echo $minusn "$*"
 172 }
 173
 174 min_per_client_size=`parse_number $min_per_client_size`
 175 min_total_size=`parse_number $min_total_size`
 176
 177 rsize_lo=`parse_number $rsize_lo`
 178 rsize_hi=`parse_number $rsize_hi`
 179
 180 nclients=`countnodes ${clients[@]}`
 181
 182 if ((clients_hi > nclients)); then clients_hi=$nclients; fi
 183
 184 for ((rsize=rsize_lo; rsize<=rsize_hi; rsize$rsize_iterator)); do
 185     pp_rsize=`pp_number $rsize`
 186
 187     for ((nclnt=clients_lo; nclnt<=clients_hi; nclnt$clients_iterator)); do
 188         test_clients="${cluster}`n2noderange $nclnt ${clients[@]}`"
 189
 190         per_client_size=$((min_total_size/nclnt))
 191         if ((per_client_size < min_per_client_size)); then
 192             per_client_size=$min_per_client_size
 193         fi
 194         total_size=`pp_number $((per_client_size * nclnt))`
 195
 196         for ((ntask=tasks_per_client_lo; ntask <= tasks_per_client_hi; ntask$tasks_per_client_iterator)); do
 197             per_task_size=$((per_client_size/ntask))
 198
 199             hdrstr=`printf "Total: %5sB rsize: %4s clients: %4d tasks: %3d: " $total_size $rsize $nclnt $ntask`
 200             print_summary -n "$hdrstr"
 201
 202             for ((test_idx=0; test_idx < ${#tests[@]}; test_idx++)); do
 203                 test=${tests[$test_idx]}
 204
 205                 print_summary -n "$test "
 206                 echo "===========> ${hdrstr} on $test_clients doing $test" >> $workf
 207                 echo -n > $tmpf
 208
 209                 if [ "$test" = "remount" ]; then
 210                     echo "=> $remount" >> $tmpf
 211                     $pdsh -S -b -w "$test_clients" >> $tmpf 2>&1 \
 212                         "$remount"
 213                     status=$?
 214                     echo "Completion Status: $status" >> $tmpf
 215
 216                     if ((status)); then
 217                         result="ERROR"
 218                     else
 219                         result="OK"
 220                     fi
 221                 else
 222                     cmd="(mount -t lustre; mount -t lustre_lite) | grep $lustre"
 223                     echo "=> $cmd" >> $tmpf
 224                     $pdsh -S -b -w "$test_clients" >> $tmpf 2>&1 \
 225                         "$cmd"
 226                     status=$?
 227                     echo "Completion Status: $status" >> $tmpf
 228
 229                     if ((status)); then
 230                         cat $tmpf >> $workf
 231                         rm $tmpf
 232                         print_summary "Lustre NOT mounted on $lustre somewhere"
 233                         exit 1
 234                     fi
 235
 236                     cmdline=(
 237                     $IOR                     # the command
 238                     -o${testfile}            # test file prefix
 239                     -b${per_task_size}       # bytes per task
 240                     -t${rsize}               # record size
 241                     -e                       # fsync before close
 242                     -q                       # quit on error
 243                     )
 244
 245                     idx=${#cmdline[@]}
 246
 247                     # keep the test file(s) unless this is the last test
 248                     ((test_idx < ${#tests[@]}-1)) && cmdline[$((idx++))]="-k"
 249
 250                     # use the existing test file(s) unless this is the first test
 251                     ((test_idx > 0)) && cmdline[$((idx++))]="-E"
 252
 253                     # file-per-task
 254                     ((file_per_task)) && cmdline[$((idx++))]="-F"
 255
 256                     case "$test" in
 257                     *write*) cmdline[$((idx++))]="-w"
 258                              awkstr="Max Write";;
 259                     *)       cmdline[$((idx++))]="-r"
 260                              awkstr="Max Read";;
 261                     esac
 262
 263                     echo "=> ${cmdline[@]}" >> $tmpf
 264
 265                     $pdsh -S -b -Rmqsh -w "$test_clients" -n $ntask >> $tmpf 2>&1 \
 266                         "${cmdline[@]}"
 267                     status=$?
 268
 269                     echo "Completion Status: $status" >> $tmpf
 270
 271                     if ((status)); then
 272                         result="ERROR"
 273                     else
 274                         result=`awk < $tmpf "/$awkstr/ {print $ 3; found=1; exit}\
 275                                              END       {if (!found) print \"ERROR\"}"`
 276                     fi
 277                 fi
 278
 279                 cat $tmpf >> $workf
 280                 rm $tmpf
 281
 282                 str=`printf "%8s" "$result"`
 283                 print_summary -n "$str "
 284             done
 285             print_summary ""
 286         done
 287     done
 288 done
 289
 290 # rm $iorcf