lustre-iokit/ior-survey/ior-survey

   1 #!/bin/bash
   2
   3 # the value of these can be set in the environment.
   4 # This test assumes a typical pdsh naming scheme, where
   5 # node names can be expressed as a string
   6 # forllowed by a number
   7 # cluster name (all node names are this followed by the node number)
   8 cluster=${cluster:-""}
   9 if [ -z "$cluster" ];then  echo  "cluster not defined"; exit 1; fi
  10
  11 # client node numbers (individual numbers or inclusive ranges)
  12 clients=${clients:-""}
  13 if [ -z "$clients" ]; then echo "clients not defined"; exit 1; fi
  14
  15 # numbers of clients to survey
  16 clients_lo=${clients_lo:-1}
  17 clients_hi=${clients_hi:-3}
  18 clients_iterator=${clients_iterator:-"+=1"}
  19
  20 # numbers of tasks per client to survey
  21 tasks_per_client_lo=${task_per_client_lo:-1}
  22 tasks_per_client_hi=${tasks_per_client_hi:-8}
  23 tasks_per_client_iterator=${tasks_per_client_iterator:-"*=2"}
  24
  25 # record sizes to survey
  26 rsize_lo=${rsize_lo:-1M}
  27 rsize_hi=${rsize_hi:-1M}
  28 rsize_iterator=${rsize_iterator:-"*=2"}
  29
  30 ## which tests to run (first must be write)
  31 # clear_cache)   not really a test; just uncache everything
  32 # *write*)   write
  33 # *)         read
  34 tests=(write rewrite clear_cache read reread)
  35
  36 # total # bytes written/read by any client node
  37 min_per_client_size=${min_per_client_size:-"1G"}
  38 min_total_size=${min_total_size:-"2G"}
  39
  40 # should each task do I/O to its own file?
  41 file_per_task=${file_per_task:-1}
  42
  43 # the binaries
  44 IOR=${IOR:-"/usr/local/sbin/IOR"}
  45 llmount=${llmount:-"llmount"}
  46 # Select mpirun, yod, or pdsh
  47 fanout_cmd=${fanout_cmd:-"pdsh"}
  48 # mpirun still uses pdsh for cleanup
  49 pdsh=${pdsh:-"pdsh"}
  50 pdsh_args="-R ssh -S -b -w "
  51
  52 # the result file prefix (date/time + hostname makes unique)
  53 rslt_loc=${rslt_loc:-"/tmp"}
  54 rslt=${rslt:-"$rslt_loc/ior_survey_`date +%F@%R`_`uname -n`"}
  55
  56 # where lustre is mounted on the clients
  57 lustre=${lustre:-"/mnt/lustre"}
  58
  59 # basename of the test file(s)
  60 testfile=${testfile:-"${lustre}/ior/ior_survey_testfile"}
  61
  62 #don't spin for MPI completions
  63 export LIBELAN_WAITTYPE=0
  64
  65 ################################################################################
  66 # dont change stuff below here unless you know what you're doing...
  67
  68 # This is to allow use of yod, pdsh, etc.
  69 fanout() {
  70         local clients=$1; shift
  71         local tmpfile=$1; shift
  72         local result
  73         case $fanout_cmd in
  74         'pdsh')
  75                 $fanout_cmd $pdsh_args "$clients" "$@" >> $tmpfile 2>&1
  76                 echo $?
  77                 return
  78                 ;;
  79         'mpirun')
  80                 # horrible misuse of globals
  81                 $fanout_cmd -np $((ntask*nclnt)) "$@" >> $tmpfile 2>&1
  82                 echo $?
  83                 return
  84                 ;;
  85         'yod')
  86                 # and another
  87                 $fanout_cmd -np $((ntask*nclnt)) "$@" >> $tmpfile 2>&1
  88                 echo $?
  89                 return
  90                 ;;
  91
  92         *)
  93                 echo "255"
  94                 return
  95                 ;;
  96         esac
  97 }
  98
  99 dump_cache() {
 100         # we are assuming mpi uses will also have pdsh
 101         local clients=$1;shift
 102         local tmpfile=$1;shift
 103         clear_cache='for LRU in /proc/fs/lustre/ldlm/namespaces/*/lru_size;
 104                 do sudo /bin/bash -c "echo clear > $LRU"; done'
 105         echo "=> $clear_cache" >> $tmpfile
 106         $pdsh $pdsh_args "$test_clients" "$clear_cache" >> $tmpfile 2>&1
 107         status=$?
 108         echo "Completion Status: $status" >> $tmpfile
 109
 110         if ((status)); then
 111               echo "ERROR"
 112         else
 113               echo "OK"
 114         fi
 115 }
 116 count_range() {
 117     echo $1 | awk '{ nvals=split($1, vals, "-");\
 118                      if (nvals == 1) print 1;\
 119                      else if (nvals == 2) printf "%d\n", vals[2] - vals[1] + 1;}'
 120 }
 121
 122 base_range() {
 123     echo $1 | awk '{ split($1, vals, "-"); print vals[1]; }'
 124 }
 125
 126 idx2nodenum() {
 127     local n=$1; shift
 128     while ((1)); do
 129         local range=$1; shift
 130         if [ -z "$range" ]; then
 131             return
 132         fi
 133         chunk=`count_range $range`
 134         if ((chunk > $n)); then
 135             base=`base_range $range`
 136             echo $((base + n))
 137             return
 138         fi
 139         n=$((n-chunk))
 140     done
 141 }
 142
 143 n2noderange() {
 144     local n=$1; shift
 145     sep=""
 146     nodes="["
 147     while ((n > 0)); do
 148         local range=$1; shift
 149         if [ -z "$range" ]; then
 150             return
 151         fi
 152         local base=`base_range $range`
 153         local chunk=`count_range $range`
 154         if ((chunk > $n)); then chunk=$n; fi
 155         local nodes="${nodes}${sep}${base}"; sep=","
 156         if ((chunk > 1)); then nodes="${nodes}-$((base+chunk-1))"; fi
 157         n=$((n-chunk))
 158     done
 159     echo "${nodes}]"
 160 }
 161
 162 countnodes() {
 163     local radix=16384
 164     local n=0
 165     while ((radix > 0)); do
 166         local nodes=`n2noderange $((n+radix)) $@`
 167         if [ -n "$nodes" ]; then
 168             n=$((n+radix))
 169         fi
 170         radix=$((radix/2))
 171     done
 172     echo $n
 173 }
 174
 175 parse_number() {
 176     local str=$1
 177     case $str in
 178         *G|*g) n=`echo $str | sed 's/[gG]//'`; echo $((n*1024*1024*1024));;
 179         *M|*m) n=`echo $str | sed 's/[Mm]//'`; echo $((n*1024*1024));;
 180         *K|*k) n=`echo $str | sed 's/[Kk]//'`; echo $((n*1024));;
 181         *)     echo $1;;
 182     esac
 183 }
 184
 185 pp_number() {
 186     local n=$1
 187     local G=$((1024*1024*1024))
 188     local M=$((1024*1024))
 189     local K=$((1024))
 190     if ((n%G == 0 && n >= $G)); then
 191         echo "$((n/G))G"
 192     elif ((n%M == 0 && n >= $M)); then
 193         echo "$((n/M))M"
 194     elif ((n%K == 0 && n >= $K)); then
 195         echo "$((n/K))K"
 196     else
 197         echo $n
 198     fi
 199 }
 200
 201 if [ ${#tests[@]} -eq 0 -o "${tests[0]}" != "write" ]; then
 202     echo "First test must be 'write'" 1>&2
 203     exit 1
 204 fi
 205
 206 rsltf="${rslt}.summary"
 207 workf="${rslt}.detail"
 208 echo -n > $rsltf
 209 echo -n > $workf
 210
 211 print_summary () {
 212     if [ "$1" = "-n" ]; then
 213         minusn=$1; shift
 214     else
 215         minusn=""
 216     fi
 217     echo $minusn "$*" >> $rsltf
 218     echo $minusn "$*"
 219 }
 220
 221 check_mount() {
 222     local lustre=$1; shift
 223     local tmpb=$1; shift
 224     local clients=$1; shift
 225     local tmpfile=${tmpb}_tmp
 226     # check lustre is mounted everywhere it's needed
 227    cmd="grep $lustre /proc/mounts"
 228    $pdsh $pdsh_args "$clients" "$cmd" >> $tmpfile
 229    status=$?
 230         if (($status)); then
 231                 print_summary "Lustre NOT mounted on $lustre somewhere"
 232         exit 1
 233         fi
 234
 235 }
 236 # convert params to actual numbers
 237 min_per_client_size=`parse_number $min_per_client_size`
 238 min_total_size=`parse_number $min_total_size`
 239
 240 rsize_lo=`parse_number $rsize_lo`
 241 rsize_hi=`parse_number $rsize_hi`
 242
 243 # check on actual numbers of client nodes
 244 nclients=`countnodes ${clients[@]}`
 245 if ((clients_hi > $nclients)); then clients_hi=$nclients; fi
 246
 247 cur_date=`date`
 248 machine=`uname -n`
 249 script_name=`echo $0 | cut -d "/" -f2`
 250
 251 echo "$cur_date $script_name on $lustre from $machine" >> $workf
 252
 253 for ((rsize=$rsize_lo; rsize<=$rsize_hi; rsize$rsize_iterator)); do
 254     pp_rsize=`pp_number $rsize`
 255
 256     for ((nclnt=$clients_lo; nclnt<=$clients_hi; nclnt$clients_iterator)); do
 257         test_clients="${cluster}`n2noderange $nclnt ${clients[@]}`"
 258         echo $test_clients
 259         if [ "$fanout_cmd" = "pdsh" ] || [ "$fanout_cmd" = "mpirun" ];then
 260                 check_mount $lustre $workf $test_clients
 261         fi
 262         per_client_size=$((min_total_size/nclnt))
 263         if ((per_client_size < $min_per_client_size)); then
 264             per_client_size=$min_per_client_size
 265         fi
 266
 267         for ((ntask=$tasks_per_client_lo; ntask <= $tasks_per_client_hi; \
 268                 ntask$tasks_per_client_iterator)); do
 269             per_task_size=$((per_client_size/ntask))
 270             if ((per_task_size%rsize != 0)); then
 271                 per_task_size=$(((per_task_size/rsize + 1)*rsize))
 272             fi
 273             total_size=`pp_number $((per_task_size*nclnt*ntask))`
 274
 275             hdrstr=`printf "Total: %5sB rsize: %4sB clients: %4d tasks: %3d: " \
 276                 $total_size $pp_rsize $nclnt $ntask`
 277             print_summary -n "$hdrstr"
 278
 279             for ((test_idx=0; test_idx < ${#tests[@]}; test_idx++)); do
 280                 test=${tests[$test_idx]}
 281
 282                 print_summary -n "$test "
 283                 echo "===========> ${hdrstr} on $test_clients doing $test" >> $workf
 284                 tmpf=${workf}_tmp
 285                 echo -n > $tmpf
 286
 287                 if [ "$test" = "clear_cache" ]; then
 288                     if [ "$fanout_cmd" = "pdsh" ] || [ "$fanout_cmd" = "mpirun" ]; then
 289                                 result=`dump_cache $test_clients $tmpf`
 290                      else
 291                                 echo "Haven't figured out how to clear cache" >> $tmpf
 292                                 result="N/A"
 293                      fi
 294                 else
 295
 296                     cmdline=(
 297                     $IOR                     # the command
 298                     -o${testfile}            # test file prefix
 299                     -b${per_task_size}       # bytes per task
 300                     -t${rsize}               # record size
 301                     -e                       # fsync before close
 302                     -q                       # quit on error
 303                     )
 304
 305                     idx=${#cmdline[@]}
 306
 307                     # keep the test file(s) unless this is the last test
 308                     #((test_idx < ${#tests[@]}-1)) && cmdline[$((idx++))]="-k"
 309                     cmdline[$((idx++))]="-k"
 310
 311                     # use the existing test file(s) unless this is the first test
 312                     ((test_idx > 0)) && cmdline[$((idx++))]="-E"
 313
 314                     # file-per-task
 315                     (($file_per_task)) && cmdline[$((idx++))]="-F"
 316
 317                     case "$test" in
 318                     *write*) cmdline[$((idx++))]="-w"
 319                              awkstr="Max Write";;
 320                     *)       cmdline[$((idx++))]="-r"
 321                              awkstr="Max Read";;
 322                     esac
 323
 324                     echo "=> ${cmdline[@]}" >> $tmpf
 325
 326                     status=`fanout $test_clients $tmpf ${cmdline[@]}`
 327
 328                     echo "Completion Status: $status" >> $tmpf
 329
 330                     if (($status)); then
 331                         result="ERROR"
 332                     else
 333                         # pdsh adds an extra field
 334                         if [ "$fanout_cmd" = "pdsh" ]; then
 335                                 result=`awk < $tmpf "/$awkstr/ {print $ 4; found=1; exit}\
 336                                              END       {if (!found) print \"ERROR\"}"`
 337                         else
 338                                 result=`awk < $tmpf "/$awkstr/ {print $ 3; found=1; exit}\
 339                                              END       {if (!found) print \"ERROR\"}"`
 340                         fi
 341                     fi
 342                 fi
 343
 344                 cat $tmpf >> $workf
 345                 rm $tmpf
 346
 347                 str=`printf "%8s" "$result"`
 348                 print_summary -n "$str "
 349             done
 350             print_summary ""
 351         done
 352     done
 353 done
 354