#!/bin/bash
-# cluster name (expect all node names to be this followed by a number)
+# cluster name (all node names are this followed by the node number)
cluster=mdev
# client node numbers (individual numbers or inclusive ranges)
# numbers of tasks per client to survey
tasks_per_client_lo=1
-tasks_per_client_hi=16
-tasks_per_client_iterator="*=4"
+tasks_per_client_hi=8
+tasks_per_client_iterator="*=2"
# record sizes to survey
rsize_lo=1M
# *write*) write
# *) read
#tests=(write rewrite read reread rewrite_again)
-tests=(write rewrite remount read)
+tests=(write rewrite remount read reread)
# total # bytes written/read by any client node
-min_per_client_size=75M
-min_total_size=100M
+min_per_client_size=4G
+min_total_size=8G
# should each task do I/O to its own file?
file_per_task=1
-# the IOR binary
+# the binaries
IOR="/home/ericb/ior/src/C/IOR"
-
-# the pdsh binary
+llmount=/home/ericb/lustre/utils/llmount
pdsh=pdsh
-# the llmount binary
-llmount=/home/ericb/lustre/utils/llmount
+# the result file prefix (date/time + hostname makes unique)
+#rslt=/home/ericb/ior_survey_`date +%F@%R`_`uname -n`
+rslt=/home/ericb/ior_survey
# where lustre is mounted on the clients
lustre=/mnt/lustre
testfile=${lustre}/ior_survey_testfile
# how to unmount and remount the F/S on a client (to clear the cache)
+# change this depending on lustre config (network type, MDS etc)
remount="umount $lustre && $llmount -o nettype=elan mdev6:/ll_mds/client $lustre"
-# the result file prefix (date/time + hostname makes unique)
-#rslt=/home/ericb/ior_survey_`date +%F@%R`_`uname -n`
-rslt=/home/ericb/ior_survey
+# pdsh args required to instantiate all instances of IOR in parallel
+# the chosen module must support '-n <procs-per-node>'
+# -R<module>, -f<fanout> etc
+pdsh_mpiargs="-Rmqsh"
#don't spin for MPI completions
export LIBELAN_WAITTYPE=0
################################################################################
-# dont change stuff below here
+# dont change stuff below here unless you know what you're doing...
count_range() {
echo $1 | awk '{ nvals=split($1, vals, "-");\
}
idx2nodenum() {
- n=$1; shift
+ local n=$1; shift
while ((1)); do
- range=$1; shift
+ local range=$1; shift
if [ -z "$range" ]; then
return
fi
}
n2noderange() {
- n=$1; shift
+ local n=$1; shift
sep=""
nodes="["
while ((n > 0)); do
- range=$1; shift
+ local range=$1; shift
if [ -z "$range" ]; then
return
fi
- base=`base_range $range`
- chunk=`count_range $range`
+ local base=`base_range $range`
+ local chunk=`count_range $range`
if ((chunk > n)); then chunk=n; fi
- nodes="${nodes}${sep}${base}"; sep=","
+ local nodes="${nodes}${sep}${base}"; sep=","
if ((chunk > 1)); then nodes="${nodes}-$((base+chunk-1))"; fi
n=$((n-chunk))
done
}
countnodes() {
- radix=16384
- n=0
+ local radix=16384
+ local n=0
while ((radix > 0)); do
- nodes=`n2noderange $((n+radix)) $@`
+ local nodes=`n2noderange $((n+radix)) $@`
if [ -n "$nodes" ]; then
n=$((n+radix))
fi
}
parse_number() {
- str=$1
+ local str=$1
case $str in
*G|*g) n=`echo $str | sed 's/[gG]//'`; echo $((n*1024*1024*1024));;
*M|*m) n=`echo $str | sed 's/[Mm]//'`; echo $((n*1024*1024));;
}
pp_number() {
- n=$1
- G=$((1024*1024*1024))
- M=$((1024*1024))
- K=$((1024))
+ local n=$1
+ local G=$((1024*1024*1024))
+ local M=$((1024*1024))
+ local K=$((1024))
if ((n%G == 0 && n >= G)); then
echo "$((n/G))G"
elif ((n%M == 0 && n >= M)); then
fi
rsltf="${rslt}.summary"
-iorcf="${rslt}.script"
workf="${rslt}.detail"
-tmpf="${workf}_tmp"
-
echo -n > $rsltf
echo -n > $workf
echo $minusn "$*"
}
+# convert params to actual numbers
min_per_client_size=`parse_number $min_per_client_size`
min_total_size=`parse_number $min_total_size`
rsize_lo=`parse_number $rsize_lo`
rsize_hi=`parse_number $rsize_hi`
+# check on actual numbers of client nodes
nclients=`countnodes ${clients[@]}`
-
if ((clients_hi > nclients)); then clients_hi=$nclients; fi
for ((rsize=rsize_lo; rsize<=rsize_hi; rsize$rsize_iterator)); do
if ((per_client_size < min_per_client_size)); then
per_client_size=$min_per_client_size
fi
- total_size=`pp_number $((per_client_size * nclnt))`
for ((ntask=tasks_per_client_lo; ntask <= tasks_per_client_hi; ntask$tasks_per_client_iterator)); do
per_task_size=$((per_client_size/ntask))
+ if ((per_task_size%rsize != 0)); then
+ per_task_size=$(((per_task_size/rsize + 1)*rsize))
+ fi
+ total_size=`pp_number $((per_task_size*nclnt*ntask))`
- hdrstr=`printf "Total: %5sB rsize: %4s clients: %4d tasks: %3d: " $total_size $rsize $nclnt $ntask`
+ hdrstr=`printf "Total: %5sB rsize: %4sB clients: %4d tasks: %3d: " \
+ $total_size $pp_rsize $nclnt $ntask`
print_summary -n "$hdrstr"
for ((test_idx=0; test_idx < ${#tests[@]}; test_idx++)); do
print_summary -n "$test "
echo "===========> ${hdrstr} on $test_clients doing $test" >> $workf
+ tmpf=${workf}_tmp
echo -n > $tmpf
if [ "$test" = "remount" ]; then
result="OK"
fi
else
+ # check lustre is mounted everywhere it's needed
cmd="(mount -t lustre; mount -t lustre_lite) | grep $lustre"
- echo "=> $cmd" >> $tmpf
+ echo "=> Mount Check: $cmd" >> $tmpf
$pdsh -S -b -w "$test_clients" >> $tmpf 2>&1 \
"$cmd"
status=$?
echo "Completion Status: $status" >> $tmpf
-
if ((status)); then
cat $tmpf >> $workf
rm $tmpf
echo "=> ${cmdline[@]}" >> $tmpf
- $pdsh -S -b -Rmqsh -w "$test_clients" -n $ntask >> $tmpf 2>&1 \
+ $pdsh -S -b $pdsh_mpiargs -w "$test_clients" -n $ntask >> $tmpf 2>&1 \
"${cmdline[@]}"
status=$?
done
done
-# rm $iorcf
######################################################################
# customize per survey
-# specify obd names (host:name if remote)
-# these can either be the echo_client names (client_names)
-# or the ost names (ost_names)
+# specify obd instances to exercise
+# these can be either...
+# obdfilter instances (set 'ost_names')
+# ...or...
+# echo_client instances (set 'client_names')
+# ... use 'host:name' for obd instances on other nodes.
+ost_names=(ost{1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16})
#client_names=(ns8:ECHO_ns8 ns9:ECHO_ns9)
-ost_names=(ns9:ost{1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16})
# result file prefix (date/time + hostname makes unique)
+# NB ensure path to it exists
rslt=/home_nfs/eeb/obdfilter_survey_`date +%F@%R`_`uname -n`
# lustre root (if running with own source tree)
lustre_root=/home_nfs/eeb/lustre
-# what to do (we always do an initial write)
-#tests="rewrite read reread rewrite_again"
-tests="rewrite read"
+# what tests to run (first must be write)
+#tests=(write rewrite read reread rewrite_again)
+tests=(write rewrite read)
-# total size (MBytes) per OST
+# total size (MBytes) per obd instance
# large enough to avoid cache effects
# and to make test startup/shutdown overhead insignificant
-size=8192
+size=16384
# record size (KBytes)
rszlo=1024
max_buffer_mem=$((1024*1024))
# how to run commands on other nodes
+# You need to make this work on your cluster if you have specified
+# non-local obd instances above
custom_remote_shell () {
host=$1
shift
#####################################################################
# leave the rest of this alone unless you know what you're doing...
+# binaries
lsmod="/sbin/lsmod"
modprobe="/sbin/modprobe"
insmod="/sbin/insmod"
rmmod="/sbin/rmmod"
+# lctl::test_brw bandwidth snapshot interval (seconds)
snap=1
+# check file contents?
verify=1
+if [ ${#tests[@]} -eq 0 -o "${tests[0]}" != "write" ]; then
+ echo "First test must be 'write'" 1>&2
+ exit 1
+fi
+
rsltf="${rslt}.summary"
-cmdsf="${rslt}.script"
workf="${rslt}.detail"
+cmdsf="${rslt}.script"
echo -n > $rsltf
echo -n > $workf
fi
}
-check_obdecho() {
+obdecho_loaded() {
local host=$1
remote_shell $host $lsmod | grep obdecho > /dev/null 2>&1
}
}
split_hostname () {
- name=$1
+ local name=$1
case $name in
*:*) host=`echo $name | sed 's/:.*$//'`
name=`echo $name | sed 's/[^:]*://'`
echo "$host $name"
}
+# split out hostnames from client/ost names
ndevs=${#client_names[@]}
if ((ndevs != 0)); then
if ((${#ost_names[@]} != 0)); then
done
fi
+# disable portals debug and get obdecho loaded on all relevant hosts
unique_hosts=(`unique ${host_names[@]}`)
-
for host in ${unique_hosts[@]}; do
remote_shell $host "echo 0 > /proc/sys/portals/debug"
do_unload_obdecho[$host]=0
- if check_obdecho $host; then
+ if obdecho_loaded $host; then
continue
fi
load_obdecho $host
- if check_obdecho $host; then
+ if obdecho_loaded $host; then
do_unload_obdecho[$host]=1
continue
fi
exit 1
done
+# get all the echo_client device numbers and names
for ((i=0; i<ndevs; i++)); do
host=${host_names[$i]}
devno=(`get_ec_devno $host "${client_names[$i]}" "${ost_names[$i]}"`)
fi
first_objs[$idx]=$first_obj
done
+ # run tests
for test in write $tests; do
print_summary -n "$test "
+ # create per-host script files
for host in ${unique_hosts[@]}; do
echo -n > ${cmdsf}_${host}
done
for host in ${unique_hosts[@]}; do
echo "wait" >> ${cmdsf}_${host}
done
+ # timed run of all the per-host script files
t0=`date +%s.%N`
for host in ${unique_hosts[@]}; do
remote_shell $host bash ${cmdsf}_${host}&
done
wait
t1=`date +%s.%N`
+ # clean up per-host script files
for host in ${unique_hosts[@]}; do
rm ${cmdsf}_${host}
done
+ # compute bandwidth from total data / elapsed time
str=`awk "BEGIN {printf \"%7.2f \",\
$total_size / (( $t1 - $t0 ) * 1024)}"`
print_summary -n "$str"
+ # collect/check individual OST stats
echo -n > $tmpf
for ((idx=0; idx < ndevs; idx++)); do
client_name="${host_names[$idx]}:${client_names[$idx]}"
get_stats $tmpfi >> $tmpf
rm $tmpfi
done
+ # compute/display global min/max stats
echo "=============> $test global" >> $workf
cat $tmpf >> $workf
stats=(`get_global_stats $tmpf`)
print_summary -n "$str"
done
print_summary ""
+ # destroy objects we created
for ((idx=0; idx < ndevs; idx++)); do
host=${host_names[$idx]}
devno=${devnos[$idx]}
done
done
+# tear down any echo clients we created
for ((i=0; i<ndevs; i++)); do
host=${host_names[$i]}
if ((${do_teardown_ec[$i]})); then
fi
done
+# unload any obdecho modules we loaded
for host in ${unique_hosts[@]}; do
if ((${do_unload_obdecho[$host]})); then
unload_obdecho $host
# the SCSI devices to measure
scsidevs="/dev/sde /dev/sdh"
-# result file prefix
+# result file prefix. date/time+hostname makes unique
# NB ensure the path exists if it includes subdirs
-rslt=/tmp/sg_dd_rslt
+rslt=/tmp/sgpdd_survey_`date +%F@%R`_`uname -n`
# what to do (read or write)
actions="write read"
thrhi=4096
#####################################################################
+# leave the rest of this alone unless you know what you're doing...
-# disk block size (Bytes)
+# sgp_dd's idea of disk sector size (Bytes)
bs=512
-
-# max # threads per individual sgp_dd instance
+# and max # threads one instance will spawn
SG_MAX_QUEUE=16
+# map given device names into SG device names
i=0
devs=()
for d in $scsidevs; do
done
ndevs=${#devs[@]}
-start=`date +%F@%R`
-rsltf=${rslt}_${start}.summary
+rsltf=${rslt}.summary
+workf=${rslt}.detail
echo -n > $rsltf
-workf=${rslt}_${start}.detail
echo -n > $workf
-tmpf=${rslt}_${start}.tmp
print_summary () {
if [ "$1" = "-n" ]; then
print_summary "ENOMEM"
continue
fi
+ # run tests
for action in $actions; do
print_summary -n "$action "
echo "=====> $action" >> $workf
+ tmpf=${workf}_tmp
# start test
t0=`date +%s.%N`
for ((i=0;i<ndevs;i++)); do
done
wait
t1=`date +%s.%N`
- # collect all results in 1 file
+ # collect/check individual stats
echo > $tmpf
ok=0
for ((i=0;i<ndevs;i++)); do