#!/bin/bash
-cluster=adev
-servers=(4 7)
-server_disks=([4]=sdd [7]=sdd)
-clients=(8-15 0-1)
+# the value of these can be set in the environment.
+# This test assumes a typical pdsh naming scheme, where
+# node names can be expressed as a string
+# forllowed by a number
+# cluster name (all node names are this followed by the node number)
+cluster=${cluster:-""}
+if [ -z "$cluster" ];then echo "cluster not defined"; exit 1; fi
-min_clients=1
-max_clients=10
+# client node numbers (individual numbers or inclusive ranges)
+clients=${clients:-""}
+if [ -z "$clients" ]; then echo "clients not defined"; exit 1; fi
-per_client_size=4G
-transfer_size=1M
-tasks_per_client=1
-file_per_task=1
+# numbers of clients to survey
+clients_lo=${clients_lo:-1}
+clients_hi=${clients_hi:-3}
+clients_iterator=${clients_iterator:-"+=1"}
-IOR="/home/ericb/ior/src/C/IOR"
-script="/home/ericb/eeb_ior_script"
+# numbers of tasks per client to survey
+tasks_per_client_lo=${task_per_client_lo:-1}
+tasks_per_client_hi=${tasks_per_client_hi:-8}
+tasks_per_client_iterator=${tasks_per_client_iterator:-"*=2"}
-testfile=/mnt/lustre/ior_survey_testfile
+# record sizes to survey
+rsize_lo=${rsize_lo:-1M}
+rsize_hi=${rsize_hi:-1M}
+rsize_iterator=${rsize_iterator:-"*=2"}
+## which tests to run (first must be write)
+# clear_cache) not really a test; just uncache everything
+# *write*) write
+# *) read
+tests=(write rewrite clear_cache read reread)
-################################################################################
-cat > $script <<EOF
-IOR START
-# -f <this file>
-# blockSize=<set from cmdline> -b 8G
-# transferSize=<set from cmdline> -t 1M
-# filePerProc=<set from cmdline> -F
-# testFile=<set from cmdline> -o /mnt/lustre/ior_testfile
-# uniqueDir=<set from cmdline> -u
-# verbose=<set from cmdline> -v
-# reorderTasks=<set from cmdline> -C
-
-# unused options
-# collective=0 (MPI only)
-# individualDataSets=0 [not working]
-# noFill=0 (HDF5 only)
-# preallocate=0 (MPI only)
-# useSharedFilePointer=0 [not working]
-# useFileView=<MPI only>
-# useStridedDataType=0(MPI only)
-# showHints=0
-# showHelp=0
-
-# setup
- api=POSIX
- fsync=1
- repetitions=1
- useO_DIRECT=0
- interTestDelay=10
- intraTestBarriers=1
- storeFileOffset=0
- quitOnError=1
- segmentCount=1
- singleXferAttempt=0
-
-# write
- readFile=0
- writeFile=1
- useExistingTestFile=0
- keepFile=1
- RUN
-
-# rewrite
- useExistingTestFile=1
- RUN
-
-# read
- readFile=1
- writeFile=0
- useExistingTestFile=1
- RUN
-
-# reread (bug in ior means it needs each run needs at least 1 directive)
- readFile=1
- RUN
-
-# write again
- readFile=0
- writeFile=1
- useExistingTestFile=1
- keepFile=0
- RUN
-IOR STOP
-
-EOF
+# total # bytes written/read by any client node
+min_per_client_size=${min_per_client_size:-"1G"}
+min_total_size=${min_total_size:-"2G"}
+
+# should each task do I/O to its own file?
+file_per_task=${file_per_task:-1}
+
+# the binaries
+IOR=${IOR:-"/usr/local/sbin/IOR"}
+llmount=${llmount:-"llmount"}
+# Select mpirun, yod, or pdsh
+fanout_cmd=${fanout_cmd:-"pdsh"}
+# mpirun still uses pdsh for cleanup
+pdsh=${pdsh:-"pdsh"}
+pdsh_args="-R ssh -S -b -w "
+
+# the result file prefix (date/time + hostname makes unique)
+rslt_loc=${rslt_loc:-"/tmp"}
+rslt=${rslt:-"$rslt_loc/ior_survey_`date +%F@%R`_`uname -n`"}
+
+# where lustre is mounted on the clients
+lustre=${lustre:-"/mnt/lustre"}
+
+# basename of the test file(s)
+testfile=${testfile:-"${lustre}/ior/ior_survey_testfile"}
+
+#don't spin for MPI completions
+export LIBELAN_WAITTYPE=0
################################################################################
+# dont change stuff below here unless you know what you're doing...
+
+# This is to allow use of yod, pdsh, etc.
+fanout() {
+ local clients=$1; shift
+ local tmpfile=$1; shift
+ local result
+ case $fanout_cmd in
+ 'pdsh')
+ $fanout_cmd $pdsh_args "$clients" "$@" >> $tmpfile 2>&1
+ echo $?
+ return
+ ;;
+ 'mpirun')
+ # horrible misuse of globals
+ $fanout_cmd -np $((ntask*nclnt)) "$@" >> $tmpfile 2>&1
+ echo $?
+ return
+ ;;
+ 'yod')
+ # and another
+ $fanout_cmd -np $((ntask*nclnt)) "$@" >> $tmpfile 2>&1
+ echo $?
+ return
+ ;;
+
+ *)
+ echo "255"
+ return
+ ;;
+ esac
+}
+
+dump_cache() {
+ # we are assuming mpi uses will also have pdsh
+ local clients=$1;shift
+ local tmpfile=$1;shift
+ clear_cache='for LRU in /proc/fs/lustre/ldlm/namespaces/*/lru_size;
+ do sudo /bin/bash -c "echo clear > $LRU"; done'
+ echo "=> $clear_cache" >> $tmpfile
+ $pdsh $pdsh_args "$test_clients" "$clear_cache" >> $tmpfile 2>&1
+ status=$?
+ echo "Completion Status: $status" >> $tmpfile
+ if ((status)); then
+ echo "ERROR"
+ else
+ echo "OK"
+ fi
+}
count_range() {
echo $1 | awk '{ nvals=split($1, vals, "-");\
if (nvals == 1) print 1;\
}
idx2nodenum() {
- n=$1; shift
+ local n=$1; shift
while ((1)); do
- range=$1; shift
+ local range=$1; shift
if [ -z "$range" ]; then
return
fi
chunk=`count_range $range`
- if ((chunk > n)); then
+ if ((chunk > $n)); then
base=`base_range $range`
echo $((base + n))
return
}
n2noderange() {
- n=$1; shift
+ local n=$1; shift
sep=""
nodes="["
while ((n > 0)); do
- range=$1; shift
+ local range=$1; shift
if [ -z "$range" ]; then
return
fi
- base=`base_range $range`
- chunk=`count_range $range`
- if ((chunk > n)); then chunk=n; fi
- nodes="${nodes}${sep}${base}"; sep=","
+ local base=`base_range $range`
+ local chunk=`count_range $range`
+ if ((chunk > $n)); then chunk=$n; fi
+ local nodes="${nodes}${sep}${base}"; sep=","
if ((chunk > 1)); then nodes="${nodes}-$((base+chunk-1))"; fi
n=$((n-chunk))
done
}
countnodes() {
- radix=16384
- n=0
+ local radix=16384
+ local n=0
while ((radix > 0)); do
- nodes=`n2noderange $((n+radix)) $@`
+ local nodes=`n2noderange $((n+radix)) $@`
if [ -n "$nodes" ]; then
n=$((n+radix))
fi
}
parse_number() {
- str=$1
+ local str=$1
case $str in
*G|*g) n=`echo $str | sed 's/[gG]//'`; echo $((n*1024*1024*1024));;
*M|*m) n=`echo $str | sed 's/[Mm]//'`; echo $((n*1024*1024));;
}
pp_number() {
- n=$1
- G=$((1024*1024*1024))
- M=$((1024*1024))
- K=$((1024))
- if ((n%G == 0 && n >= G)); then
+ local n=$1
+ local G=$((1024*1024*1024))
+ local M=$((1024*1024))
+ local K=$((1024))
+ if ((n%G == 0 && n >= $G)); then
echo "$((n/G))G"
- elif ((n%M == 0 && n >= M)); then
+ elif ((n%M == 0 && n >= $M)); then
echo "$((n/M))M"
- elif ((n%K == 0 && n >= K)); then
+ elif ((n%K == 0 && n >= $K)); then
echo "$((n/K))K"
else
echo $n
fi
}
-nservers=`countnodes ${servers[@]}`
+if [ ${#tests[@]} -eq 0 -o "${tests[0]}" != "write" ]; then
+ echo "First test must be 'write'" 1>&2
+ exit 1
+fi
+
+rsltf="${rslt}.summary"
+workf="${rslt}.detail"
+echo -n > $rsltf
+echo -n > $workf
+
+print_summary () {
+ if [ "$1" = "-n" ]; then
+ minusn=$1; shift
+ else
+ minusn=""
+ fi
+ echo $minusn "$*" >> $rsltf
+ echo $minusn "$*"
+}
+
+check_mount() {
+ local lustre=$1; shift
+ local tmpb=$1; shift
+ local clients=$1; shift
+ local tmpfile=${tmpb}_tmp
+ # check lustre is mounted everywhere it's needed
+ cmd="grep $lustre /proc/mounts"
+ $pdsh $pdsh_args "$clients" "$cmd" >> $tmpfile
+ status=$?
+ if (($status)); then
+ print_summary "Lustre NOT mounted on $lustre somewhere"
+ exit 1
+ fi
+
+}
+# convert params to actual numbers
+min_per_client_size=`parse_number $min_per_client_size`
+min_total_size=`parse_number $min_total_size`
+
+rsize_lo=`parse_number $rsize_lo`
+rsize_hi=`parse_number $rsize_hi`
+
+# check on actual numbers of client nodes
nclients=`countnodes ${clients[@]}`
+if ((clients_hi > $nclients)); then clients_hi=$nclients; fi
+
+cur_date=`date`
+machine=`uname -n`
+script_name=`echo $0 | cut -d "/" -f2`
+
+echo "$cur_date $script_name on $lustre from $machine" >> $workf
+
+for ((rsize=$rsize_lo; rsize<=$rsize_hi; rsize$rsize_iterator)); do
+ pp_rsize=`pp_number $rsize`
+
+ for ((nclnt=$clients_lo; nclnt<=$clients_hi; nclnt$clients_iterator)); do
+ test_clients="${cluster}`n2noderange $nclnt ${clients[@]}`"
+ echo $test_clients
+ if [ "$fanout_cmd" = "pdsh" ] || [ "$fanout_cmd" = "mpirun" ];then
+ check_mount $lustre $workf $test_clients
+ fi
+ per_client_size=$((min_total_size/nclnt))
+ if ((per_client_size < $min_per_client_size)); then
+ per_client_size=$min_per_client_size
+ fi
-if ((max_clients > nclients)); then max_clients=$nclients; fi
-if ((file_per_task)); then minusFopt=-F; else minusFopt=""; fi
+ for ((ntask=$tasks_per_client_lo; ntask <= $tasks_per_client_hi; \
+ ntask$tasks_per_client_iterator)); do
+ per_task_size=$((per_client_size/ntask))
+ if ((per_task_size%rsize != 0)); then
+ per_task_size=$(((per_task_size/rsize + 1)*rsize))
+ fi
+ total_size=`pp_number $((per_task_size*nclnt*ntask))`
+
+ hdrstr=`printf "Total: %5sB rsize: %4sB clients: %4d tasks: %3d: " \
+ $total_size $pp_rsize $nclnt $ntask`
+ print_summary -n "$hdrstr"
-for ((i=min_clients - 1;i<max_clients;i++)); do
- tc=`printf "%3d tasks, %4d clients" $tasks_per_client $((i+1))`
- echo "=================================== $tc ================================================"
+ for ((test_idx=0; test_idx < ${#tests[@]}; test_idx++)); do
+ test=${tests[$test_idx]}
+
+ print_summary -n "$test "
+ echo "===========> ${hdrstr} on $test_clients doing $test" >> $workf
+ tmpf=${workf}_tmp
+ echo -n > $tmpf
- clients="${cluster}`n2noderange $((i+1)) ${clients[@]}`"
- per_task_size=$((`parse_number $per_client_size`/tasks_per_client))
- per_task_size=`pp_number $per_task_size`
+ if [ "$test" = "clear_cache" ]; then
+ if [ "$fanout_cmd" = "pdsh" ] || [ "$fanout_cmd" = "mpirun" ]; then
+ result=`dump_cache $test_clients $tmpf`
+ else
+ echo "Haven't figured out how to clear cache" >> $tmpf
+ result="N/A"
+ fi
+ else
- pdsh -b -Rmqsh -w "$clients" -n $tasks_per_client \
- $IOR -b${per_task_size} -t${transfer_size} $minusFopt -o $testfile -f $script
+ cmdline=(
+ $IOR # the command
+ -o${testfile} # test file prefix
+ -b${per_task_size} # bytes per task
+ -t${rsize} # record size
+ -e # fsync before close
+ -q # quit on error
+ )
+ idx=${#cmdline[@]}
+
+ # keep the test file(s) unless this is the last test
+ #((test_idx < ${#tests[@]}-1)) && cmdline[$((idx++))]="-k"
+ cmdline[$((idx++))]="-k"
+
+ # use the existing test file(s) unless this is the first test
+ ((test_idx > 0)) && cmdline[$((idx++))]="-E"
+
+ # file-per-task
+ (($file_per_task)) && cmdline[$((idx++))]="-F"
+
+ case "$test" in
+ *write*) cmdline[$((idx++))]="-w"
+ awkstr="Max Write";;
+ *) cmdline[$((idx++))]="-r"
+ awkstr="Max Read";;
+ esac
+
+ echo "=> ${cmdline[@]}" >> $tmpf
+
+ status=`fanout $test_clients $tmpf ${cmdline[@]}`
+
+ echo "Completion Status: $status" >> $tmpf
+
+ if (($status)); then
+ result="ERROR"
+ else
+ # pdsh adds an extra field
+ if [ "$fanout_cmd" = "pdsh" ]; then
+ result=`awk < $tmpf "/$awkstr/ {print $ 4; found=1; exit}\
+ END {if (!found) print \"ERROR\"}"`
+ else
+ result=`awk < $tmpf "/$awkstr/ {print $ 3; found=1; exit}\
+ END {if (!found) print \"ERROR\"}"`
+ fi
+ fi
+ fi
+
+ cat $tmpf >> $workf
+ rm $tmpf
+
+ str=`printf "%8s" "$result"`
+ print_summary -n "$str "
+ done
+ print_summary ""
+ done
+ done
done
git://git.whamcloud.com / fs / lustre-release.git / blobdiff