X-Git-Url: https://git.whamcloud.com/?p=fs%2Flustre-release.git;a=blobdiff_plain;f=lustre-iokit%2Fobdfilter-survey%2Fobdfilter-survey;h=4f8991336528ce16853ae83d9048ef8133454748;hp=da0d7972b1feec18a01663c61300d349846fe923;hb=d0a54de6fc931066fd5cf18c78d2eba597e09942;hpb=8f9ffa16e1774f85a0cadce9c6a29cd0cd14f567 diff --git a/lustre-iokit/obdfilter-survey/obdfilter-survey b/lustre-iokit/obdfilter-survey/obdfilter-survey index da0d797..4f89913 100755 --- a/lustre-iokit/obdfilter-survey/obdfilter-survey +++ b/lustre-iokit/obdfilter-survey/obdfilter-survey @@ -3,52 +3,97 @@ ###################################################################### # customize per survey -# specify obd names (host:name if remote) -# these can either be the echo_client names (client_names) -# or the ost names (ost_names) +# specify obd instances to exercise +# these can be either... +# obdfilter instances (set 'ost_names') +# ...or... +# echo_client instances (set 'client_names') +# ... use 'host:name' for obd instances on other nodes. + +# allow these to be passed in via string... +ost_names_str=${ost_names_str:-""} +if [ -n "$ost_names_str" ]; then + declare -a ost_names + count=0 + for name in $ost_names_str; do + ost_names[$count]=$name + count=$((count+1)) + done +else + ost_names=(ost{1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16}) +fi + #client_names=(ns8:ECHO_ns8 ns9:ECHO_ns9) -ost_names=(ns9:ost{1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16}) +client_names_str=${client_names_str:-""} +if [ -n "$client_names_str" ]; then + # make sure we unset ost_names so that our client_names get noticed... + unset ost_names + declare -a client_names + count=0 + for name in $client_names_str; do + client_names[$count]=$name + count=$((count+1)) + done +fi # result file prefix (date/time + hostname makes unique) -rslt=/home_nfs/eeb/obdfilter_survey_`date +%F@%R`_`uname -n` +# NB ensure path to it exists +rslt=${rslt:-"/tmp/obdfilter_survey_`date +%F@%R`_`uname -n`"} # lustre root (if running with own source tree) -lustre_root=/home_nfs/eeb/lustre - -# what to do (we always do an initial write) -#tests="rewrite read reread rewrite_again" -tests="rewrite read" +lustre_root=${lustre_root:-"/home_nfs/eeb/lustre"} + +# what tests to run (first must be write) +tests_str=${tests_str:-""} +if [ -n "$tests_str" ]; then + declare -a tests + count=0 + for name in $tests_str; do + tests[$count]=$name + count=$((count+1)) + done +else + #tests=(write rewrite read reread rewrite_again) + tests=(write rewrite read) +fi -# total size (MBytes) per OST +# total size (MBytes) per obd instance # large enough to avoid cache effects # and to make test startup/shutdown overhead insignificant -size=8192 +size=${size:-16384} # record size (KBytes) -rszlo=1024 -rszhi=1024 +rszlo=${rszlo:-1024} +rszhi=${rszhi:-1024} # number of objects per OST -nobjlo=1 -nobjhi=512 +nobjlo=${nobjlo:-1} +nobjhi=${nobjhi:-512} # threads per OST (1024 max) -thrlo=1 -thrhi=64 +thrlo=${thrlo:-1} +thrhi=${thrhi:-64} # restart from here iff all are defined restart_rsz= restart_thr=1 restart_nobj=1 -# machine's page size -PAGE_SIZE=64 +# machine's page size (K) +if [ -z "$PAGE_SIZE" ]; then + if which python >/dev/null; then + PAGE_SIZE=`echo 'import resource; print resource.getpagesize()/1024;' |python` + fi +fi +PAGE_SIZE=${PAGE_SIZE:-4} # max buffer_mem (total_threads * buffer size) # (to avoid lctl ENOMEM problems) max_buffer_mem=$((1024*1024)) # how to run commands on other nodes +# You need to make this work on your cluster if you have specified +# non-local obd instances above custom_remote_shell () { host=$1 shift @@ -58,22 +103,46 @@ custom_remote_shell () { # commands. One of the following will probably work. ssh $host "cd $here; $cmds" #rsh $host "cd $here; $cmds" - #pdsh -w $host "cd $here; $cmds" + # we have to remove the leading `uname -n`: from pdsh output lines + #pdsh -w $host "cd $here; $cmds" | sed 's/^[^:]*://' } ##################################################################### # leave the rest of this alone unless you know what you're doing... +# binaries +lsmod="/sbin/lsmod" +modprobe="/sbin/modprobe" +insmod="/sbin/insmod" +rmmod="/sbin/rmmod" + +# lctl::test_brw bandwidth snapshot interval (seconds) snap=1 +# check file contents? verify=1 +if [ ${#tests[@]} -eq 0 -o "${tests[0]}" != "write" ]; then + echo "tests: ${tests[@]}" + echo "First test must be 'write'" 1>&2 + exit 1 +fi + rsltf="${rslt}.summary" workf="${rslt}.detail" +cmdsf="${rslt}.script" +vmstatf="${rslt}.vmstat" echo -n > $rsltf echo -n > $workf +declare -a vmstatpids + +# hide a little trick to unset this from the command line +if [ "$lustre_root" == " " ]; then + unset lustre_root +fi + if [ -z "$lustre_root" ]; then - lctl=lctl + lctl=$(which lctl) else lctl=${lustre_root}/utils/lctl fi @@ -89,25 +158,25 @@ remote_shell () { fi } -check_obdecho() { +obdecho_loaded() { local host=$1 - remote_shell $host lsmod | grep obdecho > /dev/null 2>&1 + remote_shell $host $lsmod | grep obdecho > /dev/null 2>&1 } load_obdecho () { local host=$1 if [ -z "$lustre_root" ]; then - remote_shell $host modprobe obdecho + remote_shell $host $modprobe obdecho elif [ -f ${lustre_root}/obdecho/obdecho.ko ]; then - remote_shell $host insmod ${lustre_root}/obdecho/obdecho.ko + remote_shell $host $insmod ${lustre_root}/obdecho/obdecho.ko else - remote_shell $host insmod ${lustre_root}/obdecho/obdecho.o + remote_shell $host $insmod ${lustre_root}/obdecho/obdecho.o fi } unload_obdecho () { local host=$1 - remote_shell $host rmmod obdecho + remote_shell $host $rmmod obdecho } get_devno () { @@ -174,17 +243,40 @@ create_objects () { local nobj=$3 local rfile=$4 remote_shell $host $lctl --device $devno create $nobj > $rfile 2>&1 - n=(`awk < $rfile \ - '/is object id/ {obj=strtonum($6);\ - first=!not_first; not_first=1;\ - if (first) first_obj=obj; - else if (obj != prev + 1) exit;\ - prev=obj; n++}\ - END {printf "%d %d\n", first_obj, n}'`) - if ((n[1] != nobj)); then - echo "ERROR" - else - echo ${n[0]} + first=0 + prev=0 + count=0 + error=0 + while read line; do + echo "$line" | grep -q 'is object id' + if [ $? -ne 0 ]; then + continue + fi + if [ $first -eq 0 ]; then + first=$(echo $line | awk '{print $6}') + first=$(printf "%d" $first) + prev=$first + count=1 + else + obj=$(echo $line | awk '{print $6}') + obj=$(printf "%d" $obj) + diff=$((obj - (prev+1))) + if [ $diff -ne 0 ]; then + error=1 + fi + prev=$obj + count=$((count+1)) + fi + done < $rfile + if [ $nobj -ne $count ]; then + echo "ERROR: $nobj != $count" >&2 + cat $rfile >&2 + echo "ERROR" + elif [ $error -ne 0 ]; then + echo "ERROR: non contiguous objs found" >&2 + echo "ERROR" + else + echo $first fi } @@ -249,7 +341,7 @@ unique () { } split_hostname () { - name=$1 + local name=$1 case $name in *:*) host=`echo $name | sed 's/:.*$//'` name=`echo $name | sed 's/[^:]*://'` @@ -260,6 +352,7 @@ split_hostname () { echo "$host $name" } +# split out hostnames from client/ost names ndevs=${#client_names[@]} if ((ndevs != 0)); then if ((${#ost_names[@]} != 0)); then @@ -284,23 +377,32 @@ else done fi +# get vmstat started +# disable portals debug and get obdecho loaded on all relevant hosts unique_hosts=(`unique ${host_names[@]}`) - +pidcount=0 for host in ${unique_hosts[@]}; do remote_shell $host "echo 0 > /proc/sys/portals/debug" + host_vmstatf=${vmstatf}_${host} + echo -n > $host_vmstatf + remote_shell $host "vmstat 5 >> $host_vmstatf" & + pid=$! + vmstatpids[$pidcount]=$pid + pidcount=$((pidcount+1)) do_unload_obdecho[$host]=0 - if check_obdecho $host; then - continue + if obdecho_loaded $host; then + continue fi load_obdecho $host - if check_obdecho $host; then - do_unload_obdecho[$host]=0 - continue - fi + if obdecho_loaded $host; then + do_unload_obdecho[$host]=1 + continue + fi echo "Can't load obdecho on $host" 1>&2 exit 1 done +# get all the echo_client device numbers and names for ((i=0; i> ${vmstatf}_${host} + done print_summary -n "$test " + # create per-host script files for host in ${unique_hosts[@]}; do - echo -n > ${workf}_${host}_script + echo -n > ${cmdsf}_${host} done for ((idx=0; idx < ndevs; idx++)); do host=${host_names[$idx]} devno=${devnos[$idx]} tmpfi="${tmpf}_$idx" first_obj=${first_objs[$idx]} - echo >> ${workf}_${host}_script \ + thr_per_obj=$((${thr}/${nobj})) + echo >> ${cmdsf}_${host} \ "$lctl > $tmpfi 2>&1 \\ --threads $thr -$snap $devno \\ - test_brw $count `testname2type $test` q $pages ${thr}t${first_obj} &" - done - for host in ${unique_hosts[@]}; do - echo "wait" >> ${workf}_${host}_script - done - t0=`date +%s.%N` - for host in ${unique_hosts[@]}; do - remote_shell $host bash ${workf}_${host}_script& - done - wait - t1=`date +%s.%N` - for host in ${unique_hosts[@]}; do - rm ${workf}_${host}_script - done + test_brw $count `testname2type $test` q $pages ${thr_per_obj}t${first_obj} &" + done + pidcount=0 + for host in ${unique_hosts[@]}; do + echo "wait" >> ${cmdsf}_${host} + pidarray[$pidcount]=0 + pidcount=$((pidcount+1)) + done + # timed run of all the per-host script files + t0=`date +%s.%N` + pidcount=0 + for host in ${unique_hosts[@]}; do + remote_shell $host bash ${cmdsf}_${host} & + pidarray[$pidcount]=$! + pidcount=$((pidcount+1)) + done + pidcount=0 + for host in ${unique_hosts[@]}; do + wait ${pidarray[$pidcount]} + pidcount=$((pidcount+1)) + done + #wait + t1=`date +%s.%N` + # clean up per-host script files + for host in ${unique_hosts[@]}; do + rm ${cmdsf}_${host} + done + # compute bandwidth from total data / elapsed time str=`awk "BEGIN {printf \"%7.2f \",\ $total_size / (( $t1 - $t0 ) * 1024)}"` print_summary -n "$str" + # collect/check individual OST stats echo -n > $tmpf for ((idx=0; idx < ndevs; idx++)); do client_name="${host_names[$idx]}:${client_names[$idx]}" @@ -402,15 +526,16 @@ for ((rsz=$rszlo;rsz<=$rszhi;rsz*=2)); do get_stats $tmpfi >> $tmpf rm $tmpfi done + # compute/display global min/max stats echo "=============> $test global" >> $workf cat $tmpf >> $workf stats=(`get_global_stats $tmpf`) rm $tmpf if ((stats[0] <= 0)); then if ((stats[0] < 0)); then - str=`printf "%15s " ERROR` + str=`printf "%17s " ERROR` else - str=`printf "%15s " SHORT` + str=`printf "%17s " SHORT` fi else str=`awk "BEGIN {printf \"[%7.2f,%7.2f] \",\ @@ -420,6 +545,7 @@ for ((rsz=$rszlo;rsz<=$rszhi;rsz*=2)); do print_summary -n "$str" done print_summary "" + # destroy objects we created for ((idx=0; idx < ndevs; idx++)); do host=${host_names[$idx]} devno=${devnos[$idx]} @@ -434,6 +560,7 @@ for ((rsz=$rszlo;rsz<=$rszhi;rsz*=2)); do done done +# tear down any echo clients we created for ((i=0; i/dev/null + wait $pid + pidcount=$((pidcount+1)) if ((${do_unload_obdecho[$host]})); then - unload_obdecho $host + unload_obdecho $host fi done + +exit 0