From ebea3603c2ee92826e01874745fcbb331a4336b5 Mon Sep 17 00:00:00 2001 From: eeb Date: Thu, 7 Oct 2004 23:30:35 +0000 Subject: [PATCH] * ior-survey taking shape * minor fixes to obdfilter-survey --- lustre-iokit/ior-survey/ior-survey | 271 +++++++++++++++++-------- lustre-iokit/obdfilter-survey/README | 13 +- lustre-iokit/obdfilter-survey/obdfilter-survey | 37 ++-- 3 files changed, 215 insertions(+), 106 deletions(-) diff --git a/lustre-iokit/ior-survey/ior-survey b/lustre-iokit/ior-survey/ior-survey index 764744a..28e91ef 100644 --- a/lustre-iokit/ior-survey/ior-survey +++ b/lustre-iokit/ior-survey/ior-survey @@ -1,91 +1,67 @@ #!/bin/bash -cluster=adev -servers=(4 7) -server_disks=([4]=sdd [7]=sdd) -clients=(8-15 0-1) +# cluster name (expect all node names to be this followed by a number) +cluster=mdev -min_clients=1 -max_clients=10 +# client node numbers (individual numbers or inclusive ranges) +clients=(7-8) -per_client_size=4G -transfer_size=1M -tasks_per_client=1 +# numbers of clients to survey +clients_lo=1 +clients_hi=2 +clients_iterator="+=1" + +# numbers of tasks per client to survey +tasks_per_client_lo=1 +tasks_per_client_hi=16 +tasks_per_client_iterator="*=4" + +# record sizes to survey +rsize_lo=1M +rsize_hi=1M +rsize_iterator="*=2" + +## which tests to run (first must be write) +# remount) not really a test; just remount to uncache everything +# *write*) write +# *) read +#tests=(write rewrite read reread rewrite_again) +tests=(write rewrite remount read) + +# total # bytes written/read by any client node +min_per_client_size=75M +min_total_size=100M + +# should each task do I/O to its own file? file_per_task=1 +# the IOR binary IOR="/home/ericb/ior/src/C/IOR" -script="/home/ericb/eeb_ior_script" -testfile=/mnt/lustre/ior_survey_testfile +# the pdsh binary +pdsh=pdsh +# the llmount binary +llmount=/home/ericb/lustre/utils/llmount -################################################################################ -cat > $script < -# blockSize= -b 8G -# transferSize= -t 1M -# filePerProc= -F -# testFile= -o /mnt/lustre/ior_testfile -# uniqueDir= -u -# verbose= -v -# reorderTasks= -C - -# unused options -# collective=0 (MPI only) -# individualDataSets=0 [not working] -# noFill=0 (HDF5 only) -# preallocate=0 (MPI only) -# useSharedFilePointer=0 [not working] -# useFileView= -# useStridedDataType=0(MPI only) -# showHints=0 -# showHelp=0 - -# setup - api=POSIX - fsync=1 - repetitions=1 - useO_DIRECT=0 - interTestDelay=10 - intraTestBarriers=1 - storeFileOffset=0 - quitOnError=1 - segmentCount=1 - singleXferAttempt=0 - -# write - readFile=0 - writeFile=1 - useExistingTestFile=0 - keepFile=1 - RUN - -# rewrite - useExistingTestFile=1 - RUN - -# read - readFile=1 - writeFile=0 - useExistingTestFile=1 - RUN - -# reread (bug in ior means it needs each run needs at least 1 directive) - readFile=1 - RUN - -# write again - readFile=0 - writeFile=1 - useExistingTestFile=1 - keepFile=0 - RUN -IOR STOP - -EOF +# where lustre is mounted on the clients +lustre=/mnt/lustre + +# basename of the test file(s) +testfile=${lustre}/ior_survey_testfile + +# how to unmount and remount the F/S on a client (to clear the cache) +remount="umount $lustre && $llmount -o nettype=elan mdev6:/ll_mds/client $lustre" + +# the result file prefix (date/time + hostname makes unique) +#rslt=/home/ericb/ior_survey_`date +%F@%R`_`uname -n` +rslt=/home/ericb/ior_survey + +#don't spin for MPI completions +export LIBELAN_WAITTYPE=0 ################################################################################ +# dont change stuff below here count_range() { echo $1 | awk '{ nvals=split($1, vals, "-");\ @@ -172,22 +148,143 @@ pp_number() { fi } -nservers=`countnodes ${servers[@]}` +if [ ${#tests[@]} -eq 0 -o "${tests[0]}" != "write" ]; then + echo "First test must be 'write'" 1>&2 + exit 1 +fi + +rsltf="${rslt}.summary" +iorcf="${rslt}.script" +workf="${rslt}.detail" +tmpf="${workf}_tmp" + +echo -n > $rsltf +echo -n > $workf + +print_summary () { + if [ "$1" = "-n" ]; then + minusn=$1; shift + else + minusn="" + fi + echo $minusn "$*" >> $rsltf + echo $minusn "$*" +} + +min_per_client_size=`parse_number $min_per_client_size` +min_total_size=`parse_number $min_total_size` + +rsize_lo=`parse_number $rsize_lo` +rsize_hi=`parse_number $rsize_hi` + nclients=`countnodes ${clients[@]}` -if ((max_clients > nclients)); then max_clients=$nclients; fi -if ((file_per_task)); then minusFopt=-F; else minusFopt=""; fi +if ((clients_hi > nclients)); then clients_hi=$nclients; fi + +for ((rsize=rsize_lo; rsize<=rsize_hi; rsize$rsize_iterator)); do + pp_rsize=`pp_number $rsize` + + for ((nclnt=clients_lo; nclnt<=clients_hi; nclnt$clients_iterator)); do + test_clients="${cluster}`n2noderange $nclnt ${clients[@]}`" + + per_client_size=$((min_total_size/nclnt)) + if ((per_client_size < min_per_client_size)); then + per_client_size=$min_per_client_size + fi + total_size=`pp_number $((per_client_size * nclnt))` + + for ((ntask=tasks_per_client_lo; ntask <= tasks_per_client_hi; ntask$tasks_per_client_iterator)); do + per_task_size=$((per_client_size/ntask)) + + hdrstr=`printf "Total: %5sB rsize: %4s clients: %4d tasks: %3d: " $total_size $rsize $nclnt $ntask` + print_summary -n "$hdrstr" + + for ((test_idx=0; test_idx < ${#tests[@]}; test_idx++)); do + test=${tests[$test_idx]} + + print_summary -n "$test " + echo "===========> ${hdrstr} on $test_clients doing $test" >> $workf + echo -n > $tmpf + + if [ "$test" = "remount" ]; then + echo "=> $remount" >> $tmpf + $pdsh -S -b -w "$test_clients" >> $tmpf 2>&1 \ + "$remount" + status=$? + echo "Completion Status: $status" >> $tmpf + + if ((status)); then + result="ERROR" + else + result="OK" + fi + else + cmd="(mount -t lustre; mount -t lustre_lite) | grep $lustre" + echo "=> $cmd" >> $tmpf + $pdsh -S -b -w "$test_clients" >> $tmpf 2>&1 \ + "$cmd" + status=$? + echo "Completion Status: $status" >> $tmpf -for ((i=min_clients - 1;i> $workf + rm $tmpf + print_summary "Lustre NOT mounted on $lustre somewhere" + exit 1 + fi - clients="${cluster}`n2noderange $((i+1)) ${clients[@]}`" - per_task_size=$((`parse_number $per_client_size`/tasks_per_client)) - per_task_size=`pp_number $per_task_size` + cmdline=( + $IOR # the command + -o${testfile} # test file prefix + -b${per_task_size} # bytes per task + -t${rsize} # record size + -e # fsync before close + -q # quit on error + ) - pdsh -b -Rmqsh -w "$clients" -n $tasks_per_client \ - $IOR -b${per_task_size} -t${transfer_size} $minusFopt -o $testfile -f $script + idx=${#cmdline[@]} + # keep the test file(s) unless this is the last test + ((test_idx < ${#tests[@]}-1)) && cmdline[$((idx++))]="-k" + + # use the existing test file(s) unless this is the first test + ((test_idx > 0)) && cmdline[$((idx++))]="-E" + + # file-per-task + ((file_per_task)) && cmdline[$((idx++))]="-F" + + case "$test" in + *write*) cmdline[$((idx++))]="-w" + awkstr="Max Write";; + *) cmdline[$((idx++))]="-r" + awkstr="Max Read";; + esac + + echo "=> ${cmdline[@]}" >> $tmpf + + $pdsh -S -b -Rmqsh -w "$test_clients" -n $ntask >> $tmpf 2>&1 \ + "${cmdline[@]}" + status=$? + + echo "Completion Status: $status" >> $tmpf + + if ((status)); then + result="ERROR" + else + result=`awk < $tmpf "/$awkstr/ {print $ 3; found=1; exit}\ + END {if (!found) print \"ERROR\"}"` + fi + fi + + cat $tmpf >> $workf + rm $tmpf + + str=`printf "%8s" "$result"` + print_summary -n "$str " + done + print_summary "" + done + done done +# rm $iorcf diff --git a/lustre-iokit/obdfilter-survey/README b/lustre-iokit/obdfilter-survey/README index 1ff1eb2..cea897a 100644 --- a/lustre-iokit/obdfilter-survey/README +++ b/lustre-iokit/obdfilter-survey/README @@ -64,7 +64,11 @@ leave 'ost_names' undefined. You can optionally prefix any name in 'ost_names' or 'client_names' with the hostname that it is running on (e.g. remote_node:ost4) if your obdfilters or echo_clients are running on more than one node. In this -case, you need to ensure 'custom_remote_shell()' works on your cluster. +case, you need to ensure... + +(a) 'custom_remote_shell()' works on your cluster +(b) all pathnames you specify in the script are mounted on the node you + start the survey from and all the remote nodes. Use 'lctl device_list' to verify the obdfilter/echo_client instance names e.g... @@ -83,9 +87,10 @@ on node 'ns9' you could simply add 'ost3' to 'ost_names'. When the script runs, it creates a number of working files and a pair of result files. All files start with the prefix given by ${rslt}. -${rslt}_.summary same as stdout -${rslt}_.detail_tmp* tmp files -${rslt}_.detail collected tmp files for post-mortem +${rslt}.summary same as stdout +${rslt}.script_* per-host test script files +${rslt}.detail_tmp* per-ost result files +${rslt}.detail collected result files for post-mortem The script iterates over the given numbers of threads and objects performing all the specified tests and checking that all test processes diff --git a/lustre-iokit/obdfilter-survey/obdfilter-survey b/lustre-iokit/obdfilter-survey/obdfilter-survey index da0d797..06c26e8 100755 --- a/lustre-iokit/obdfilter-survey/obdfilter-survey +++ b/lustre-iokit/obdfilter-survey/obdfilter-survey @@ -41,7 +41,7 @@ restart_rsz= restart_thr=1 restart_nobj=1 -# machine's page size +# machine's page size (K) PAGE_SIZE=64 # max buffer_mem (total_threads * buffer size) @@ -58,16 +58,23 @@ custom_remote_shell () { # commands. One of the following will probably work. ssh $host "cd $here; $cmds" #rsh $host "cd $here; $cmds" - #pdsh -w $host "cd $here; $cmds" + # we have to remove the leading `uname -n`: from pdsh output lines + #pdsh -w $host "cd $here; $cmds" | sed 's/^[^:]*://' } ##################################################################### # leave the rest of this alone unless you know what you're doing... +lsmod="/sbin/lsmod" +modprobe="/sbin/modprobe" +insmod="/sbin/insmod" +rmmod="/sbin/rmmod" + snap=1 verify=1 rsltf="${rslt}.summary" +cmdsf="${rslt}.script" workf="${rslt}.detail" echo -n > $rsltf echo -n > $workf @@ -91,23 +98,23 @@ remote_shell () { check_obdecho() { local host=$1 - remote_shell $host lsmod | grep obdecho > /dev/null 2>&1 + remote_shell $host $lsmod | grep obdecho > /dev/null 2>&1 } load_obdecho () { local host=$1 if [ -z "$lustre_root" ]; then - remote_shell $host modprobe obdecho + remote_shell $host $modprobe obdecho elif [ -f ${lustre_root}/obdecho/obdecho.ko ]; then - remote_shell $host insmod ${lustre_root}/obdecho/obdecho.ko + remote_shell $host $insmod ${lustre_root}/obdecho/obdecho.ko else - remote_shell $host insmod ${lustre_root}/obdecho/obdecho.o + remote_shell $host $insmod ${lustre_root}/obdecho/obdecho.o fi } unload_obdecho () { local host=$1 - remote_shell $host rmmod obdecho + remote_shell $host $rmmod obdecho } get_devno () { @@ -294,7 +301,7 @@ for host in ${unique_hosts[@]}; do fi load_obdecho $host if check_obdecho $host; then - do_unload_obdecho[$host]=0 + do_unload_obdecho[$host]=1 continue fi echo "Can't load obdecho on $host" 1>&2 @@ -366,29 +373,29 @@ for ((rsz=$rszlo;rsz<=$rszhi;rsz*=2)); do for test in write $tests; do print_summary -n "$test " for host in ${unique_hosts[@]}; do - echo -n > ${workf}_${host}_script + echo -n > ${cmdsf}_${host} done for ((idx=0; idx < ndevs; idx++)); do host=${host_names[$idx]} devno=${devnos[$idx]} tmpfi="${tmpf}_$idx" first_obj=${first_objs[$idx]} - echo >> ${workf}_${host}_script \ + echo >> ${cmdsf}_${host} \ "$lctl > $tmpfi 2>&1 \\ --threads $thr -$snap $devno \\ test_brw $count `testname2type $test` q $pages ${thr}t${first_obj} &" done for host in ${unique_hosts[@]}; do - echo "wait" >> ${workf}_${host}_script + echo "wait" >> ${cmdsf}_${host} done t0=`date +%s.%N` for host in ${unique_hosts[@]}; do - remote_shell $host bash ${workf}_${host}_script& + remote_shell $host bash ${cmdsf}_${host}& done wait t1=`date +%s.%N` for host in ${unique_hosts[@]}; do - rm ${workf}_${host}_script + rm ${cmdsf}_${host} done str=`awk "BEGIN {printf \"%7.2f \",\ $total_size / (( $t1 - $t0 ) * 1024)}"` @@ -408,9 +415,9 @@ for ((rsz=$rszlo;rsz<=$rszhi;rsz*=2)); do rm $tmpf if ((stats[0] <= 0)); then if ((stats[0] < 0)); then - str=`printf "%15s " ERROR` + str=`printf "%17s " ERROR` else - str=`printf "%15s " SHORT` + str=`printf "%17s " SHORT` fi else str=`awk "BEGIN {printf \"[%7.2f,%7.2f] \",\ -- 1.8.3.1