3 ######################################################################
6 # specify obd instances to exercise
7 # these can be either...
8 # obdfilter instances (set 'ost_names')
10 # echo_client instances (set 'client_names')
11 # ... use 'host:name' for obd instances on other nodes.
12 ost_names=(ost{1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16})
13 #client_names=(ns8:ECHO_ns8 ns9:ECHO_ns9)
15 # result file prefix (date/time + hostname makes unique)
16 # NB ensure path to it exists
17 rslt=/home_nfs/eeb/obdfilter_survey_`date +%F@%R`_`uname -n`
19 # lustre root (if running with own source tree)
20 lustre_root=/home_nfs/eeb/lustre
22 # what tests to run (first must be write)
23 #tests=(write rewrite read reread rewrite_again)
24 tests=(write rewrite read)
26 # total size (MBytes) per obd instance
27 # large enough to avoid cache effects
28 # and to make test startup/shutdown overhead insignificant
31 # record size (KBytes)
35 # number of objects per OST
39 # threads per OST (1024 max)
43 # restart from here iff all are defined
48 # machine's page size (K)
51 # max buffer_mem (total_threads * buffer size)
52 # (to avoid lctl ENOMEM problems)
53 max_buffer_mem=$((1024*1024))
55 # how to run commands on other nodes
56 # You need to make this work on your cluster if you have specified
57 # non-local obd instances above
58 custom_remote_shell () {
63 # Hop on to the remote node, chdir to 'here' and run the given
64 # commands. One of the following will probably work.
65 ssh $host "cd $here; $cmds"
66 #rsh $host "cd $here; $cmds"
67 # we have to remove the leading `uname -n`: from pdsh output lines
68 #pdsh -w $host "cd $here; $cmds" | sed 's/^[^:]*://'
71 #####################################################################
72 # leave the rest of this alone unless you know what you're doing...
76 modprobe="/sbin/modprobe"
80 # lctl::test_brw bandwidth snapshot interval (seconds)
82 # check file contents?
85 if [ ${#tests[@]} -eq 0 -o "${tests[0]}" != "write" ]; then
86 echo "First test must be 'write'" 1>&2
90 rsltf="${rslt}.summary"
91 workf="${rslt}.detail"
92 cmdsf="${rslt}.script"
96 if [ -z "$lustre_root" ]; then
99 lctl=${lustre_root}/utils/lctl
106 if [ "$host" = "localhost" -o "$host" = `uname -n` ]; then
109 custom_remote_shell $host "$cmds"
115 remote_shell $host $lsmod | grep obdecho > /dev/null 2>&1
120 if [ -z "$lustre_root" ]; then
121 remote_shell $host $modprobe obdecho
122 elif [ -f ${lustre_root}/obdecho/obdecho.ko ]; then
123 remote_shell $host $insmod ${lustre_root}/obdecho/obdecho.ko
125 remote_shell $host $insmod ${lustre_root}/obdecho/obdecho.o
131 remote_shell $host $rmmod obdecho
138 remote_shell $host $lctl device_list | \
139 awk "{if (\$2 == \"UP\" && \$3 == \"$type\" && \$4 == \"$name\") {\
145 local client_name="$2"
147 if [ -z "$client_name" ]; then
148 if [ -z "$ost_name" ]; then
149 echo "client and ost name both null" 1>&2
152 client_name=${ost_name}_echo_client
154 ec=`get_devno $host echo_client $client_name`
155 if [ -n "$ec" ]; then
156 echo $ec $client_name 0
159 if [ -z "$ost_name" ]; then
160 echo "no echo client and ost_name not set" 1>&2
163 ost=`get_devno $host obdfilter $ost_name`
164 if [ -z "$ost" ]; then
165 echo "OST $ost_name not setup" 1>&2
168 remote_shell $host "$lctl <<EOF
169 attach echo_client $client_name ${client_name}_UUID
172 ec=`get_devno $host echo_client $client_name`
173 if [ -z "$ec" ]; then
174 echo "Can't setup echo client" 1>&2
177 echo $ec $client_name 1
180 teardown_ec_devno () {
183 remote_shell $host "$lctl <<EOF
191 # create a set of objects, check there are 'n' contiguous ones and
192 # return the first or 'ERROR'
197 remote_shell $host $lctl --device $devno create $nobj > $rfile 2>&1
199 '/is object id/ {obj=strtonum($6);\
200 first=!not_first; not_first=1;\
201 if (first) first_obj=obj;
202 else if (obj != prev + 1) exit;\
204 END {printf "%d %d\n", first_obj, n}'`)
205 if ((n[1] != nobj)); then
218 remote_shell $host $lctl --device $devno destroy $obj0 $nobj > $rfile 2>&1
224 '/^Selected device [0-9]+$/ {n = 0; next}\
225 /error/ {n = -1; exit}\
226 /^[0-9]+\/[0-9]+ Total: [0-9]+\.[0-9]+\/second$/ {n++; v=strtonum($3); \
227 if (n == 1 || v < min) min = v;\
228 if (n == 1 || v > max) max = v;\
230 {if (n != 0) {n = -1; exit}}\
231 END {printf "%d %f %f\n", n, min, max}'
234 get_global_stats () {
236 awk < $rfile 'BEGIN {n = 0;}\
237 {n++; if (n == 1) {err = $1; min = $2; max = $3} else\
238 {if ($1 < err) err = $1;\
239 if ($2 < min) min = $2;\
240 if ($3 > max) max = $3}}\
241 END {if (n == 0) err = 0;\
242 printf "%d %f %f\n", err, min, max}'
246 # 'x' disables data check
253 *write*) echo "w$x";;
259 if [ "$1" = "-n" ]; then
264 echo $minusn "$*" >> $rsltf
269 echo "$@" | xargs -n1 echo | sort -u
275 *:*) host=`echo $name | sed 's/:.*$//'`
276 name=`echo $name | sed 's/[^:]*://'`
284 # split out hostnames from client/ost names
285 ndevs=${#client_names[@]}
286 if ((ndevs != 0)); then
287 if ((${#ost_names[@]} != 0)); then
288 echo "Please specify client_names or ost_names, but not both" 1>&2
291 for ((i=0; i<ndevs;i++)); do
292 str=(`split_hostname ${client_names[$i]}`)
293 host_names[$i]=${str[0]}
294 client_names[$i]=${str[1]}
297 ndevs=${#ost_names[@]}
298 if ((ndevs == 0)); then
299 echo "Please specify either client_names or ost_names" 1>&2
302 for ((i=0; i<ndevs;i++)); do
303 str=(`split_hostname ${ost_names[$i]}`)
304 host_names[$i]=${str[0]}
305 ost_names[$i]=${str[1]}
309 # disable portals debug and get obdecho loaded on all relevant hosts
310 unique_hosts=(`unique ${host_names[@]}`)
311 for host in ${unique_hosts[@]}; do
312 remote_shell $host "echo 0 > /proc/sys/portals/debug"
313 do_unload_obdecho[$host]=0
314 if obdecho_loaded $host; then
318 if obdecho_loaded $host; then
319 do_unload_obdecho[$host]=1
322 echo "Can't load obdecho on $host" 1>&2
326 # get all the echo_client device numbers and names
327 for ((i=0; i<ndevs; i++)); do
328 host=${host_names[$i]}
329 devno=(`get_ec_devno $host "${client_names[$i]}" "${ost_names[$i]}"`)
330 if ((${#devno[@]} != 3)); then
333 devnos[$i]=${devno[0]}
334 client_names[$i]=${devno[1]}
335 do_teardown_ec[$i]=${devno[2]}
338 for ((rsz=$rszlo;rsz<=$rszhi;rsz*=2)); do
339 for ((nobj=$nobjlo;nobj<=$nobjhi;nobj*=2)); do
340 for ((thr=$thrlo;thr<=$thrhi;thr*=2)); do
341 if ((thr < nobj)); then
345 if [ -n "$restart_rsz" -a\
346 -n "$restart_nobj" -a\
347 -n "$restart_thr" ]; then
348 if ((rsz < restart_rsz ||\
349 (rsz == restart_rsz &&\
350 (nobj < restart_nobj ||\
351 (nobj == restart_nobj &&\
352 thr < restart_thr))))); then
357 total_thr=$((ndevs*thr))
358 total_nobj=$((ndevs*nobj))
359 pages=$((rsz/PAGE_SIZE))
360 actual_rsz=$((pages*PAGE_SIZE))
361 count=$((size*1024/(actual_rsz*thr)))
362 actual_size=$((actual_rsz*count*thr))
363 total_size=$((actual_size*ndevs))
364 # show computed parameters
365 str=`printf 'ost %2d sz %8dK rsz %4d obj %4d thr %4d ' \
366 $ndevs $total_size $actual_rsz $total_nobj $total_thr`
367 echo "=======================> $str" >> $workf
368 print_summary -n "$str"
369 if ((total_thr * actual_rsz > max_buffer_mem)); then
370 print_summary "Too much buffer space"
375 for ((idx=0; idx < ndevs; idx++)); do
376 host=${host_names[$idx]}
377 devno=${devnos[$idx]}
378 client_name="${host}:${client_names[$idx]}"
379 echo "=============> Create $nobj on $client_name" >> $workf
380 first_obj=`create_objects $host $devno $nobj $tmpf`
383 if [ $first_obj = "ERROR" ]; then
384 print_summary "created object #s on $client_name not contiguous"
387 first_objs[$idx]=$first_obj
390 for test in ${tests[@]}; do
391 print_summary -n "$test "
392 # create per-host script files
393 for host in ${unique_hosts[@]}; do
394 echo -n > ${cmdsf}_${host}
396 for ((idx=0; idx < ndevs; idx++)); do
397 host=${host_names[$idx]}
398 devno=${devnos[$idx]}
400 first_obj=${first_objs[$idx]}
401 echo >> ${cmdsf}_${host} \
402 "$lctl > $tmpfi 2>&1 \\
403 --threads $thr -$snap $devno \\
404 test_brw $count `testname2type $test` q $pages ${thr}t${first_obj} &"
406 for host in ${unique_hosts[@]}; do
407 echo "wait" >> ${cmdsf}_${host}
409 # timed run of all the per-host script files
411 for host in ${unique_hosts[@]}; do
412 remote_shell $host bash ${cmdsf}_${host}&
416 # clean up per-host script files
417 for host in ${unique_hosts[@]}; do
420 # compute bandwidth from total data / elapsed time
421 str=`awk "BEGIN {printf \"%7.2f \",\
422 $total_size / (( $t1 - $t0 ) * 1024)}"`
423 print_summary -n "$str"
424 # collect/check individual OST stats
426 for ((idx=0; idx < ndevs; idx++)); do
427 client_name="${host_names[$idx]}:${client_names[$idx]}"
429 echo "=============> $test $client_name" >> $workf
431 get_stats $tmpfi >> $tmpf
434 # compute/display global min/max stats
435 echo "=============> $test global" >> $workf
437 stats=(`get_global_stats $tmpf`)
439 if ((stats[0] <= 0)); then
440 if ((stats[0] < 0)); then
441 str=`printf "%17s " ERROR`
443 str=`printf "%17s " SHORT`
446 str=`awk "BEGIN {printf \"[%7.2f,%7.2f] \",\
447 (${stats[1]} * $actual_rsz)/1024,\
448 (${stats[2]} * $actual_rsz)/1024; exit}"`
450 print_summary -n "$str"
453 # destroy objects we created
454 for ((idx=0; idx < ndevs; idx++)); do
455 host=${host_names[$idx]}
456 devno=${devnos[$idx]}
457 client_name="${host}:${client_names[$idx]}"
458 first_obj=${first_objs[$idx]}
459 echo "=============> Destroy $nobj on $client_name" >> $workf
460 destroy_objects $host $devno $first_obj $nobj $tmpf
468 # tear down any echo clients we created
469 for ((i=0; i<ndevs; i++)); do
470 host=${host_names[$i]}
471 if ((${do_teardown_ec[$i]})); then
472 teardown_ec_devno $host ${client_names[$i]}
476 # unload any obdecho modules we loaded
477 for host in ${unique_hosts[@]}; do
478 if ((${do_unload_obdecho[$host]})); then