3 ######################################################################
6 # specify obd instances to exercise
7 # these can be either...
8 # obdfilter instances (set 'ost_names')
10 # echo_client instances (set 'client_names')
11 # ... use 'host:name' for obd instances on other nodes.
13 # allow these to be passed in via string...
14 ost_names_str=${ost_names_str:-""}
15 if [ -n "$ost_names_str" ]; then
18 for name in $ost_names_str; do
19 ost_names[$count]=$name
23 ost_names=(ost{1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16})
26 #client_names=(ns8:ECHO_ns8 ns9:ECHO_ns9)
27 client_names_str=${client_names_str:-""}
28 if [ -n "$client_names_str" ]; then
29 # make sure we unset ost_names so that our client_names get noticed...
31 declare -a client_names
33 for name in $client_names_str; do
34 client_names[$count]=$name
39 # result file prefix (date/time + hostname makes unique)
40 # NB ensure path to it exists
41 rslt=${rslt:-"/tmp/obdfilter_survey_`date +%F@%R`_`uname -n`"}
43 # lustre root (if running with own source tree)
44 lustre_root=${lustre_root:-"/home_nfs/eeb/lustre"}
46 # what tests to run (first must be write)
47 tests_str=${tests_str:-""}
48 if [ -n "$tests_str" ]; then
51 for name in $tests_str; do
56 #tests=(write rewrite read reread rewrite_again)
57 tests=(write rewrite read)
60 # total size (MBytes) per obd instance
61 # large enough to avoid cache effects
62 # and to make test startup/shutdown overhead insignificant
65 # record size (KBytes)
69 # number of objects per OST
73 # threads per OST (1024 max)
77 # restart from here iff all are defined
82 # machine's page size (K)
83 if [ -z "$PAGE_SIZE" ]; then
84 if which python >/dev/null; then
85 PAGE_SIZE=`echo 'import resource; print resource.getpagesize()/1024;' |python`
88 PAGE_SIZE=${PAGE_SIZE:-4}
90 # max buffer_mem (total_threads * buffer size)
91 # (to avoid lctl ENOMEM problems)
92 max_buffer_mem=$((1024*1024))
94 # how to run commands on other nodes
95 # You need to make this work on your cluster if you have specified
96 # non-local obd instances above
97 custom_remote_shell () {
102 # Hop on to the remote node, chdir to 'here' and run the given
103 # commands. One of the following will probably work.
104 ssh $host "cd $here; $cmds"
105 #rsh $host "cd $here; $cmds"
106 # we have to remove the leading `uname -n`: from pdsh output lines
107 #pdsh -w $host "cd $here; $cmds" | sed 's/^[^:]*://'
110 #####################################################################
111 # leave the rest of this alone unless you know what you're doing...
115 modprobe="/sbin/modprobe"
116 insmod="/sbin/insmod"
119 # lctl::test_brw bandwidth snapshot interval (seconds)
121 # check file contents?
124 if [ ${#tests[@]} -eq 0 -o "${tests[0]}" != "write" ]; then
125 echo "tests: ${tests[@]}"
126 echo "First test must be 'write'" 1>&2
130 rsltf="${rslt}.summary"
131 workf="${rslt}.detail"
132 cmdsf="${rslt}.script"
133 vmstatf="${rslt}.vmstat"
137 declare -a vmstatpids
139 # hide a little trick to unset this from the command line
140 if [ "$lustre_root" == " " ]; then
144 if [ -z "$lustre_root" ]; then
147 lctl=${lustre_root}/utils/lctl
154 if [ "$host" = "localhost" -o "$host" = `uname -n` ]; then
157 custom_remote_shell $host "$cmds"
163 remote_shell $host $lsmod | grep obdecho > /dev/null 2>&1
168 if [ -z "$lustre_root" ]; then
169 remote_shell $host $modprobe obdecho
170 elif [ -f ${lustre_root}/obdecho/obdecho.ko ]; then
171 remote_shell $host $insmod ${lustre_root}/obdecho/obdecho.ko
173 remote_shell $host $insmod ${lustre_root}/obdecho/obdecho.o
179 remote_shell $host $rmmod obdecho
186 remote_shell $host $lctl device_list | \
187 awk "{if (\$2 == \"UP\" && \$3 == \"$type\" && \$4 == \"$name\") {\
193 local client_name="$2"
195 if [ -z "$client_name" ]; then
196 if [ -z "$ost_name" ]; then
197 echo "client and ost name both null" 1>&2
200 client_name=${ost_name}_echo_client
202 ec=`get_devno $host echo_client $client_name`
203 if [ -n "$ec" ]; then
204 echo $ec $client_name 0
207 if [ -z "$ost_name" ]; then
208 echo "no echo client and ost_name not set" 1>&2
211 ost=`get_devno $host obdfilter $ost_name`
212 if [ -z "$ost" ]; then
213 echo "OST $ost_name not setup" 1>&2
216 remote_shell $host "$lctl <<EOF
217 attach echo_client $client_name ${client_name}_UUID
220 ec=`get_devno $host echo_client $client_name`
221 if [ -z "$ec" ]; then
222 echo "Can't setup echo client" 1>&2
225 echo $ec $client_name 1
228 teardown_ec_devno () {
231 remote_shell $host "$lctl <<EOF
239 # create a set of objects, check there are 'n' contiguous ones and
240 # return the first or 'ERROR'
245 remote_shell $host $lctl --device $devno create $nobj > $rfile 2>&1
251 echo "$line" | grep -q 'is object id'
252 if [ $? -ne 0 ]; then
255 if [ $first -eq 0 ]; then
256 first=$(echo $line | awk '{print $6}')
257 first=$(printf "%d" $first)
261 obj=$(echo $line | awk '{print $6}')
262 obj=$(printf "%d" $obj)
263 diff=$((obj - (prev+1)))
264 if [ $diff -ne 0 ]; then
271 if [ $nobj -ne $count ]; then
272 echo "ERROR: $nobj != $count" >&2
275 elif [ $error -ne 0 ]; then
276 echo "ERROR: non contiguous objs found" >&2
289 remote_shell $host $lctl --device $devno destroy $obj0 $nobj > $rfile 2>&1
295 '/^Selected device [0-9]+$/ {n = 0; next}\
296 /error/ {n = -1; exit}\
297 /^[0-9]+\/[0-9]+ Total: [0-9]+\.[0-9]+\/second$/ {n++; v=strtonum($3); \
298 if (n == 1 || v < min) min = v;\
299 if (n == 1 || v > max) max = v;\
301 {if (n != 0) {n = -1; exit}}\
302 END {printf "%d %f %f\n", n, min, max}'
305 get_global_stats () {
307 awk < $rfile 'BEGIN {n = 0;}\
308 {n++; if (n == 1) {err = $1; min = $2; max = $3} else\
309 {if ($1 < err) err = $1;\
310 if ($2 < min) min = $2;\
311 if ($3 > max) max = $3}}\
312 END {if (n == 0) err = 0;\
313 printf "%d %f %f\n", err, min, max}'
317 # 'x' disables data check
324 *write*) echo "w$x";;
330 if [ "$1" = "-n" ]; then
335 echo $minusn "$*" >> $rsltf
340 echo "$@" | xargs -n1 echo | sort -u
346 *:*) host=`echo $name | sed 's/:.*$//'`
347 name=`echo $name | sed 's/[^:]*://'`
355 # split out hostnames from client/ost names
356 ndevs=${#client_names[@]}
357 if ((ndevs != 0)); then
358 if ((${#ost_names[@]} != 0)); then
359 echo "Please specify client_names or ost_names, but not both" 1>&2
362 for ((i=0; i<ndevs;i++)); do
363 str=(`split_hostname ${client_names[$i]}`)
364 host_names[$i]=${str[0]}
365 client_names[$i]=${str[1]}
368 ndevs=${#ost_names[@]}
369 if ((ndevs == 0)); then
370 echo "Please specify either client_names or ost_names" 1>&2
373 for ((i=0; i<ndevs;i++)); do
374 str=(`split_hostname ${ost_names[$i]}`)
375 host_names[$i]=${str[0]}
376 ost_names[$i]=${str[1]}
381 # disable portals debug and get obdecho loaded on all relevant hosts
382 unique_hosts=(`unique ${host_names[@]}`)
384 for host in ${unique_hosts[@]}; do
385 remote_shell $host "echo 0 > /proc/sys/portals/debug"
386 host_vmstatf=${vmstatf}_${host}
387 echo -n > $host_vmstatf
388 remote_shell $host "vmstat 5 >> $host_vmstatf" &
390 vmstatpids[$pidcount]=$pid
391 pidcount=$((pidcount+1))
392 do_unload_obdecho[$host]=0
393 if obdecho_loaded $host; then
397 if obdecho_loaded $host; then
398 do_unload_obdecho[$host]=1
401 echo "Can't load obdecho on $host" 1>&2
405 # get all the echo_client device numbers and names
406 for ((i=0; i<ndevs; i++)); do
407 host=${host_names[$i]}
408 devno=(`get_ec_devno $host "${client_names[$i]}" "${ost_names[$i]}"`)
409 if ((${#devno[@]} != 3)); then
412 devnos[$i]=${devno[0]}
413 client_names[$i]=${devno[1]}
414 do_teardown_ec[$i]=${devno[2]}
417 for ((rsz=$rszlo;rsz<=$rszhi;rsz*=2)); do
418 for ((nobj=$nobjlo;nobj<=$nobjhi;nobj*=2)); do
419 for ((thr=$thrlo;thr<=$thrhi;thr*=2)); do
420 if ((thr < nobj)); then
424 if [ -n "$restart_rsz" -a\
425 -n "$restart_nobj" -a\
426 -n "$restart_thr" ]; then
427 if ((rsz < restart_rsz ||\
428 (rsz == restart_rsz &&\
429 (nobj < restart_nobj ||\
430 (nobj == restart_nobj &&\
431 thr < restart_thr))))); then
436 total_thr=$((ndevs*thr))
437 total_nobj=$((ndevs*nobj))
438 pages=$((rsz/PAGE_SIZE))
439 actual_rsz=$((pages*PAGE_SIZE))
440 count=$((size*1024/(actual_rsz*thr)))
441 actual_size=$((actual_rsz*count*thr))
442 total_size=$((actual_size*ndevs))
443 # show computed parameters
444 str=`printf 'ost %2d sz %8dK rsz %4d obj %4d thr %4d ' \
445 $ndevs $total_size $actual_rsz $total_nobj $total_thr`
446 echo "=======================> $str" >> $workf
447 print_summary -n "$str"
448 if ((total_thr * actual_rsz > max_buffer_mem)); then
449 print_summary "Too much buffer space"
454 for ((idx=0; idx < ndevs; idx++)); do
455 host=${host_names[$idx]}
456 devno=${devnos[$idx]}
457 client_name="${host}:${client_names[$idx]}"
458 echo "=============> Create $nobj on $client_name" >> $workf
459 first_obj=`create_objects $host $devno $nobj $tmpf`
462 if [ $first_obj = "ERROR" ]; then
463 print_summary "created object #s on $client_name not contiguous"
466 first_objs[$idx]=$first_obj
469 for test in ${tests[@]}; do
471 for host in ${unique_hosts[@]}; do
472 echo "starting run for test: $test rsz: $rsz threads: $thr objects: $nobj" >> ${vmstatf}_${host}
474 print_summary -n "$test "
475 # create per-host script files
476 for host in ${unique_hosts[@]}; do
477 echo -n > ${cmdsf}_${host}
479 for ((idx=0; idx < ndevs; idx++)); do
480 host=${host_names[$idx]}
481 devno=${devnos[$idx]}
483 first_obj=${first_objs[$idx]}
484 echo >> ${cmdsf}_${host} \
485 "$lctl > $tmpfi 2>&1 \\
486 --threads $thr -$snap $devno \\
487 test_brw $count `testname2type $test` q $pages ${thr}t${first_obj} &"
490 for host in ${unique_hosts[@]}; do
491 echo "wait" >> ${cmdsf}_${host}
492 pidarray[$pidcount]=0
493 pidcount=$((pidcount+1))
495 # timed run of all the per-host script files
498 for host in ${unique_hosts[@]}; do
499 remote_shell $host bash ${cmdsf}_${host} &
500 pidarray[$pidcount]=$!
501 pidcount=$((pidcount+1))
504 for host in ${unique_hosts[@]}; do
505 wait ${pidarray[$pidcount]}
506 pidcount=$((pidcount+1))
510 # clean up per-host script files
511 for host in ${unique_hosts[@]}; do
514 # compute bandwidth from total data / elapsed time
515 str=`awk "BEGIN {printf \"%7.2f \",\
516 $total_size / (( $t1 - $t0 ) * 1024)}"`
517 print_summary -n "$str"
518 # collect/check individual OST stats
520 for ((idx=0; idx < ndevs; idx++)); do
521 client_name="${host_names[$idx]}:${client_names[$idx]}"
523 echo "=============> $test $client_name" >> $workf
525 get_stats $tmpfi >> $tmpf
528 # compute/display global min/max stats
529 echo "=============> $test global" >> $workf
531 stats=(`get_global_stats $tmpf`)
533 if ((stats[0] <= 0)); then
534 if ((stats[0] < 0)); then
535 str=`printf "%17s " ERROR`
537 str=`printf "%17s " SHORT`
540 str=`awk "BEGIN {printf \"[%7.2f,%7.2f] \",\
541 (${stats[1]} * $actual_rsz)/1024,\
542 (${stats[2]} * $actual_rsz)/1024; exit}"`
544 print_summary -n "$str"
547 # destroy objects we created
548 for ((idx=0; idx < ndevs; idx++)); do
549 host=${host_names[$idx]}
550 devno=${devnos[$idx]}
551 client_name="${host}:${client_names[$idx]}"
552 first_obj=${first_objs[$idx]}
553 echo "=============> Destroy $nobj on $client_name" >> $workf
554 destroy_objects $host $devno $first_obj $nobj $tmpf
562 # tear down any echo clients we created
563 for ((i=0; i<ndevs; i++)); do
564 host=${host_names[$i]}
565 if ((${do_teardown_ec[$i]})); then
566 teardown_ec_devno $host ${client_names[$i]}
570 # unload any obdecho modules we loaded
572 for host in ${unique_hosts[@]}; do
573 remote_shell $host "killall vmstat" &
575 kill -term ${vmstatpids[$pidcount]}
576 kill -kill ${vmstatpids[$pidcount]} 2>/dev/null
578 pidcount=$((pidcount+1))
579 if ((${do_unload_obdecho[$host]})); then