3 ######################################################################
6 # specify obd names (host:name if remote)
7 # these can either be the echo_client names (client_names)
8 # or the ost names (ost_names)
9 #client_names=(ns8:ECHO_ns8 ns9:ECHO_ns9)
10 ost_names=(ns9:ost{1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16})
12 # result file prefix (date/time + hostname makes unique)
13 rslt=/home_nfs/eeb/obdfilter_survey_`date +%F@%R`_`uname -n`
15 # lustre root (if running with own source tree)
16 lustre_root=/home_nfs/eeb/lustre
18 # what to do (we always do an initial write)
19 #tests="rewrite read reread rewrite_again"
22 # total size (MBytes) per OST
23 # large enough to avoid cache effects
24 # and to make test startup/shutdown overhead insignificant
27 # record size (KBytes)
31 # number of objects per OST
35 # threads per OST (1024 max)
39 # restart from here iff all are defined
47 # max buffer_mem (total_threads * buffer size)
48 # (to avoid lctl ENOMEM problems)
49 max_buffer_mem=$((1024*1024))
51 # how to run commands on other nodes
52 custom_remote_shell () {
57 # Hop on to the remote node, chdir to 'here' and run the given
58 # commands. One of the following will probably work.
59 ssh $host "cd $here; $cmds"
60 #rsh $host "cd $here; $cmds"
61 #pdsh -w $host "cd $here; $cmds"
64 #####################################################################
65 # leave the rest of this alone unless you know what you're doing...
70 rsltf="${rslt}.summary"
71 workf="${rslt}.detail"
75 if [ -z "$lustre_root" ]; then
78 lctl=${lustre_root}/utils/lctl
85 if [ "$host" = "localhost" -o "$host" = `uname -n` ]; then
88 custom_remote_shell $host "$cmds"
94 remote_shell $host lsmod | grep obdecho > /dev/null 2>&1
99 if [ -z "$lustre_root" ]; then
100 remote_shell $host modprobe obdecho
101 elif [ -f ${lustre_root}/obdecho/obdecho.ko ]; then
102 remote_shell $host insmod ${lustre_root}/obdecho/obdecho.ko
104 remote_shell $host insmod ${lustre_root}/obdecho/obdecho.o
110 remote_shell $host rmmod obdecho
117 remote_shell $host $lctl device_list | \
118 awk "{if (\$2 == \"UP\" && \$3 == \"$type\" && \$4 == \"$name\") {\
124 local client_name="$2"
126 if [ -z "$client_name" ]; then
127 if [ -z "$ost_name" ]; then
128 echo "client and ost name both null" 1>&2
131 client_name=${ost_name}_echo_client
133 ec=`get_devno $host echo_client $client_name`
134 if [ -n "$ec" ]; then
135 echo $ec $client_name 0
138 if [ -z "$ost_name" ]; then
139 echo "no echo client and ost_name not set" 1>&2
142 ost=`get_devno $host obdfilter $ost_name`
143 if [ -z "$ost" ]; then
144 echo "OST $ost_name not setup" 1>&2
147 remote_shell $host "$lctl <<EOF
148 attach echo_client $client_name ${client_name}_UUID
151 ec=`get_devno $host echo_client $client_name`
152 if [ -z "$ec" ]; then
153 echo "Can't setup echo client" 1>&2
156 echo $ec $client_name 1
159 teardown_ec_devno () {
162 remote_shell $host "$lctl <<EOF
170 # create a set of objects, check there are 'n' contiguous ones and
171 # return the first or 'ERROR'
176 remote_shell $host $lctl --device $devno create $nobj > $rfile 2>&1
178 '/is object id/ {obj=strtonum($6);\
179 first=!not_first; not_first=1;\
180 if (first) first_obj=obj;
181 else if (obj != prev + 1) exit;\
183 END {printf "%d %d\n", first_obj, n}'`)
184 if ((n[1] != nobj)); then
197 remote_shell $host $lctl --device $devno destroy $obj0 $nobj > $rfile 2>&1
203 '/^Selected device [0-9]+$/ {n = 0; next}\
204 /error/ {n = -1; exit}\
205 /^[0-9]+\/[0-9]+ Total: [0-9]+\.[0-9]+\/second$/ {n++; v=strtonum($3); \
206 if (n == 1 || v < min) min = v;\
207 if (n == 1 || v > max) max = v;\
209 {if (n != 0) {n = -1; exit}}\
210 END {printf "%d %f %f\n", n, min, max}'
213 get_global_stats () {
215 awk < $rfile 'BEGIN {n = 0;}\
216 {n++; if (n == 1) {err = $1; min = $2; max = $3} else\
217 {if ($1 < err) err = $1;\
218 if ($2 < min) min = $2;\
219 if ($3 > max) max = $3}}\
220 END {if (n == 0) err = 0;\
221 printf "%d %f %f\n", err, min, max}'
225 # 'x' disables data check
232 *write*) echo "w$x";;
238 if [ "$1" = "-n" ]; then
243 echo $minusn "$*" >> $rsltf
248 echo "$@" | xargs -n1 echo | sort -u
254 *:*) host=`echo $name | sed 's/:.*$//'`
255 name=`echo $name | sed 's/[^:]*://'`
263 ndevs=${#client_names[@]}
264 if ((ndevs != 0)); then
265 if ((${#ost_names[@]} != 0)); then
266 echo "Please specify client_names or ost_names, but not both" 1>&2
269 for ((i=0; i<ndevs;i++)); do
270 str=(`split_hostname ${client_names[$i]}`)
271 host_names[$i]=${str[0]}
272 client_names[$i]=${str[1]}
275 ndevs=${#ost_names[@]}
276 if ((ndevs == 0)); then
277 echo "Please specify either client_names or ost_names" 1>&2
280 for ((i=0; i<ndevs;i++)); do
281 str=(`split_hostname ${ost_names[$i]}`)
282 host_names[$i]=${str[0]}
283 ost_names[$i]=${str[1]}
287 unique_hosts=(`unique ${host_names[@]}`)
289 for host in ${unique_hosts[@]}; do
290 remote_shell $host "echo 0 > /proc/sys/portals/debug"
291 do_unload_obdecho[$host]=0
292 if check_obdecho $host; then
296 if check_obdecho $host; then
297 do_unload_obdecho[$host]=0
300 echo "Can't load obdecho on $host" 1>&2
304 for ((i=0; i<ndevs; i++)); do
305 host=${host_names[$i]}
306 devno=(`get_ec_devno $host "${client_names[$i]}" "${ost_names[$i]}"`)
307 if ((${#devno[@]} != 3)); then
310 devnos[$i]=${devno[0]}
311 client_names[$i]=${devno[1]}
312 do_teardown_ec[$i]=${devno[2]}
315 for ((rsz=$rszlo;rsz<=$rszhi;rsz*=2)); do
316 for ((nobj=$nobjlo;nobj<=$nobjhi;nobj*=2)); do
317 for ((thr=$thrlo;thr<=$thrhi;thr*=2)); do
318 if ((thr < nobj)); then
322 if [ -n "$restart_rsz" -a\
323 -n "$restart_nobj" -a\
324 -n "$restart_thr" ]; then
325 if ((rsz < restart_rsz ||\
326 (rsz == restart_rsz &&\
327 (nobj < restart_nobj ||\
328 (nobj == restart_nobj &&\
329 thr < restart_thr))))); then
334 total_thr=$((ndevs*thr))
335 total_nobj=$((ndevs*nobj))
336 pages=$((rsz/PAGE_SIZE))
337 actual_rsz=$((pages*PAGE_SIZE))
338 count=$((size*1024/(actual_rsz*thr)))
339 actual_size=$((actual_rsz*count*thr))
340 total_size=$((actual_size*ndevs))
341 # show computed parameters
342 str=`printf 'ost %2d sz %8dK rsz %4d obj %4d thr %4d ' \
343 $ndevs $total_size $actual_rsz $total_nobj $total_thr`
344 echo "=======================> $str" >> $workf
345 print_summary -n "$str"
346 if ((total_thr * actual_rsz > max_buffer_mem)); then
347 print_summary "Too much buffer space"
352 for ((idx=0; idx < ndevs; idx++)); do
353 host=${host_names[$idx]}
354 devno=${devnos[$idx]}
355 client_name="${host}:${client_names[$idx]}"
356 echo "=============> Create $nobj on $client_name" >> $workf
357 first_obj=`create_objects $host $devno $nobj $tmpf`
360 if [ $first_obj = "ERROR" ]; then
361 print_summary "created object #s on $client_name not contiguous"
364 first_objs[$idx]=$first_obj
366 for test in write $tests; do
367 print_summary -n "$test "
368 for host in ${unique_hosts[@]}; do
369 echo -n > ${workf}_${host}_script
371 for ((idx=0; idx < ndevs; idx++)); do
372 host=${host_names[$idx]}
373 devno=${devnos[$idx]}
375 first_obj=${first_objs[$idx]}
376 echo >> ${workf}_${host}_script \
377 "$lctl > $tmpfi 2>&1 \\
378 --threads $thr -$snap $devno \\
379 test_brw $count `testname2type $test` q $pages ${thr}t${first_obj} &"
381 for host in ${unique_hosts[@]}; do
382 echo "wait" >> ${workf}_${host}_script
385 for host in ${unique_hosts[@]}; do
386 remote_shell $host bash ${workf}_${host}_script&
390 for host in ${unique_hosts[@]}; do
391 rm ${workf}_${host}_script
393 str=`awk "BEGIN {printf \"%7.2f \",\
394 $total_size / (( $t1 - $t0 ) * 1024)}"`
395 print_summary -n "$str"
397 for ((idx=0; idx < ndevs; idx++)); do
398 client_name="${host_names[$idx]}:${client_names[$idx]}"
400 echo "=============> $test $client_name" >> $workf
402 get_stats $tmpfi >> $tmpf
405 echo "=============> $test global" >> $workf
407 stats=(`get_global_stats $tmpf`)
409 if ((stats[0] <= 0)); then
410 if ((stats[0] < 0)); then
411 str=`printf "%15s " ERROR`
413 str=`printf "%15s " SHORT`
416 str=`awk "BEGIN {printf \"[%7.2f,%7.2f] \",\
417 (${stats[1]} * $actual_rsz)/1024,\
418 (${stats[2]} * $actual_rsz)/1024; exit}"`
420 print_summary -n "$str"
423 for ((idx=0; idx < ndevs; idx++)); do
424 host=${host_names[$idx]}
425 devno=${devnos[$idx]}
426 client_name="${host}:${client_names[$idx]}"
427 first_obj=${first_objs[$idx]}
428 echo "=============> Destroy $nobj on $client_name" >> $workf
429 destroy_objects $host $devno $first_obj $nobj $tmpf
437 for ((i=0; i<ndevs; i++)); do
438 host=${host_names[$i]}
439 if ((${do_teardown_ec[$i]})); then
440 teardown_ec_devno $host ${client_names[$i]}
444 for host in ${unique_hosts[@]}; do
445 if ((${do_unload_obdecho[$host]})); then