2 # -*- mode: Bash; tab-width: 4; indent-tabs-mode: t; -*-
3 # vim:shiftwidth=4:softtabstop=4:tabstop=4:
5 ######################################################################
8 # Prerequisite: For "stripe_count > 0" you need to have ost setup and mounted.
11 # case 1 (stripe_count=0 default):
12 # $ thrhi=8 dir_count=4 sh mds-survey
13 # one can also run test with user defined targets as follows,
14 # $ thrhi=8 dir_count=4 file_count=50000 targets="lustre-MDT0000" sh mds-survey
15 # case 2 (stripe_count > 0, must have ost mounted):
16 # $ thrhi=8 dir_count=4 file_count=50000 stripe_count=2
17 # targets="lustre-MDT0000" sh mds-survey
18 # [ NOTE: It is advised to have automated login (passwordless entry) on server ]
21 source $(dirname $0)/libecho
23 # Customisation variables
24 #####################################################################
25 # One can change variable values in this section as per requirements
26 # The following variables can be set in the environment, or on the
28 # result file prefix (date/time + hostname makes unique)
29 # NB ensure path to it exists
30 rslt_loc=${rslt_loc:-"/tmp"}
31 rslt=${rslt:-"$rslt_loc/mds_survey_`date +%F@%R`_`uname -n`"}
33 # min and max thread count
37 # number of directories to test
38 dir_count=${dir_count:-$thrlo}
39 # number of files per thread
40 file_count=${file_count:-100000}
42 targets=${targets:-""}
43 stripe_count=${stripe_count:-0}
44 # what tests to run (first must be create, and last must be destroy)
45 # default=(create lookup md_getattr setxattr destroy)
46 tests_str=${tests_str:-"create lookup md_getattr setxattr destroy"}
48 # start number for each thread
49 start_number=${start_number:-2}
53 # Customisation variables ends here.
54 #####################################################################
55 # leave the rest of this alone unless you know what you're doing...
59 create_directories () {
66 for ((idx = 0; idx < $ndir; idx++)); do
67 if (( idx == 0 )); then
70 dirname=${basedir}${idx}
72 remote_shell $host $lctl --device $devno test_mkdir /$dirname > $rfile 2>&1
74 echo "$line" | grep -q 'error: test_mkdir'
77 echo "ERROR: fail test_mkdir" >&2
86 destroy_directories () {
93 for ((idx = 0; idx < $ndir; idx++)); do
94 if (( idx == 0 )); then
97 dirname=${basedir}${idx}
99 remote_shell $host $lctl --device $devno test_rmdir /$dirname > $rfile 2>&1
106 '/starting/ { n=0; next } \
107 /error/ {n = -1; exit} \
108 /^Total: total [0-9]+ threads [0-9]+ sec [0-9\.]+ [0-9]+\.[0-9]+\/second$/ \
109 { ave = strtonum($8); n++; next} \
110 /^[0-9]+\/[0-9]+ Total: [0-9]+\.[0-9]+\/second$/ \
111 { n++; v = strtonum($3); \
112 if (n == 1 || v < min) min = v; \
113 if (n == 1 || v > max) max = v; \
116 { if (n != 0) {n = -1; exit } } \
117 END { if (n == 1) { min = ave; max = ave; } \
118 printf "%d %f %f %f\n", n, ave, min, max}'
121 get_global_stats () {
126 if (n == 1) { err = $1; ave = $2; min = $3; max = $4} \
128 { if ($1 < err) err = $1; \
129 if ($2 < min) min = $2; \
130 if ($3 > max) max = $3; \
133 END { if (n == 0) err = 0; \
134 printf "%d %f %f %f\n", err, ave, min, max}'
138 if [ "$1" = "-n" ]; then
143 echo $minusn "$*" >> $rsltf
149 for name in $tests_str; do
154 # hide a little trick to unset this from the command line
155 if [ "$lustre_root" == " " ]; then
159 if [ -z "$lustre_root" ]; then
162 lctl=${lustre_root}/utils/lctl
165 declare -a client_names
166 declare -a host_names
167 if [ -z "$targets" ]; then
168 targets=$($lctl device_list | awk "{if (\$2 == \"UP\" && \
169 \$3 == \"mdt\") {print \$4} }")
170 if [ -z "$targets" ]; then
171 echo "Can't find any MDT to test. Please set targets=..."
176 # split out hostnames from mdt names
178 for trgt in $targets; do
179 str=(`split_hostname $trgt`)
180 host_names[$ndevs]=${str[0]}
181 client_names[$ndevs]=${str[1]}
186 if (( $stripe_count > 0 )); then
187 for ((i=0; i < $ndevs; i++)); do
188 host=${host_names[$i]}
189 obd=$(remote_shell $host $lctl device_list | awk "{if (\$2 == \"UP\" &&
190 \$3 == \"osc\") { print \$4 } }")
191 if [ -z "$obd" ]; then
192 echo "Need obdfilter to test stripe_count"
198 # check and insert obdecho module
199 if ! lsmod | grep obdecho > /dev/null; then
203 if [ $count -eq 0 -o "${tests[0]}" != "create" -o "${tests[(($count - 1))]}" != "destroy" ]; then
204 echo "tests: ${tests[@]}"
205 echo "First test must be 'create', and last test must be 'destroy'" 1>&2
209 rsltf="${rslt}.summary"
210 workf="${rslt}.detail"
211 cmdsf="${rslt}.script"
212 vmstatf="${rslt}.vmstat"
217 # disable portals debug and get obdecho loaded on all relevant hosts
218 unique_hosts=(`unique ${host_names[@]}`)
221 for host in ${unique_hosts[@]}; do
222 host_vmstatf=${vmstatf}_${host}
223 echo -n > $host_vmstatf
224 remote_shell $host "vmstat 5 >> $host_vmstatf" &> /dev/null &
226 vmstatpids[$pidcount]=$pid
227 pidcount=$((pidcount+1))
229 # get all the echo_client device numbers and names
230 for ((i=0; i < $ndevs; i++)); do
231 host=${host_names[$i]}
232 devno=(`get_ec_devno $host "${client_names[$i]}" "${client_names[$i]}" "mdt" $layer`)
233 if ((${#devno[@]} != 3)); then
236 devnos[$i]=${devno[0]}
237 client_names[$i]=${devno[1]}
238 do_teardown_ec[$i]=${devno[2]}
240 if (($ndevs <= 0 || ${#host_names[@]} <= 0)); then
241 echo "no devices or hosts specified"
244 print_summary "$(date) $0 from $(hostname)"
247 for ((idx = 0; idx < $ndevs; idx++)); do
248 host=${host_names[$idx]}
249 devno=${devnos[$idx]}
250 client_name="${host}:${client_names[$idx]}"
251 echo "=============> Create $dir_count directories on $client_name" >> $workf
252 destroy_directories $host $devno $dir_count $tmpf
253 ret=`create_directories $host $devno $dir_count $tmpf`
256 if [ $ret = "ERROR" ]; then
257 print_summary "created directories on $client_name failed"
264 for ((thr = $thrlo; thr <= $thrhi; thr*=2)); do
265 thr_per_dir=$((${thr}/${dir_count}))
266 # skip if no enough thread
267 if (( thr_per_dir <= 0 )); then
270 file_count_per_thread=$((${file_count}/${thr}))
271 str=`printf 'mdt %1d file %7d dir %4d thr %4d ' \
272 $ndevs $file_count $dir_count $thr`
273 echo "=======================> $str" >> $workf
274 print_summary -n "$str"
276 for test in ${tests[@]}; do
278 for host in ${unique_hosts[@]}; do
279 echo "starting run for config: $config test: $test file: \
280 $file_count threads: $thr directories: $dir_count" >> ${vmstatf}_${host}
282 print_summary -n "$test "
283 # create per-host script files
284 for host in ${unique_hosts[@]}; do
285 echo -n > ${cmdsf}_${host}
287 for ((idx = 0; idx < $ndevs; idx++)); do
288 host=${host_names[$idx]}
289 devno=${devnos[$idx]}
291 [ $test = "create" ] && test="create -c $stripe_count"
292 echo >> ${cmdsf}_${host} \
293 "$lctl > $tmpfi 2>&1 \
294 --threads $thr -$snap $devno test_$test -d /$basedir -D $dir_count \
295 -b $start_number -n $file_count_per_thread"
298 for host in ${unique_hosts[@]}; do
299 echo "wait" >> ${cmdsf}_${host}
300 pidarray[$pidcount]=0
301 pidcount=$((pidcount+1))
304 for host in ${unique_hosts[@]}; do
305 remote_shell $host bash < ${cmdsf}_${host} &
306 pidarray[$pidcount]=$!
307 pidcount=$((pidcount+1))
310 for host in ${unique_hosts[@]}; do
311 wait ${pidarray[$pidcount]}
312 pidcount=$((pidcount+1))
315 # clean up per-host script files
316 for host in ${unique_hosts[@]}; do
320 # collect/check individual MDT stats
322 for ((idx = 0; idx < $ndevs; idx++)); do
323 client_name="${host_names[$idx]}:${client_names[$idx]}"
325 echo "=============> $test $client_name" >> $workf
326 host="${host_names[$idx]}"
327 remote_shell $host cat $tmpfi > ${tmpfi}_local
328 cat ${tmpfi}_local >> $workf
329 get_stats ${tmpfi}_local >> $tmpf
330 rm -f $tmpfi ${tmpfi}_local
332 # compute/display global min/max stats
333 echo "=============> $test global" >> $workf
335 stats=(`get_global_stats $tmpf`)
337 if ((stats[0] <= 0)); then
338 str=`printf "%17s " ERROR`
341 str=`awk "BEGIN {printf \"%7.2f [%7.2f,%7.2f] \", \
342 ${stats[1]}, ${stats[2]}, ${stats[3]}; exit}"`
344 print_summary -n "$str"
348 # destroy directories
350 for ((idx = 0; idx < $ndevs; idx++)); do
351 host=${host_names[$idx]}
352 devno=${devnos[$idx]}
353 client_name="${host}:${client_names[$idx]}"
354 echo "=============> Destroy $dir_count directories on $client_name" >> $workf
355 destroy_directories $host $devno $dir_count $tmpf