2 # vim:expandtab:shiftwidth=4:softtabstop=4:tabstop=4:
4 ######################################################################
7 # Prerequisite: For "stripe_count > 0" you need to have ost setup and mounted.
10 # case 1 (stripe_count=0 default):
11 # $ thrhi=8 dir_count=4 sh mds-survey
12 # one can also run test with user defined targets as follows,
13 # $ thrhi=8 dir_count=4 file_count=50000 targets="lustre-MDT0000" sh mds-survey
14 # case 2 (stripe_count > 0, must have ost mounted):
15 # $ thrhi=8 dir_count=4 file_count=50000 stripe_count=2
16 # targets="lustre-MDT0000" sh mds-survey
17 # [ NOTE: It is advised to have automated login (passwordless entry) on server ]
20 source $(dirname $0)/libecho
22 # Customisation variables
23 #####################################################################
24 # One can change variable values in this section as per requirements
25 # The following variables can be set in the environment, or on the
27 # result file prefix (date/time + hostname makes unique)
28 # NB ensure path to it exists
29 rslt_loc=${rslt_loc:-"/tmp"}
30 rslt=${rslt:-"$rslt_loc/mds_survey_`date +%F@%R`_`uname -n`"}
32 # min and max thread count
36 # number of directories to test
37 dir_count=${dir_count:-$thrlo}
38 # number of files per thread
39 file_count=${file_count:-100000}
41 targets=${targets:-""}
42 stripe_count=${stripe_count:-0}
43 # what tests to run (first must be create, and last must be destroy)
44 # default=(create lookup md_getattr setxattr destroy)
45 tests_str=${tests_str:-"create lookup md_getattr setxattr destroy"}
47 # start number for each thread
48 start_number=${start_number:-2}
52 # Customisation variables ends here.
53 #####################################################################
54 # leave the rest of this alone unless you know what you're doing...
58 create_directories () {
65 for ((idx = 0; idx < $ndir; idx++)); do
66 if (( idx == 0 )); then
69 dirname=${basedir}${idx}
71 remote_shell $host $lctl --device $devno test_mkdir /$dirname > $rfile 2>&1
73 echo "$line" | grep -q 'error: test_mkdir'
76 echo "ERROR: fail test_mkdir" >&2
85 destroy_directories () {
92 for ((idx = 0; idx < $ndir; idx++)); do
93 if (( idx == 0 )); then
96 dirname=${basedir}${idx}
98 remote_shell $host $lctl --device $devno test_rmdir /$dirname > $rfile 2>&1
105 '/starting/ { n=0; next } \
106 /error/ {n = -1; exit} \
107 /^Total: total [0-9]+ threads [0-9]+ sec [0-9\.]+ [0-9]+\.[0-9]+\/second$/ \
108 { ave = strtonum($8); n++; next} \
109 /^[0-9]+\/[0-9]+ Total: [0-9]+\.[0-9]+\/second$/ \
110 { n++; v = strtonum($3); \
111 if (n == 1 || v < min) min = v; \
112 if (n == 1 || v > max) max = v; \
115 { if (n != 0) {n = -1; exit } } \
116 END { if (n == 1) { min = ave; max = ave; } \
117 printf "%d %f %f %f\n", n, ave, min, max}'
120 get_global_stats () {
125 if (n == 1) { err = $1; ave = $2; min = $3; max = $4} \
127 { if ($1 < err) err = $1; \
128 if ($2 < min) min = $2; \
129 if ($3 > max) max = $3; \
132 END { if (n == 0) err = 0; \
133 printf "%d %f %f %f\n", err, ave, min, max}'
137 if [ "$1" = "-n" ]; then
142 echo $minusn "$*" >> $rsltf
148 for name in $tests_str; do
153 # hide a little trick to unset this from the command line
154 if [ "$lustre_root" == " " ]; then
158 if [ -z "$lustre_root" ]; then
161 lctl=${lustre_root}/utils/lctl
164 declare -a client_names
165 declare -a host_names
166 if [ -z "$targets" ]; then
167 targets=$($lctl device_list | awk "{if (\$2 == \"UP\" && \
168 \$3 == \"mdt\") {print \$4} }")
169 if [ -z "$targets" ]; then
170 echo "Can't find any MDT to test. Please set targets=..."
175 # split out hostnames from mdt names
177 for trgt in $targets; do
178 str=(`split_hostname $trgt`)
179 host_names[$ndevs]=${str[0]}
180 client_names[$ndevs]=${str[1]}
185 if (( $stripe_count > 0 )); then
186 for ((i=0; i < $ndevs; i++)); do
187 host=${host_names[$i]}
188 obd=$(remote_shell $host $lctl device_list | awk "{if (\$2 == \"UP\" &&
189 \$3 == \"osc\") { print \$4 } }")
190 if [ -z "$obd" ]; then
191 echo "Need obdfilter to test stripe_count"
197 # check and insert obdecho module
198 if ! lsmod | grep obdecho > /dev/null; then
202 if [ $count -eq 0 -o "${tests[0]}" != "create" -o "${tests[(($count - 1))]}" != "destroy" ]; then
203 echo "tests: ${tests[@]}"
204 echo "First test must be 'create', and last test must be 'destroy'" 1>&2
208 rsltf="${rslt}.summary"
209 workf="${rslt}.detail"
210 cmdsf="${rslt}.script"
211 vmstatf="${rslt}.vmstat"
216 # disable portals debug and get obdecho loaded on all relevant hosts
217 unique_hosts=(`unique ${host_names[@]}`)
220 for host in ${unique_hosts[@]}; do
221 host_vmstatf=${vmstatf}_${host}
222 echo -n > $host_vmstatf
223 remote_shell $host "vmstat 5 >> $host_vmstatf" &> /dev/null &
225 vmstatpids[$pidcount]=$pid
226 pidcount=$((pidcount+1))
228 # get all the echo_client device numbers and names
229 for ((i=0; i < $ndevs; i++)); do
230 host=${host_names[$i]}
231 devno=(`get_ec_devno $host "${client_names[$i]}" "${client_names[$i]}" "mdt" $layer`)
232 if ((${#devno[@]} != 3)); then
235 devnos[$i]=${devno[0]}
236 client_names[$i]=${devno[1]}
237 do_teardown_ec[$i]=${devno[2]}
239 if (($ndevs <= 0 || ${#host_names[@]} <= 0)); then
240 echo "no devices or hosts specified"
243 print_summary "$(date) $0 from $(hostname)"
246 for ((idx = 0; idx < $ndevs; idx++)); do
247 host=${host_names[$idx]}
248 devno=${devnos[$idx]}
249 client_name="${host}:${client_names[$idx]}"
250 echo "=============> Create $dir_count directories on $client_name" >> $workf
251 destroy_directories $host $devno $dir_count $tmpf
252 ret=`create_directories $host $devno $dir_count $tmpf`
255 if [ $ret = "ERROR" ]; then
256 print_summary "created directories on $client_name failed"
263 for ((thr = $thrlo; thr <= $thrhi; thr*=2)); do
264 thr_per_dir=$((${thr}/${dir_count}))
265 # skip if no enough thread
266 if (( thr_per_dir <= 0 )); then
269 str=`printf 'mdt %1d file %7d dir %4d thr %4d ' \
270 $ndevs $file_count $dir_count $thr`
271 echo "=======================> $str" >> $workf
272 print_summary -n "$str"
274 for test in ${tests[@]}; do
276 for host in ${unique_hosts[@]}; do
277 echo "starting run for config: $config test: $test file: \
278 $file_count threads: $thr directories: $dir_count" >> ${vmstatf}_${host}
280 print_summary -n "$test "
281 # create per-host script files
282 for host in ${unique_hosts[@]}; do
283 echo -n > ${cmdsf}_${host}
285 for ((idx = 0; idx < $ndevs; idx++)); do
286 host=${host_names[$idx]}
287 devno=${devnos[$idx]}
289 [ $test = "create" ] && test="create -c $stripe_count"
290 echo >> ${cmdsf}_${host} \
291 "$lctl > $tmpfi 2>&1 \
292 --threads $thr -$snap $devno test_$test -d /$basedir -D $dir_count \
293 -b $start_number -n $file_count"
296 for host in ${unique_hosts[@]}; do
297 echo "wait" >> ${cmdsf}_${host}
298 pidarray[$pidcount]=0
299 pidcount=$((pidcount+1))
302 for host in ${unique_hosts[@]}; do
303 remote_shell $host bash < ${cmdsf}_${host} &
304 pidarray[$pidcount]=$!
305 pidcount=$((pidcount+1))
308 for host in ${unique_hosts[@]}; do
309 wait ${pidarray[$pidcount]}
310 pidcount=$((pidcount+1))
313 # clean up per-host script files
314 for host in ${unique_hosts[@]}; do
318 # collect/check individual MDT stats
320 for ((idx = 0; idx < $ndevs; idx++)); do
321 client_name="${host_names[$idx]}:${client_names[$idx]}"
323 echo "=============> $test $client_name" >> $workf
324 host="${host_names[$idx]}"
325 remote_shell $host cat $tmpfi > ${tmpfi}_local
326 cat ${tmpfi}_local >> $workf
327 get_stats ${tmpfi}_local >> $tmpf
328 rm -f $tmpfi ${tmpfi}_local
330 # compute/display global min/max stats
331 echo "=============> $test global" >> $workf
333 stats=(`get_global_stats $tmpf`)
335 if ((stats[0] <= 0)); then
336 str=`printf "%17s " ERROR`
339 str=`awk "BEGIN {printf \"%7.2f [%7.2f,%7.2f] \", \
340 ${stats[1]}, ${stats[2]}, ${stats[3]}; exit}"`
342 print_summary -n "$str"
346 # destroy directories
348 for ((idx = 0; idx < $ndevs; idx++)); do
349 host=${host_names[$idx]}
350 devno=${devnos[$idx]}
351 client_name="${host}:${client_names[$idx]}"
352 echo "=============> Destroy $dir_count directories on $client_name" >> $workf
353 destroy_directories $host $devno $dir_count $tmpf