2 # -*- mode: Bash; tab-width: 4; indent-tabs-mode: t; -*-
3 # vim:shiftwidth=4:softtabstop=4:tabstop=4:
5 ######################################################################
8 # Prerequisite: For "stripe_count > 0" you need to have ost setup and mounted.
11 # case 1 (stripe_count=0 default):
12 # $ thrhi=8 dir_count=4 sh mds-survey
13 # one can also run test with user defined targets as follows,
14 # $ thrhi=8 dir_count=4 file_count=50000 targets="lustre-MDT0000" sh mds-survey
15 # case 2 (stripe_count > 0, must have ost mounted):
16 # $ thrhi=8 dir_count=4 file_count=50000 stripe_count=2
17 # targets="lustre-MDT0000" sh mds-survey
18 # [ NOTE: It is advised to have automated login (passwordless entry) on server ]
21 source $(dirname $0)/libecho
23 # Customisation variables
24 #####################################################################
25 # One can change variable values in this section as per requirements
26 # The following variables can be set in the environment, or on the
28 # result file prefix (date/time + hostname makes unique)
29 # NB ensure path to it exists
30 rslt_loc=${rslt_loc:-"/tmp"}
31 rslt=${rslt:-"$rslt_loc/mds_survey_`date +%F@%R`_`uname -n`"}
33 # min and max thread count
37 # number of directories to test
38 dir_count=${dir_count:-$thrlo}
39 # number of files per thread
40 file_count=${file_count:-100000}
42 targets=${targets:-""}
43 stripe_count=${stripe_count:-0}
44 # what tests to run (first must be create, and last must be destroy)
45 # default=(create lookup md_getattr setxattr destroy)
46 tests_str=${tests_str:-"create lookup md_getattr setxattr destroy"}
48 # start number for each thread
49 start_number=${start_number:-2}
53 # Customisation variables ends here.
54 #####################################################################
55 # leave the rest of this alone unless you know what you're doing...
60 create_directories () {
68 for ((idx = 0; idx < $ndir; idx++)); do
69 if (( idx == 0 )); then
70 dirname="$(printf "${mdtbasedir}" $mdtidx)${basedir}"
72 dirname="$(printf "${mdtbasedir}" $mdtidx)${basedir}${idx}"
74 remote_shell $host $lctl --device $devno test_mkdir /$dirname > $rfile 2>&1
76 echo "$line" | grep -q 'error: test_mkdir'
79 echo "ERROR: fail test_mkdir" >&2
88 destroy_directories () {
96 for ((idx = 0; idx < $ndir; idx++)); do
97 if (( idx == 0 )); then
98 dirname="$(printf "${mdtbasedir}" $mdtidx)${basedir}"
100 dirname="$(printf "${mdtbasedir}" $mdtidx)${basedir}${idx}"
102 remote_shell $host $lctl --device $devno test_rmdir /$dirname > $rfile 2>&1
109 '/starting/ { n=0; next } \
110 /error/ {n = -1; exit} \
111 /^Total: total [0-9]+ threads [0-9]+ sec [0-9\.]+ [0-9]+\.[0-9]+\/second$/ \
112 { ave = strtonum($8); n++; next} \
113 /^[0-9]+\/[0-9]+ Total: [0-9]+\.[0-9]+\/second$/ \
114 { n++; v = strtonum($3); \
115 if (n == 1 || v < min) min = v; \
116 if (n == 1 || v > max) max = v; \
119 { if (n != 0) {n = -1; exit } } \
120 END { if (n == 1) { min = ave; max = ave; } \
121 printf "%d %f %f %f\n", n, ave, min, max}'
124 get_global_stats () {
129 if (n == 1) { err = $1; ave = $2; min = $3; max = $4} \
131 { if ($1 < err) err = $1; \
133 if ($3 < min) min = $3; \
134 if ($4 > max) max = $4; \
137 END { if (n == 0) err = 0; \
138 printf "%d %f %f %f\n", err, ave/n, min, max}'
142 if [ "$1" = "-n" ]; then
147 echo $minusn "$*" >> $rsltf
153 for name in $tests_str; do
158 # hide a little trick to unset this from the command line
159 if [ "$lustre_root" == " " ]; then
163 if [ -z "$lustre_root" ]; then
166 lctl=${lustre_root}/utils/lctl
169 declare -a client_names
170 declare -a host_names
171 declare -a client_indexes
172 if [ -z "$targets" ]; then
173 targets=$($lctl device_list | awk "{if (\$2 == \"UP\" && \
174 \$3 == \"mdt\") {print \$4} }")
175 if [ -z "$targets" ]; then
176 echo "Can't find any MDT to test. Please set targets=..."
181 # split out hostnames from mdt names
183 for trgt in $targets; do
184 str=(`split_hostname $trgt`)
185 host_names[$ndevs]=${str[0]}
186 client_names[$ndevs]=${str[1]}
187 client_indexes[$ndevs]=0x$(echo ${str[1]} |
188 sed 's/.*MDT\([0-9a-f][0-9a-f][0-9a-f][0-9a-f]\).*/\1/')
193 if (( $stripe_count > 0 )); then
194 for ((i=0; i < $ndevs; i++)); do
195 host=${host_names[$i]}
196 obd=$(remote_shell $host $lctl device_list | awk "{if (\$2 == \"UP\" &&
197 (\$3 == \"osc\" || \$3 == \"osp\")) { print \$4 } }")
198 if [ -z "$obd" ]; then
199 echo "Need obdfilter to test stripe_count"
205 # check and insert obdecho module
206 if ! lsmod | grep obdecho > /dev/null; then
210 if [ $count -eq 0 -o "${tests[0]}" != "create" -o "${tests[(($count - 1))]}" != "destroy" ]; then
211 echo "tests: ${tests[@]}"
212 echo "First test must be 'create', and last test must be 'destroy'" 1>&2
216 rsltf="${rslt}.summary"
217 workf="${rslt}.detail"
218 cmdsf="${rslt}.script"
219 vmstatf="${rslt}.vmstat"
224 # disable portals debug and get obdecho loaded on all relevant hosts
225 unique_hosts=(`unique ${host_names[@]}`)
228 for host in ${unique_hosts[@]}; do
229 host_vmstatf=${vmstatf}_${host}
230 echo -n > $host_vmstatf
231 remote_shell $host "vmstat 5 >> $host_vmstatf" &> /dev/null &
233 vmstatpids[$pidcount]=$pid
234 pidcount=$((pidcount+1))
236 # get all the echo_client device numbers and names
237 for ((i=0; i < $ndevs; i++)); do
238 host=${host_names[$i]}
239 devno=(`get_ec_devno $host "${client_names[$i]}" "${client_names[$i]}" "mdt" $layer`)
240 if ((${#devno[@]} != 3)); then
243 devnos[$i]=${devno[0]}
244 client_names[$i]=${devno[1]}
245 do_teardown_ec[$i]=${devno[2]}
247 if (($ndevs <= 0 || ${#host_names[@]} <= 0)); then
248 echo "no devices or hosts specified"
251 print_summary "$(date) $0 from $(hostname)"
254 for ((idx = 0; idx < $ndevs; idx++)); do
255 host=${host_names[$idx]}
256 devno=${devnos[$idx]}
257 client_name="${host}:${client_names[$idx]}"
258 mdtidx=${client_indexes[$idx]}
259 echo "=============> Create $dir_count directories on $client_name" >> $workf
260 destroy_directories $host $devno $dir_count $tmpf $mdtidx
261 ret=`create_directories $host $devno $dir_count $tmpf $mdtidx`
264 if [ $ret = "ERROR" ]; then
265 print_summary "created directories on $client_name failed"
272 for ((thr = $thrlo; thr <= $thrhi; thr*=2)); do
273 thr_per_dir=$((${thr}/${dir_count}))
274 # skip if no enough thread
275 if (( thr_per_dir <= 0 )); then
278 file_count_per_thread=$((${file_count}/${thr}))
279 str=`printf 'mdt %1d file %7d dir %4d thr %4d ' \
280 $ndevs $file_count $dir_count $thr`
281 echo "=======================> $str" >> $workf
282 print_summary -n "$str"
284 for test in ${tests[@]}; do
286 for host in ${unique_hosts[@]}; do
287 echo "starting run for config: $config test: $test file: \
288 $file_count threads: $thr directories: $dir_count" >> ${vmstatf}_${host}
290 print_summary -n "$test "
291 # create per-host script files
292 for host in ${unique_hosts[@]}; do
293 echo -n > ${cmdsf}_${host}
295 for ((idx = 0; idx < $ndevs; idx++)); do
296 host=${host_names[$idx]}
297 devno=${devnos[$idx]}
298 dirname="$(printf "${mdtbasedir}" ${client_indexes[$idx]})$basedir"
300 [ "$test" = "create" ] && test="create -c $stripe_count"
301 echo >> ${cmdsf}_${host} \
302 "$lctl > $tmpfi 2>&1 \
303 --threads $thr -$snap $devno test_$test -d /$dirname -D $dir_count \
304 -b $start_number -n $file_count_per_thread"
307 for host in ${unique_hosts[@]}; do
308 echo "wait" >> ${cmdsf}_${host}
309 pidarray[$pidcount]=0
310 pidcount=$((pidcount+1))
313 for host in ${unique_hosts[@]}; do
314 remote_shell $host bash < ${cmdsf}_${host} &
315 pidarray[$pidcount]=$!
316 pidcount=$((pidcount+1))
319 for host in ${unique_hosts[@]}; do
320 wait ${pidarray[$pidcount]}
321 pidcount=$((pidcount+1))
324 # clean up per-host script files
325 for host in ${unique_hosts[@]}; do
329 # collect/check individual MDT stats
331 for ((idx = 0; idx < $ndevs; idx++)); do
332 client_name="${host_names[$idx]}:${client_names[$idx]}"
334 echo "=============> $test $client_name" >> $workf
335 host="${host_names[$idx]}"
336 remote_shell $host cat $tmpfi > ${tmpfi}_local
337 cat ${tmpfi}_local >> $workf
338 get_stats ${tmpfi}_local >> $tmpf
339 rm -f $tmpfi ${tmpfi}_local
341 # compute/display global min/max stats
342 echo "=============> $test global" >> $workf
344 stats=(`get_global_stats $tmpf`)
346 if ((stats[0] <= 0)); then
347 str=`printf "%17s " ERROR`
350 str=`awk "BEGIN {printf \"%7.2f [ %7.2f, %7.2f] \", \
351 ${stats[1]}, ${stats[2]}, ${stats[3]}; exit}"`
353 print_summary -n "$str"
357 # destroy directories
359 for ((idx = 0; idx < $ndevs; idx++)); do
360 host=${host_names[$idx]}
361 devno=${devnos[$idx]}
362 mdtidx=${client_indexes[$idx]}
363 client_name="${host}:${client_names[$idx]}"
364 echo "=============> Destroy $dir_count directories on $client_name" >> $workf
365 destroy_directories $host $devno $dir_count $tmpf $mdtidx