3 ######################################################################
6 # Prerequisite: For "stripe_count > 0" you need to have ost setup and mounted.
9 # case 1 (stripe_count=0 default):
10 # $ thrhi=8 dir_count=4 sh mds-survey
11 # one can also run test with user defined targets as follows,
12 # $ thrhi=8 dir_count=4 file_count=50000 targets="lustre-MDT0000" sh mds-survey
13 # case 2 (stripe_count > 0, must have ost mounted):
14 # $ thrhi=8 dir_count=4 file_count=50000 stripe_count=2
15 # targets="lustre-MDT0000" sh mds-survey
16 # [ NOTE: It is advised to have automated login (passwordless entry) on server ]
19 source $(dirname $0)/iokit-libecho
21 # Customisation variables
22 #####################################################################
23 # One can change variable values in this section as per requirements
24 # The following variables can be set in the environment, or on the
26 # result file prefix (date/time + hostname makes unique)
27 # NB ensure path to it exists
28 rslt_loc=${rslt_loc:-"/tmp"}
29 rslt=${rslt:-"$rslt_loc/mds_survey_`date +%F@%R`_`uname -n`"}
31 # min and max thread count
35 # number of directories to test
36 dir_count=${dir_count:-$thrlo}
37 # number of files per thread
38 file_count=${file_count:-100000}
40 targets=${targets:-""}
41 stripe_count=${stripe_count:-0}
42 # what tests to run (first must be create, and last must be destroy)
43 # default=(create lookup md_getattr setxattr destroy)
44 tests_str=${tests_str:-"create lookup md_getattr setxattr destroy"}
46 # start number for each thread
47 start_number=${start_number:-2}
51 # Customisation variables ends here.
52 #####################################################################
53 # leave the rest of this alone unless you know what you're doing...
58 create_directories () {
66 for ((idx = 0; idx < $ndir; idx++)); do
67 if (( idx == 0 )); then
68 dirname="$(printf "${mdtbasedir}" $mdtidx)${basedir}"
70 dirname="$(printf "${mdtbasedir}" $mdtidx)${basedir}${idx}"
72 remote_shell $host $lctl --device $devno test_mkdir /$dirname > $rfile 2>&1
74 echo "$line" | grep -q 'error: test_mkdir'
77 echo "ERROR: fail test_mkdir" >&2
86 destroy_directories () {
94 for ((idx = 0; idx < $ndir; idx++)); do
95 if (( idx == 0 )); then
96 dirname="$(printf "${mdtbasedir}" $mdtidx)${basedir}"
98 dirname="$(printf "${mdtbasedir}" $mdtidx)${basedir}${idx}"
100 remote_shell $host $lctl --device $devno test_rmdir /$dirname > $rfile 2>&1
115 /^Total: total [0-9]+ threads [0-9]+ sec [0-9\.]+ [0-9]+\.[0-9]+\/second$/ { \
116 ave = strtonum($8); \
120 /^[0-9]+\/[0-9]+ Total: [0-9]+\.[0-9]+\/second$/ { \
121 n++; v = strtonum($3); \
122 if (n == 1 || v < min) min = v; \
123 if (n == 1 || v > max) max = v; \
137 printf "%d %f %f %f\n", n, ave, min, max \
141 get_global_stats () {
168 printf "%d %f %f %f\n", err, ave/n, min, max \
173 if [ "$1" = "-n" ]; then
178 echo $minusn "$*" >> $rsltf
184 for name in $tests_str; do
189 # hide a little trick to unset this from the command line
190 if [ "$lustre_root" == " " ]; then
194 if [ -z "$lustre_root" ]; then
197 lctl=${lustre_root}/utils/lctl
200 declare -a client_names
201 declare -a host_names
202 declare -a client_indexes
203 if [ -z "$targets" ]; then
204 targets=$($lctl device_list | awk "{if (\$2 == \"UP\" && \
205 \$3 == \"mdt\") {print \$4} }")
206 if [ -z "$targets" ]; then
207 echo "Can't find any MDT to test. Please set targets=..."
212 # split out hostnames from mdt names
214 for trgt in $targets; do
215 str=($(split_hostname $trgt))
216 host_names[$ndevs]=${str[0]}
217 client_names[$ndevs]=${str[1]}
218 client_indexes[$ndevs]=0x$(echo ${str[1]} |
219 sed 's/.*MDT\([0-9a-f][0-9a-f][0-9a-f][0-9a-f]\).*/\1/')
224 if (( $stripe_count > 0 )); then
225 for ((i=0; i < $ndevs; i++)); do
226 host=${host_names[$i]}
227 obd=$(remote_shell $host $lctl device_list |
228 awk "{ if (\$2 == \"UP\" &&
229 (\$3 == \"osc\" || \$3 == \"osp\"))
231 if [ -z "$obd" ]; then
232 echo "Need obdfilter to test stripe_count"
238 # check and insert obdecho module
239 if ! lsmod | grep obdecho > /dev/null; then
243 if [ $count -eq 0 -o "${tests[0]}" != "create" -o "${tests[(($count - 1))]}" != "destroy" ]; then
244 echo "tests: ${tests[@]}"
245 echo "First test must be 'create', and last test must be 'destroy'" 1>&2
249 rsltf="${rslt}.summary"
250 workf="${rslt}.detail"
251 cmdsf="${rslt}.script"
252 vmstatf="${rslt}.vmstat"
257 # disable portals debug and get obdecho loaded on all relevant hosts
258 unique_hosts=($(unique ${host_names[@]}))
261 for host in ${unique_hosts[@]}; do
262 host_vmstatf=${vmstatf}_${host}
263 echo -n > $host_vmstatf
264 remote_shell $host "vmstat 5 >> $host_vmstatf" &> /dev/null &
266 vmstatpids[$pidcount]=$pid
267 pidcount=$((pidcount+1))
269 # get all the echo_client device numbers and names
270 for ((i=0; i < $ndevs; i++)); do
271 host=${host_names[$i]}
272 devno=($(get_ec_devno $host "${client_names[$i]}" "${client_names[$i]}" "mdt" $layer))
273 if ((${#devno[@]} != 3)); then
276 devnos[$i]=${devno[0]}
277 client_names[$i]=${devno[1]}
278 do_teardown_ec[$i]=${devno[2]}
280 if (($ndevs <= 0 || ${#host_names[@]} <= 0)); then
281 echo "no devices or hosts specified"
284 print_summary "$(date) $0 from $(hostname)"
287 for ((idx = 0; idx < $ndevs; idx++)); do
288 host=${host_names[$idx]}
289 devno=${devnos[$idx]}
290 client_name="${host}:${client_names[$idx]}"
291 mdtidx=${client_indexes[$idx]}
292 echo "=======> Create $dir_count directories on $client_name" >> $workf
293 destroy_directories $host $devno $dir_count $tmpf $mdtidx
294 ret=$(create_directories $host $devno $dir_count $tmpf $mdtidx)
297 if [ $ret = "ERROR" ]; then
298 print_summary "created directories on $client_name failed"
305 for ((thr = $thrlo; thr <= $thrhi; thr*=2)); do
306 thr_per_dir=$((${thr}/${dir_count}))
307 # skip if no enough thread
308 if (( thr_per_dir <= 0 )); then
311 file_count_per_thread=$((${file_count}/${thr}))
312 str=$(printf 'mdt %1d file %7d dir %4d thr %4d ' \
313 $ndevs $file_count $dir_count $thr)
314 echo "=======> $str" >> $workf
315 print_summary -n "$str"
317 for test in ${tests[@]}; do
319 for host in ${unique_hosts[@]}; do
320 echo "starting run for config: $config test: $test " \
321 "file: $file_count threads: $thr " \
322 "directories: $dir_count" >> ${vmstatf}_${host}
324 print_summary -n "$test "
325 # create per-host script files
326 for host in ${unique_hosts[@]}; do
327 echo -n > ${cmdsf}_${host}
329 for ((idx = 0; idx < $ndevs; idx++)); do
330 host=${host_names[$idx]}
331 devno=${devnos[$idx]}
332 dirname="$(printf "${mdtbasedir}" ${client_indexes[$idx]})$basedir"
334 [ "$test" = "create" ] && test="create -c $stripe_count"
335 echo >> ${cmdsf}_${host} \
336 "$lctl > $tmpfi 2>&1 \
337 --threads $thr -$snap $devno test_$test \
338 -d /$dirname -D $dir_count \
339 -b $start_number -n $file_count_per_thread"
342 for host in ${unique_hosts[@]}; do
343 echo "wait" >> ${cmdsf}_${host}
344 pidarray[$pidcount]=0
345 pidcount=$((pidcount+1))
348 for host in ${unique_hosts[@]}; do
349 remote_shell $host bash < ${cmdsf}_${host} &
350 pidarray[$pidcount]=$!
351 pidcount=$((pidcount+1))
354 for host in ${unique_hosts[@]}; do
355 wait ${pidarray[$pidcount]}
356 pidcount=$((pidcount+1))
359 # clean up per-host script files
360 for host in ${unique_hosts[@]}; do
364 # collect/check individual MDT stats
366 for ((idx = 0; idx < $ndevs; idx++)); do
367 client_name="${host_names[$idx]}:${client_names[$idx]}"
369 echo "=============> $test $client_name" >> $workf
370 host="${host_names[$idx]}"
371 remote_shell $host cat $tmpfi > ${tmpfi}_local
372 cat ${tmpfi}_local >> $workf
373 get_stats ${tmpfi}_local >> $tmpf
374 rm -f $tmpfi ${tmpfi}_local
376 # compute/display global min/max stats
377 echo "=============> $test global" >> $workf
379 stats=($(get_global_stats $tmpf))
381 if ((stats[0] <= 0)); then
382 str=$(printf "%17s " ERROR)
385 str=$(awk "BEGIN {printf \"%7.2f [ %7.2f, %7.2f] \", \
386 ${stats[1]}, ${stats[2]}, ${stats[3]}; exit}")
388 print_summary -n "$str"
393 # destroy directories
395 for ((idx = 0; idx < $ndevs; idx++)); do
396 host=${host_names[$idx]}
397 devno=${devnos[$idx]}
398 mdtidx=${client_indexes[$idx]}
399 client_name="${host}:${client_names[$idx]}"
400 echo "====> Destroy $dir_count directories on $client_name" >> $workf
401 destroy_directories $host $devno $dir_count $tmpf $mdtidx