2 # vim:expandtab:shiftwidth=4:softtabstop=4:tabstop=4:
4 ######################################################################
7 # Prerequisite: For "stripe_count > 0" you need to have ost setup and mounted.
10 # case 1 (stripe_count=0 default):
11 # $ thrhi=8 dir_count=4 sh md-survey
12 # one can also run test with user defined targets as follows,
13 # $ thrhi=8 dir_count=4 file_count=50000 targets="lustre-MDT0000" sh md-survey
14 # case 2 (stripe_count > 0, must have ost mounted):
15 # $ thrhi=8 dir_count=4 file_count=50000 stripe_count=2 targets="lustre-MDT0000" sh md-survey
16 # [ NOTE: It is advised to have automated login (passwordless entry) on server ]
21 # Customisation variables
22 #####################################################################
23 # One can change variable values in this section as per requirements
24 # The following variables can be set in the environment, or on the
26 # result file prefix (date/time + hostname makes unique)
27 # NB ensure path to it exists
28 rslt_loc=${rslt_loc:-"/tmp"}
29 rslt=${rslt:-"$rslt_loc/mdt_survey_`date +%F@%R`_`uname -n`"}
31 # min and max thread count
35 # number of directories to test
36 dir_count=${dir_count:-$thrlo}
37 # number of files per thread
38 file_count=${file_count:-100000}
40 targets=${targets:-""}
41 stripe_count=${stripe_count:-0}
42 # what tests to run (first must be create, and last must be destroy)
43 # default=(create lookup md_getattr setxattr destroy)
44 tests_str=${tests_str:-"create lookup md_getattr setxattr destroy"}
46 # start number for each thread
47 start_number=${start_number:-2}
48 # Customisation variables ends here.
49 #####################################################################
50 # leave the rest of this alone unless you know what you're doing...
55 create_directories () {
62 for ((idx = 0; idx < $ndir; idx++)); do
63 if (( idx == 0 )); then
66 dirname=${basedir}${idx}
68 remote_shell $host $lctl --device $devno test_mkdir /$dirname > $rfile 2>&1
70 echo "$line" | grep -q 'error: test_mkdir'
73 echo "ERROR: fail test_mkdir" >&2
82 destroy_directories () {
89 for ((idx = 0; idx < $ndir; idx++)); do
90 if (( idx == 0 )); then
93 dirname=${basedir}${idx}
95 remote_shell $host $lctl --device $devno test_rmdir /$dirname > $rfile 2>&1
102 '/start at/ { n=0; next } \
103 /error at/ {n = -1; exit} \
105 /^[0-9]+\/[0-9]+ Total: [0-9]+\.[0-9]+\/second$/ \
106 { n++; v=strtonum($3); \
107 if (n == 1 || v < min) min = v; \
108 if (n == 1 || v > max) max = v; \
111 { if (n != 0) {n = -1; exit } } \
112 END {printf "%d %f %f\n", n, min, max}'
115 get_global_stats () {
120 if (n == 1) { err = $1; min = $2; max = $3}\
122 { if ($1 < err) err = $1; \
123 if ($2 < min) min = $2; \
124 if ($3 > max) max = $3; \
127 END { if (n == 0) err = 0; \
128 printf "%d %f %f\n", err, min, max}'
132 if [ "$1" = "-n" ]; then
137 echo $minusn "$*" >> $rsltf
143 for name in $tests_str; do
148 # hide a little trick to unset this from the command line
149 if [ "$lustre_root" == " " ]; then
153 if [ -z "$lustre_root" ]; then
156 lctl=${lustre_root}/utils/lctl
159 declare -a client_names
160 declare -a host_names
161 if [ -z "$targets" ]; then
162 targets=$($lctl device_list | awk "{if (\$2 == \"UP\" && \
163 \$3 == \"mdt\") {print \$4} }")
164 if [ -z "$targets" ]; then
165 echo "Can't find any MDT to test. Please set targets=..."
171 if (( $stripe_count > 0 )); then
172 obd=$($lctl device_list | awk "{if (\$2 == \"UP\" && \
173 \$3 == \"obdfilter\") {print \$4} }")
174 if [ -z "$obd" ]; then
175 echo "Need obdfilter to test stripe_count"
180 # split out hostnames from mdt names
182 for trgt in $targets; do
183 str=(`split_hostname $trgt`)
184 host_names[$ndevs]=${str[0]}
185 client_names[$ndevs]=${str[1]}
189 # check and insert obdecho module
190 if ! lsmod | grep obdecho > /dev/null; then
194 if [ $count -eq 0 -o "${tests[0]}" != "create" -o "${tests[(($count - 1))]}" != "destroy" ]; then
195 echo "tests: ${tests[@]}"
196 echo "First test must be 'create', and last test must be 'destroy'" 1>&2
200 rsltf="${rslt}.summary"
201 workf="${rslt}.detail"
202 cmdsf="${rslt}.script"
203 vmstatf="${rslt}.vmstat"
208 # disable portals debug and get obdecho loaded on all relevant hosts
209 unique_hosts=(`unique ${host_names[@]}`)
212 for host in ${unique_hosts[@]}; do
213 host_vmstatf=${vmstatf}_${host}
214 echo -n > $host_vmstatf
215 remote_shell $host "vmstat 5 >> $host_vmstatf" &> /dev/null &
217 vmstatpids[$pidcount]=$pid
218 pidcount=$((pidcount+1))
220 # get all the echo_client device numbers and names
221 for ((i=0; i < $ndevs; i++)); do
222 host=${host_names[$i]}
223 devno=(`get_ec_devno $host "${client_names[$i]}" "${client_names[$i]}" "mdt" "mdd"`)
224 if ((${#devno[@]} != 3)); then
227 devnos[$i]=${devno[0]}
228 client_names[$i]=${devno[1]}
229 do_teardown_ec[$i]=${devno[2]}
231 if (($ndevs <= 0 || ${#host_names[@]} <= 0)); then
232 echo "no devices or hosts specified"
235 print_summary "$(date) Mdt-survey from $(hostname)"
238 for ((idx = 0; idx < $ndevs; idx++)); do
239 host=${host_names[$idx]}
240 devno=${devnos[$idx]}
241 client_name="${host}:${client_names[$idx]}"
242 echo "=============> Create $dir_count directories on $client_name" >> $workf
243 destroy_directories $host $devno $dir_count $tmpf
244 ret=`create_directories $host $devno $dir_count $tmpf`
247 if [ $ret = "ERROR" ]; then
248 print_summary "created directories on $client_name failed"
254 for ((thr = $thrlo; thr <= $thrhi; thr*=2)); do
255 thr_per_dir=$((${thr}/${dir_count}))
256 # skip if no enough thread
257 if (( thr_per_dir <= 0 )); then
260 str=`printf 'mdt %1d file %7d dir %4d thr %4d ' \
261 $ndevs $file_count $dir_count $thr`
262 echo "=======================> $str" >> $workf
263 print_summary -n "$str"
265 for test in ${tests[@]}; do
267 for host in ${unique_hosts[@]}; do
268 echo "starting run for config: $config test: $test file: \
269 $file_count threads: $thr directories: $dir_count" >> ${vmstatf}_${host}
271 print_summary -n "$test "
272 # create per-host script files
273 for host in ${unique_hosts[@]}; do
274 echo -n > ${cmdsf}_${host}
276 for ((idx = 0; idx < $ndevs; idx++)); do
277 host=${host_names[$idx]}
278 devno=${devnos[$idx]}
280 [ $test = "create" ] && test="create -c $stripe_count"
281 echo >> ${cmdsf}_${host} \
282 "$lctl > $tmpfi 2>&1 \
283 --threads $thr -$snap $devno test_$test -d /$basedir -D $dir_count \
284 -b $start_number -n $file_count"
287 for host in ${unique_hosts[@]}; do
288 echo "wait" >> ${cmdsf}_${host}
289 pidarray[$pidcount]=0
290 pidcount=$((pidcount+1))
292 # timed run of all the per-host script files
295 for host in ${unique_hosts[@]}; do
296 remote_shell $host bash < ${cmdsf}_${host} &
297 pidarray[$pidcount]=$!
298 pidcount=$((pidcount+1))
301 for host in ${unique_hosts[@]}; do
302 wait ${pidarray[$pidcount]}
303 pidcount=$((pidcount+1))
307 # clean up per-host script files
308 for host in ${unique_hosts[@]}; do
312 # compute bandwidth from total data / elapsed time
313 str=`awk "BEGIN {printf \"%7.2f \", \
314 ( $file_count * $thr_per_dir ) / ( $t1 - $t0 )}"`
315 print_summary -n "$str"
316 # collect/check individual MDT stats
318 for ((idx = 0; idx < $ndevs; idx++)); do
319 client_name="${host_names[$idx]}:${client_names[$idx]}"
321 echo "=============> $test $client_name" >> $workf
322 host="${host_names[$idx]}"
323 remote_shell $host cat $tmpfi > ${tmpfi}_local
324 cat ${tmpfi}_local >> $workf
325 get_stats ${tmpfi}_local >> $tmpf
326 rm -f $tmpfi ${tmpfi}_local
328 # compute/display global min/max stats
329 echo "=============> $test global" >> $workf
331 stats=(`get_global_stats $tmpf`)
333 if ((stats[0] <= 0)); then
334 if ((stats[0] < 0)); then
335 str=`printf "%17s " ERROR`
337 str=`printf "%17s " SHORT`
340 str=`awk "BEGIN {printf \"[%7.2f,%7.2f] \", \
341 ${stats[1]}, ${stats[2]}; exit}"`
343 print_summary -n "$str"
347 # destroy directories
349 for ((idx = 0; idx < $ndevs; idx++)); do
350 host=${host_names[$idx]}
351 devno=${devnos[$idx]}
352 client_name="${host}:${client_names[$idx]}"
353 echo "=============> Destroy $dir_count directories on $client_name" >> $workf
354 destroy_directories $host $devno $dir_count $tmpf