5 LUSTRE=${LUSTRE:-$(cd $(dirname $0)/..; echo $PWD)}
6 . $LUSTRE/tests/test-framework.sh
8 . ${CONFIG:=$LUSTRE/tests/cfg/$NAME.sh}
11 ALWAYS_EXCEPT="$PARALLEL_SCALE_EXCEPT"
16 cbench_DIR=${cbench_DIR:-""}
17 cbench_IDIRS=${cbench_IDIRS:-10}
18 cbench_RUNS=${cbench_RUNS:-10} # FIXME: wiki page requirements is 30, do we really need 30 ?
20 if [ "$SLOW" = "no" ]; then
28 METABENCH=${METABENCH:-$(which metabench 2> /dev/null || true)}
29 mbench_NFILES=${mbench_NFILES:-30400}
30 [ "$SLOW" = "no" ] && mbench_NFILES=10000
31 MACHINEFILE=${MACHINEFILE:-$TMP/$(basename $0 .sh).machines}
33 mbench_THREADS=${mbench_THREADS:-4}
38 SIMUL=${SIMUL:=$(which simul 2> /dev/null || true)}
40 simul_THREADS=${simul_THREADS:-2}
41 simul_REP=${simul_REP:-20}
42 [ "$SLOW" = "no" ] && simul_REP=2
47 cnt_DIR=${cnt_DIR:-""}
48 cnt_NRUN=${cnt_NRUN:-10}
49 [ "$SLOW" = "no" ] && cnt_NRUN=2
54 CASC_RW=${CASC_RW:-$(which cascading_rw 2> /dev/null || true)}
56 casc_THREADS=${casc_THREADS:-2}
57 casc_REP=${casc_REP:-300}
58 [ "$SLOW" = "no" ] && casc_REP=10
63 IOR=${IOR:-$(which IOR 2> /dev/null || true)}
65 ior_THREADS=${ior_THREADS:-2}
66 ior_blockSize=${ior_blockSize:-6} # Gb
67 ior_DURATION=${ior_DURATION:-30} # minutes
68 [ "$SLOW" = "no" ] && ior_DURATION=5
71 # write_append_truncate
74 write_THREADS=${write_THREADS:-8}
75 write_REP=${write_REP:-10000}
76 [ "$SLOW" = "no" ] && write_REP=100
81 WRITE_DISJOINT=${WRITE_DISJOINT:-$(which write_disjoint 2> /dev/null || true)}
83 wdisjoint_THREADS=${wdisjoint_THREADS:-4}
84 wdisjoint_REP=${wdisjoint_REP:-10000}
85 [ "$SLOW" = "no" ] && wdisjoint_REP=100
91 PARALLEL_GROUPLOCK=${PARALLEL_GROUPLOCK:-$(which parallel_grouplock 2> /dev/null || true)}
92 parallel_grouplock_MINTASKS=${parallel_grouplock_MINTASKS:-5}
95 check_and_setup_lustre
104 echo "${var}=${!var}"
106 [ -e $MACHINEFILE ] && cat $MACHINEFILE
110 # 5 min * cbench_RUNS
114 # compile dir kernel-1 680MB
115 # required space 680MB * cbench_IDIRS = ~7 Gb
117 test_compilebench() {
118 print_opts cbench_DIR cbench_IDIRS cbench_RUNS
120 [ x$cbench_DIR = x ] &&
121 { skip_env "compilebench not found" && return; }
123 [ -e $cbench_DIR/compilebench ] || \
124 { skip_env "No compilebench build" && return; }
126 local space=$(df -P $DIR | tail -n 1 | awk '{ print $4 }')
127 if [ $space -le $((680 * 1024 * cbench_IDIRS)) ]; then
128 cbench_IDIRS=$(( space / 680 / 1024))
129 [ $cbench_IDIRS = 0 ] && \
130 skip_env "Need free space atleast 680 Mb, have $space" && return
132 log free space=$space, reducing initial dirs to $cbench_IDIRS
135 # t-f _base needs to be modifyed to set properly tdir
136 # for new "test_foo" functions names
137 # local testdir=$DIR/$tdir
138 local testdir=$DIR/d0.compilebench
143 local cmd="./compilebench -D $testdir -i $cbench_IDIRS -r $cbench_RUNS --makej"
152 [ $rc = 0 ] || error "compilebench failed: $rc"
155 run_test compilebench "compilebench"
158 [ x$METABENCH = x ] &&
159 { skip_env "metabench not found" && return; }
161 local clients=$CLIENTS
162 [ -z $clients ] && clients=$(hostname)
164 num_clients=$(get_node_count ${clients//,/ })
167 # Need space estimation here.
169 generate_machine_file $clients $MACHINEFILE || return $?
171 print_opts METABENCH clients mbench_NFILES mbench_THREADS
173 local testdir=$DIR/d0.metabench
175 # mpi_run uses mpiuser
178 # -C Run the file creation tests.
179 # -S Run the file stat tests.
180 # -c nfile Number of files to be used in each test.
181 # -k Cleanup. Remove the test directories.
182 local cmd="$METABENCH -w $testdir -c $mbench_NFILES -C -S -k"
184 mpi_run -np $((num_clients * $mbench_THREADS)) -machinefile ${MACHINEFILE} $cmd
186 if [ $rc != 0 ] ; then
187 error "metabench failed! $rc"
191 run_test metabench "metabench"
194 if [ "$NFSCLIENT" ]; then
195 skip "skipped for NFSCLIENT mode"
200 { skip_env "simul not found" && return; }
202 local clients=$CLIENTS
203 [ -z $clients ] && clients=$(hostname)
205 local num_clients=$(get_node_count ${clients//,/ })
208 # Need space estimation here.
210 generate_machine_file $clients $MACHINEFILE || return $?
212 print_opts SIMUL clients simul_REP simul_THREADS
214 local testdir=$DIR/d0.simul
216 # mpi_run uses mpiuser
219 # -n # : repeat each test # times
220 # -N # : repeat the entire set of tests # times
222 local cmd="$SIMUL -d $testdir -n $simul_REP -N $simul_REP"
225 mpi_run -np $((num_clients * $simul_THREADS)) -machinefile ${MACHINEFILE} $cmd
228 if [ $rc != 0 ] ; then
229 error "simul failed! $rc"
233 run_test simul "simul"
235 test_connectathon() {
236 print_opts cnt_DIR cnt_NRUN
239 { skip_env "connectathon dir not found" && return; }
241 [ -e $cnt_DIR/runtests ] || \
242 { skip_env "No connectathon runtests found" && return; }
244 local testdir=$DIR/d0.connectathon
251 # cthon options (must be in this order)
253 # -N numpasses - will be passed to the runtests script. This argument
254 # is optional. It specifies the number of times to run
257 # One of these test types
262 # -a all of the above
264 # -f a quick functionality test
268 # Include lock tests unless we're running on nfsv4
269 local fstype=$(df -TP $testdir | awk 'NR==2 {print $2}')
270 echo "$testdir: $fstype"
271 if [[ $fstype != "nfs4" ]]; then
275 for test in $tests; do
276 local cmd="./runtests -N $cnt_NRUN $test -f $testdir"
282 [ $rc = 0 ] || error "connectathon failed: $rc"
288 run_test connectathon "connectathon"
292 { skip_env "IOR not found" && return; }
294 local clients=$CLIENTS
295 [ -z $clients ] && clients=$(hostname)
297 local num_clients=$(get_node_count ${clients//,/ })
299 local space=$(df -P $DIR | tail -n 1 | awk '{ print $4 }')
300 echo "+ $ior_blockSize * 1024 * 1024 * $num_clients * $ior_THREADS "
301 if [ $((space / 2)) -le $(( ior_blockSize * 1024 * 1024 * num_clients * ior_THREADS)) ]; then
302 echo "+ $space * 9/10 / 1024 / 1024 / $num_clients / $ior_THREADS"
303 ior_blockSize=$(( space /2 /1024 /1024 / num_clients / ior_THREADS ))
304 [ $ior_blockSize = 0 ] && \
305 skip_env "Need free space more than ($num_clients * $ior_THREADS )Gb: $((num_clients*ior_THREADS *1024 *1024*2)), have $space" && return
307 echo "free space=$space, Need: $num_clients x $ior_THREADS x $ior_blockSize Gb (blockSize reduced to $ior_blockSize Gb)"
310 generate_machine_file $clients $MACHINEFILE || return $?
312 print_opts IOR ior_THREADS ior_DURATION MACHINEFILE
314 local testdir=$DIR/d0.ior
316 # mpi_run uses mpiuser
318 if [ "$NFSCLIENT" ]; then
319 setstripe_nfsserver $testdir -c -1 ||
320 { error "setstripe on nfsserver failed" && return 1; }
322 $LFS setstripe $testdir -c -1 ||
323 { error "setstripe failed" && return 2; }
326 # -b N blockSize -- contiguous bytes to write per task (e.g.: 8, 4k, 2m, 1g)"
328 # -t N transferSize -- size of transfer in bytes (e.g.: 8, 4k, 2m, 1g)"
329 # -w writeFile -- write file"
330 # -r readFile -- read existing file"
331 # -T maxTimeDuration -- max time in minutes to run tests"
332 # -k keepFile -- keep testFile(s) on program exit
333 local cmd="$IOR -a POSIX -b ${ior_blockSize}g -o $testdir/iorData -t 2m -v -w -r -T $ior_DURATION -k"
336 mpi_run -np $((num_clients * $ior_THREADS)) -machinefile ${MACHINEFILE} $cmd
339 if [ $rc != 0 ] ; then
340 error "ior failed! $rc"
346 test_cascading_rw() {
347 if [ "$NFSCLIENT" ]; then
348 skip "skipped for NFSCLIENT mode"
353 { skip_env "cascading_rw not found" && return; }
355 local clients=$CLIENTS
356 [ -z $clients ] && clients=$(hostname)
358 num_clients=$(get_node_count ${clients//,/ })
361 # Need space estimation here.
363 generate_machine_file $clients $MACHINEFILE || return $?
365 print_opts CASC_RW clients casc_THREADS casc_REP MACHINEFILE
367 local testdir=$DIR/d0.cascading_rw
369 # mpi_run uses mpiuser
373 # -n: repeat test # times
375 local cmd="$CASC_RW -g -d $testdir -n $casc_REP"
378 mpi_run -np $((num_clients * $casc_THREADS)) -machinefile ${MACHINEFILE} $cmd
381 if [ $rc != 0 ] ; then
382 error "cascading_rw failed! $rc"
386 run_test cascading_rw "cascading_rw"
388 test_write_append_truncate() {
389 if [ "$NFSCLIENT" ]; then
390 skip "skipped for NFSCLIENT mode"
394 # location is lustre/tests dir
395 if ! which write_append_truncate > /dev/null 2>&1 ; then
396 skip_env "write_append_truncate not found"
400 local clients=$CLIENTS
401 [ -z $clients ] && clients=$(hostname)
403 local num_clients=$(get_node_count ${clients//,/ })
406 # Need space estimation here.
408 generate_machine_file $clients $MACHINEFILE || return $?
410 local testdir=$DIR/d0.write_append_truncate
411 local file=$testdir/f0.wat
413 print_opts clients write_REP write_THREADS MACHINEFILE
416 # mpi_run uses mpiuser
419 local cmd="write_append_truncate -n $write_REP $file"
422 mpi_run -np $((num_clients * $write_THREADS)) -machinefile ${MACHINEFILE} $cmd
425 if [ $rc != 0 ] ; then
426 error "write_append_truncate failed! $rc"
431 run_test write_append_truncate "write_append_truncate"
433 test_write_disjoint() {
434 if [ "$NFSCLIENT" ]; then
435 skip "skipped for NFSCLIENT mode"
439 [ x$WRITE_DISJOINT = x ] &&
440 { skip_env "write_disjoint not found" && return; }
442 local clients=$CLIENTS
443 [ -z $clients ] && clients=$(hostname)
445 local num_clients=$(get_node_count ${clients//,/ })
448 # Need space estimation here.
450 generate_machine_file $clients $MACHINEFILE || return $?
452 print_opts WRITE_DISJOINT clients wdisjoint_THREADS wdisjoint_REP MACHINEFILE
453 local testdir=$DIR/d0.write_disjoint
455 # mpi_run uses mpiuser
458 local cmd="$WRITE_DISJOINT -f $testdir/file -n $wdisjoint_REP"
461 mpi_run -np $((num_clients * $wdisjoint_THREADS)) -machinefile ${MACHINEFILE} $cmd
464 if [ $rc != 0 ] ; then
465 error "write_disjoint failed! $rc"
469 run_test write_disjoint "write_disjoint"
471 test_parallel_grouplock() {
472 if [ "$NFSCLIENT" ]; then
473 skip "skipped for NFSCLIENT mode"
477 [ x$PARALLEL_GROUPLOCK = x ] &&
478 { skip "PARALLEL_GROUPLOCK not found" && return; }
480 local clients=$CLIENTS
481 [ -z $clients ] && clients=$(hostname)
483 local num_clients=$(get_node_count ${clients//,/ })
485 generate_machine_file $clients $MACHINEFILE || return $?
487 print_opts clients parallel_grouplock_MINTASKS MACHINEFILE
489 local testdir=$DIR/d0.parallel_grouplock
491 # mpi_run uses mpiuser
494 do_nodes $clients "lctl set_param llite.*.max_rw_chunk=0" ||
495 error "set_param max_rw_chunk=0 failed "
500 for i in $(seq 12); do
502 local cmd="$PARALLEL_GROUPLOCK -g -v -d $testdir $subtest"
505 mpi_run -np $parallel_grouplock_MINTASKS -machinefile ${MACHINEFILE} $cmd
507 if [ $rc != 0 ] ; then
508 error_noexit "parallel_grouplock subtests $subtest failed! $rc"
510 echo "parallel_grouplock subtests $subtest PASS"
512 let status=$((status + rc))
513 # clear debug to collect one log per one test
514 do_nodes $(comma_list $(nodes_list)) lctl clear
516 [ $status -eq 0 ] || error "parallel_grouplock status: $status"
519 run_test parallel_grouplock "parallel_grouplock"
521 statahead_NUMMNTPTS=${statahead_NUMMNTPTS:-5}
522 statahead_NUMFILES=${statahead_NUMFILES:-500000}
524 cleanup_statahead () {
531 for i in $(seq 0 $num_mntpts);do
532 zconf_umount_clients $clients ${mntpt_root}$i ||
533 error_exit "Failed to umount lustre on ${mntpt_root}$i"
538 if [ "$NFSCLIENT" ]; then
539 skip "skipped for NFSCLIENT mode"
544 { skip_env "mdsrate not found" && return; }
546 local clients=$CLIENTS
547 [ -z $clients ] && clients=$(hostname)
549 local num_clients=$(get_node_count ${clients//,/ })
551 generate_machine_file $clients $MACHINEFILE || return $?
553 print_opts MDSRATE clients statahead_NUMMNTPTS statahead_NUMFILES
557 # do not use default "d[0-9]*" dir name
558 # to avoid of rm $statahead_NUMFILES (500k) files in t-f cleanup
560 local testdir=$DIR/$dir
562 # cleanup only if dir exists
563 # cleanup only $statahead_NUMFILES number of files
564 # ignore the other files created by someone else
566 mdsrate_cleanup $((num_clients * 32)) $MACHINEFILE $statahead_NUMFILES $testdir 'f%%d' --ignore
569 # mpi_run uses mpiuser
572 local num_files=$statahead_NUMFILES
574 local IFree=$(inodes_available)
575 if [ $IFree -lt $num_files ]; then
581 local cmd="${MDSRATE} ${MDSRATE_DEBUG} --mknod --dir $testdir --nfiles $num_files --filefmt 'f%%d'"
584 mpi_run -np $((num_clients * 32)) -machinefile ${MACHINEFILE} $cmd
587 if [ $rc != 0 ] ; then
588 error "mdsrate failed to create $rc"
592 local num_mntpts=$statahead_NUMMNTPTS
593 local mntpt_root=$TMP/mntpt/lustre
594 mntopts=${MNTOPTSTATAHEAD:-$MOUNTOPT}
596 echo "Mounting $num_mntpts lustre clients starts on $clients"
597 trap "cleanup_statahead $clients $mntpt_root $num_mntpts" EXIT ERR
598 for i in $(seq 0 $num_mntpts); do
599 zconf_mount_clients $clients ${mntpt_root}$i $mntopts ||
600 error_exit "Failed to mount lustre on ${mntpt_root}$i on $clients"
603 do_rpc_nodes $clients cancel_lru_locks mdc
605 do_rpc_nodes $clients do_ls $mntpt_root $num_mntpts $dir
607 mdsrate_cleanup $((num_clients * 32)) $MACHINEFILE $num_files $testdir 'f%%d' --ignore
609 # use rm instead of rmdir because of
610 # testdir could contain the files created by someone else,
611 # or by previous run where is num_files prev > num_files current
613 cleanup_statahead $clients $mntpt_root $num_mntpts
616 run_test statahead "statahead test, multiple clients"
618 equals_msg `basename $0`: test complete, cleaning up
619 check_and_cleanup_lustre
620 [ -f "$TESTSUITELOG" ] && cat $TESTSUITELOG && grep -q FAIL $TESTSUITELOG && exit 1 || true