3 # Simple function used by run_*.sh scripts
8 if [ -z "${!name}" ]; then
9 echo "$0: $name must be set"
13 [ $failed ] && exit 1 || true
16 # lrepl - Lustre test Read-Eval-Print Loop.
18 # This function implements a REPL for the Lustre test framework. It
19 # doesn't exec an actual shell because the user may want to inspect
20 # variables and use functions from the test framework.
27 This is an interactive read-eval-print loop interactive shell
28 simulation that you can use to debug failing tests. You can
29 enter most bash command lines (see notes below).
31 Use this REPL to inspect variables, set them, call test
32 framework shell functions, etcetera.
34 'exit' or EOF to exit this shell.
36 set \$retcode to 0 to cause the assertion failure that
37 triggered this REPL to be ignored.
40 do_facet ost1 lctl get_param ost.*.ost.threads_*
41 do_rpc_nodes \$OSTNODES unload_modules
44 All but the last line of multi-line statements or blocks
45 must end in a backslash.
47 "Here documents" are not supported.
49 History is not supported, but command-line editing is.
53 # Prompt escapes don't work in read -p, sadly.
54 prompt=":${TESTNAME:-UNKNOWN}:$(uname -n):$(basename $PWD)% "
56 # We use read -r to get close to a shell experience
57 while read -e -r -p "$prompt" rawline; do
60 # Don't want to exit-exit, just exit the REPL
62 # We need to handle continuations, and read -r doesn't do
63 # that for us. Yet we need read -r.
65 # We also use case/esac to compare lines read to "*\\"
66 # because [ "$line" = *\\ ] and variants of that don't work.
68 while read -e -r -p '> ' rawline
70 line="$line"$'\n'"$rawline"
72 # We could check for here documents by matching
73 # against *<<*, but who cares.
86 # Finally! Time to eval.
90 echo $'\n\tExiting interactive shell...\n'
94 # lassert - Lustre test framework assert
96 # Arguments: failure code, failure message, expression/statement
98 # lassert evaluates the expression given, and, if false, calls
99 # error() to trigger test failure. If REPL_ON_LASSERT is true then
100 # lassert will call lrepl() to give the user an interactive shell.
101 # If the REPL sets retcode=0 then the assertion failure will be
108 echo "checking $* ($(eval echo \""$*"\"))..."
109 eval "$@" && return 0;
111 if ${REPL_ON_LASSERT:-false}; then
112 echo "Assertion $retcode failed: $* (expanded: $(eval echo \""$*"\"))
117 error "Assertion $retcode failed: $* (expanded: $(eval echo \""$*"\"))
122 # setmodopts- set module options for subsequent calls to load_modules
124 # Usage: setmodopts module_name new_value [var_in_which_to_save_old_value]
125 # setmodopts -a module_name new_value [var_in_which_to_save_old_value]
127 # In the second usage the new value is appended to the old.
131 if [ "$1" = -a ]; then
136 local _var=MODOPTS_$1
141 # Dynamic naming of variables is a pain in bash. In ksh93 we could
142 # write "nameref opts_var=${modname}_MODOPTS" then assign directly
143 # to opts_var. Associative arrays would also help, alternatively.
144 # Alas, we're stuck with eval until all distros move to a more recent
145 # version of bash. Fortunately we don't need to eval unset and export.
147 if [ -z "$_newvalue" ]; then
153 $_append && _newvalue="$_oldvalue $_newvalue"
154 export $_var="$_newvalue"
155 echo setmodopts: ${_var}=${_newvalue}
157 [ -n "$_savevar" ] && eval $_savevar=\""$_oldvalue"\"
160 echoerr () { echo "$@" 1>&2 ; }
163 echoerr "$(date +'%F %H:%M:%S'): client load was signaled to terminate"
165 local PGID=$(ps -eo "%c %p %r" | awk "/ $PPID / {print \$3}")
172 local mpirun="$MPIRUN $MPIRUN_OPTIONS"
173 local command="$mpirun $@"
174 local mpilog=$TMP/mpi.log
177 if [ -n "$MPI_USER" -a "$MPI_USER" != root -a -n "$mpirun" ]; then
178 echo "+ chmod 0777 $MOUNT"
180 command="su $MPI_USER bash -c \"$command \""
185 eval $command 2>&1 | tee $mpilog || true
188 if [ $rc -eq 0 ] && grep -q "p4_error:" $mpilog ; then
197 for i in ${1//,/ }; do
198 if [ "$list" = "" ]; then
201 list="$list$escape $i@$NETTYPE"
207 # FIXME: all setup/cleanup can be done without rpc.sh
210 [ x$1 = x--verbose ] && verbose=true
212 export LST_SESSION=`$LST show_session 2>/dev/null | awk -F " " '{print $5}'`
213 [ "$LST_SESSION" == "" ] && return
222 lst_session_cleanup_all () {
223 local list=$(comma_list $(nodes_list))
224 do_rpc_nodes $list lst_end_session
228 lsmod | grep -q lnet_selftest && \
229 rmmod lnet_selftest > /dev/null 2>&1 || true
233 local list=$(comma_list $(nodes_list))
235 # lst end_session needs to be executed only locally
236 # i.e. on node where lst new_session was called
237 lst_end_session --verbose
238 do_rpc_nodes $list lst_cleanup
242 load_module lnet_selftest
246 local list=$(comma_list $(nodes_list))
247 do_rpc_nodes $list lst_setup
253 # Passed a single argument, strips everything off following
254 # and includes the first period.
255 # client-20.lab.whamcloud.com becomes client-20
257 echo $(sed 's/\..*//' <<< $1)
263 # Find remote nodename, stripped of any domain, etc.
264 # 'hostname -s' is easy, but not implemented on all systems
266 local rname=$(do_node $1 "uname -n" || echo -1)
267 if [[ "$rname" = "-1" ]]; then
270 echo $(short_hostname $rname)
280 echo "${var}=${!var}"
282 [ -e $MACHINEFILE ] && cat $MACHINEFILE
286 [ "$(stat -f -c %T $1)" = "lustre" ]
289 setstripe_getstripe () {
294 is_lustre $file || return 0
296 if [ -n "$params" ]; then
297 $LFS setstripe $params $file ||
298 error "setstripe $params failed"
300 $LFS getstripe $file ||
301 error "getstripe $file failed"
306 local cbench_DIR=${cbench_DIR:-""}
307 local cbench_IDIRS=${cbench_IDIRS:-2}
308 local cbench_RUNS=${cbench_RUNS:-2}
310 print_opts cbench_DIR cbench_IDIRS cbench_RUNS
312 [ x$cbench_DIR = x ] &&
313 skip_env "compilebench not found"
315 [ -e $cbench_DIR/compilebench ] ||
316 skip_env "No compilebench build"
319 # compile dir kernel-0 ~1GB
320 # required space ~1GB * cbench_IDIRS
321 local space=$(df -P $dir | tail -n 1 | awk '{ print $4 }')
322 if [[ $space -le $((1024 * 1024 * cbench_IDIRS)) ]]; then
323 cbench_IDIRS=$((space / 1024 / 1024))
324 [[ $cbench_IDIRS -eq 0 ]] &&
325 skip_env "Need free space at least 1GB, have $space"
327 echo "reducing initial dirs to $cbench_IDIRS"
329 echo "free space = $space KB"
332 # t-f _base needs to be modifyed to set properly tdir
333 # for new "test_foo" functions names
334 # local testdir=$DIR/$tdir
335 local testdir=$dir/d0.compilebench.$$
336 test_mkdir -p $testdir
337 setstripe_getstripe $testdir $cbench_STRIPEPARAMS
341 local cmd="./compilebench -D $testdir -i $cbench_IDIRS \
342 -r $cbench_RUNS --makej"
351 [ $rc = 0 ] || error "compilebench failed: $rc"
357 local mntpt=${2:-$MOUNT}
358 METABENCH=${METABENCH:-$(which metabench 2> /dev/null || true)}
359 mbench_NFILES=${mbench_NFILES:-30400}
361 mbench_THREADS=${mbench_THREADS:-4}
362 mbench_OPTIONS=${mbench_OPTIONS:-}
363 mbench_CLEANUP=${mbench_CLEANUP:-true}
365 [ x$METABENCH = x ] && skip_env "metabench not found"
367 print_opts METABENCH clients mbench_NFILES mbench_THREADS
369 local testdir=$dir/d0.metabench
370 test_mkdir -p $testdir
371 setstripe_getstripe $testdir $mbench_STRIPEPARAMS
373 # mpi_run uses mpiuser
376 # -C Run the file creation tests. Creates zero byte files.
377 # -S Run the file stat tests.
378 # -c nfile Number of files to be used in each test.
379 # -k Cleanup files when finished.
380 local cmd="$METABENCH -w $testdir -c $mbench_NFILES -C -S $mbench_OPTIONS"
383 # find out if we need to use srun by checking $SRUN_PARTITION
384 if [ "$SRUN_PARTITION" ]; then
385 $SRUN $SRUN_OPTIONS -D $testdir -w $clients -N $num_clients \
386 -n $((num_clients * mbench_THREADS)) \
387 -p $SRUN_PARTITION -- $cmd
389 mpi_run ${MACHINEFILE_OPTION} ${MACHINEFILE} \
390 -np $((num_clients * $mbench_THREADS)) $cmd
394 if [ $rc != 0 ] ; then
395 error "metabench failed! $rc"
398 if $mbench_CLEANUP; then
401 mv $dir/d0.metabench $mntpt/_xxx.$(date +%s).d0.metabench
406 SIMUL=${SIMUL:=$(which simul 2> /dev/null || true)}
407 [ x$SIMUL = x ] && skip_env "simul not found"
408 [ "$NFSCLIENT" ] && skip "skipped for NFSCLIENT mode"
411 simul_THREADS=${simul_THREADS:-2}
412 simul_REP=${simul_REP:-20}
415 # Need space estimation here.
417 print_opts SIMUL clients simul_REP simul_THREADS
419 local testdir=$DIR/d0.simul
421 setstripe_getstripe $testdir $simul_STRIPEPARAMS
423 # mpi_run uses mpiuser
426 # -n # : repeat each test # times
427 # -N # : repeat the entire set of tests # times
429 local cmd="$SIMUL -d $testdir -n $simul_REP -N $simul_REP"
432 # find out if we need to use srun by checking $SRUN_PARTITION
433 if [ "$SRUN_PARTITION" ]; then
434 $SRUN $SRUN_OPTIONS -D $testdir -w $clients -N $num_clients \
435 -n $((num_clients * simul_THREADS)) -p $SRUN_PARTITION \
438 mpi_run ${MACHINEFILE_OPTION} ${MACHINEFILE} \
439 -np $((num_clients * simul_THREADS)) $cmd
443 if [ $rc != 0 ] ; then
444 error "simul failed! $rc"
450 MDTEST=${MDTEST:=$(which mdtest 2> /dev/null || true)}
451 [ x$MDTEST = x ] && skip_env "mdtest not found"
452 [ "$NFSCLIENT" ] && skip "skipped for NFSCLIENT mode"
455 mdtest_THREADS=${mdtest_THREADS:-2}
456 mdtest_nFiles=${mdtest_nFiles:-"100000"}
457 # We devide the files by number of core
458 mdtest_nFiles=$((mdtest_nFiles/mdtest_THREADS/num_clients))
459 mdtest_iteration=${mdtest_iteration:-1}
460 local mdtest_custom_params=${mdtest_custom_params:-""}
461 local type=${1:-"ssf"}
463 local mdtest_Nmntp=${mdtest_Nmntp:-1}
465 if [ $type = "ssf" ] && [ $mdtest_Nmntp -ne 1 ]; then
466 skip "shared directory mode is not compatible" \
467 "with multiple directory paths"
471 # Need space estimation here.
473 print_opts MDTEST mdtest_iteration mdtest_THREADS mdtest_nFiles
475 local testdir=$DIR/d0.mdtest
477 setstripe_getstripe $testdir $mdtest_STRIPEPARAMS
480 for ((i=1; i<mdtest_Nmntp; i++)); do
481 zconf_mount_clients $clients $MOUNT$i "$mntopts" ||
482 error_exit "Failed $clients on $MOUNT$i"
483 local dir=$DIR$i/d0.mdtest$i
485 setstripe_getstripe $dir $mdtest_SETSTRIPEPARAMS
487 testdir="$testdir@$dir"
489 # mpi_run uses mpiuser
491 # -i # : repeat each test # times
493 # -n # : number of file/dir to create/stat/remove
494 # -u : each process create/stat/remove individually
496 local cmd="$MDTEST -d $testdir -i $mdtest_iteration \
497 -n $mdtest_nFiles $mdtest_custom_params"
499 [ $type = "fpp" ] && cmd="$cmd -u"
502 # find out if we need to use srun by checking $SRUN_PARTITION
503 if [ "$SRUN_PARTITION" ]; then
504 $SRUN $SRUN_OPTIONS -D $testdir -w $clients -N $num_clients \
505 -n $((num_clients * mdtest_THREADS)) \
506 -p $SRUN_PARTITION -- $cmd
508 mpi_run ${MACHINEFILE_OPTION} ${MACHINEFILE} \
509 -np $((num_clients * mdtest_THREADS)) $cmd
513 if [ $rc != 0 ] ; then
514 error "mdtest failed! $rc"
517 for ((i=1; i<mdtest_Nmntp; i++)); do
518 local dir=$DIR$i/d0.mdtest$i
520 zconf_umount_clients $clients $MOUNT$i ||
521 error_exit "Failed umount $MOUNT$i on $clients"
527 cnt_DIR=${cnt_DIR:-""}
528 cnt_NRUN=${cnt_NRUN:-10}
530 print_opts cnt_DIR cnt_NRUN
532 [ x$cnt_DIR = x ] && skip_env "connectathon dir not found"
533 [ -e $cnt_DIR/runtests ] || skip_env "No connectathon runtests found"
536 # "special" tests create a 30 MB file + misc. small files
537 # required space ~40 MB
538 local space=$(df -P $dir | tail -n 1 | awk '{ print $4 }')
539 if [[ $space -le $((1024 * 40)) ]]; then
540 skip_env "Need free space at least 40MB, have $space KB"
542 echo "free space = $space KB"
544 local testdir=$dir/d0.connectathon
545 test_mkdir -p $testdir
546 setstripe_getstripe $testdir $cnt_STRIPEPARAMS
552 # To run connectathon:
553 # runtests [-a|-b|-g|-s|-l] [-f|-n|-t] [-N numpasses] [test-directory]
555 # One of the following test types
560 # -a all of the above
562 # -f a quick functional test
563 # -n suppress directory operations (mkdir and rmdir)
564 # -t run with time statistics (default for basic tests)
566 # -N numpasses - specifies the number of times to run
567 # the tests. Optional.
570 # Include lock tests unless we're running on nfsv4
571 local fstype=$(df -TP $testdir | awk 'NR==2 {print $2}')
572 echo "$testdir: $fstype"
573 if [[ $fstype != "nfs4" ]]; then
577 for test in $tests; do
578 local cmd="sh ./runtests -N $cnt_NRUN $test -f $testdir"
584 [ $rc = 0 ] || error "connectathon failed: $rc"
592 local type=${1:="ssf"}
594 local testdir=$dir/d0.ior.$type
595 local nfs_srvmntpt=$3
597 if [ "$NFSCLIENT" ]; then
598 [[ -n $nfs_srvmntpt ]] ||
599 { error "NFSCLIENT mode, but nfs exported dir"\
600 "is not set!" && return 1; }
603 IOR=${IOR:-$(which IOR 2> /dev/null || true)}
604 [ x$IOR = x ] && skip_env "IOR not found"
607 ior_THREADS=${ior_THREADS:-2}
608 ior_iteration=${ior_iteration:-1}
609 ior_blockSize=${ior_blockSize:-6}
610 ior_blockUnit=${ior_blockUnit:-M} # K, M, G
611 ior_xferSize=${ior_xferSize:-1M}
612 ior_type=${ior_type:-POSIX}
613 ior_DURATION=${ior_DURATION:-30} # minutes
614 ior_CLEANUP=${ior_CLEANUP:-true}
616 case ${ior_blockUnit} in
618 multiplier=$((1024 * 1024 * 1024))
621 multiplier=$((1024 * 1024))
626 *) error "Incorrect block unit should be one of [KMG]"
630 # calculate the space in bytes
631 local space=$(df -B 1 -P $dir | tail -n 1 | awk '{ print $4 }')
632 local total_threads=$((num_clients * ior_THREADS))
633 echo "+ $ior_blockSize * $multiplier * $total_threads "
634 if [ $((space / 2)) -le \
635 $((ior_blockSize * multiplier * total_threads)) ]; then
636 ior_blockSize=$((space / 2 / multiplier / total_threads))
637 [ $ior_blockSize -eq 0 ] &&
638 skip_env "Need free space more than $((2 * total_threads)) \
639 ${ior_blockUnit}: have $((space / multiplier))"
641 echo "(reduced blockSize to $ior_blockSize \
642 ${ior_blockUnit} bytes)"
645 print_opts IOR ior_THREADS ior_DURATION MACHINEFILE
647 test_mkdir -p $testdir
649 # mpi_run uses mpiuser
651 [[ "$ior_stripe_params" && -z "$ior_STRIPEPARAMS" ]] &&
652 ior_STRIPEPARAMS="$ior_stripe_params" &&
653 echo "got deprecated ior_stripe_params,"\
654 "use ior_STRIPEPARAMS instead"
655 setstripe_getstripe $testdir $ior_STRIPEPARAMS
659 # contiguous bytes to write per task (e.g.: 8, 4K, 2M, 1G)"
661 # -t N transferSize -- size of transfer in bytes (e.g.: 8, 4K, 2M, 1G)"
662 # -w writeFile -- write file"
663 # -r readFile -- read existing file"
664 # -W checkWrite -- check read after write"
665 # -C reorderTasks -- changes task ordering to n+1 ordering for readback
666 # -T maxTimeDuration -- max time in minutes to run tests"
667 # -k keepFile -- keep testFile(s) on program exit
670 if [ -n "$ior_custom_params" ]; then
671 cmd="$IOR -o $testdir/iorData $ior_custom_params"
673 cmd="$IOR -a $ior_type -b ${ior_blockSize}${ior_blockUnit} \
674 -o $testdir/iorData -t $ior_xferSize -v -C -w -r -W \
675 -i $ior_iteration -T $ior_DURATION -k"
678 [ $type = "fpp" ] && cmd="$cmd -F"
681 # find out if we need to use srun by checking $SRUN_PARTITION
682 if [ "$SRUN_PARTITION" ]; then
683 $SRUN $SRUN_OPTIONS -D $testdir -w $clients -N $num_clients \
684 -n $((num_clients * ior_THREADS)) -p $SRUN_PARTITION \
687 mpi_ior_custom_threads=${mpi_ior_custom_threads:-"$((num_clients * ior_THREADS))"}
688 mpi_run ${MACHINEFILE_OPTION} ${MACHINEFILE} \
689 -np $mpi_ior_custom_threads $cmd
693 if [ $rc != 0 ] ; then
694 error "ior failed! $rc"
696 $ior_CLEANUP && rm -rf $testdir || true
700 MIB=${MIB:=$(which mib 2> /dev/null || true)}
701 [ "$NFSCLIENT" ] && skip "skipped for NFSCLIENT mode"
702 [ x$MIB = x ] && skip_env "MIB not found"
705 mib_THREADS=${mib_THREADS:-2}
706 mib_xferSize=${mib_xferSize:-1m}
707 mib_xferLimit=${mib_xferLimit:-5000}
708 mib_timeLimit=${mib_timeLimit:-300}
709 mib_STRIPEPARAMS=${mib_STRIPEPARAMS:-"-c -1"}
711 print_opts MIB mib_THREADS mib_xferSize mib_xferLimit mib_timeLimit \
714 local testdir=$DIR/d0.mib
716 setstripe_getstripe $testdir $mib_STRIPEPARAMS
718 # mpi_run uses mpiuser
722 # -I Show intermediate values in output
723 # -H Show headers in output
724 # -L Do not issue new system calls after this many seconds
725 # -s Use system calls of this size
727 # -l Issue no more than this many system calls
728 local cmd="$MIB -t $testdir -s $mib_xferSize -l $mib_xferLimit \
729 -L $mib_timeLimit -HI -p mib.$(date +%Y%m%d%H%M%S)"
732 # find out if we need to use srun by checking $SRUN_PARTITION
733 if [ "$SRUN_PARTITION" ]; then
734 $SRUN $SRUN_OPTIONS -D $testdir -w $clients -N $num_clients \
735 -n $((num_clients * mib_THREADS)) -p $SRUN_PARTITION \
738 mpi_run ${MACHINEFILE_OPTION} ${MACHINEFILE} \
739 -np $((num_clients * mib_THREADS)) $cmd
743 if [ $rc != 0 ] ; then
744 error "mib failed! $rc"
750 CASC_RW=${CASC_RW:-$(which cascading_rw 2> /dev/null || true)}
751 [ x$CASC_RW = x ] && skip_env "cascading_rw not found"
752 [ "$NFSCLIENT" ] && skip "skipped for NFSCLIENT mode"
755 casc_THREADS=${casc_THREADS:-2}
756 casc_REP=${casc_REP:-300}
759 # Need space estimation here.
761 print_opts CASC_RW clients casc_THREADS casc_REP MACHINEFILE
763 local testdir=$DIR/d0.cascading_rw
765 setstripe_getstripe $testdir $casc_STRIPEPARAMS
767 # mpi_run uses mpiuser
771 # -n: repeat test # times
773 local cmd="$CASC_RW -g -d $testdir -n $casc_REP"
776 mpi_run ${MACHINEFILE_OPTION} ${MACHINEFILE} \
777 -np $((num_clients * $casc_THREADS)) $cmd
780 if [ $rc != 0 ] ; then
781 error "cascading_rw failed! $rc"
786 run_write_append_truncate() {
787 [ "$NFSCLIENT" ] && skip "skipped for NFSCLIENT mode"
788 # location is lustre/tests dir
789 if ! which write_append_truncate > /dev/null 2>&1 ; then
790 skip_env "write_append_truncate not found"
794 write_THREADS=${write_THREADS:-8}
795 write_REP=${write_REP:-10000}
798 # Need space estimation here.
800 local testdir=$DIR/d0.write_append_truncate
801 local file=$testdir/f0.wat
803 print_opts clients write_REP write_THREADS MACHINEFILE
806 # mpi_run uses mpiuser
807 setstripe_getstripe $testdir $write_STRIPEPARAMS
811 local cmd="write_append_truncate -n $write_REP $file"
814 mpi_run ${MACHINEFILE_OPTION} ${MACHINEFILE} \
815 -np $((num_clients * $write_THREADS)) $cmd
818 if [ $rc != 0 ] ; then
819 error "write_append_truncate failed! $rc"
825 run_write_disjoint() {
826 WRITE_DISJOINT=${WRITE_DISJOINT:-$(which write_disjoint 2> /dev/null ||
828 [ x$WRITE_DISJOINT = x ] && skip_env "write_disjoint not found"
829 [ "$NFSCLIENT" ] && skip "skipped for NFSCLIENT mode"
832 wdisjoint_THREADS=${wdisjoint_THREADS:-4}
833 wdisjoint_REP=${wdisjoint_REP:-10000}
837 # Need space estimation here.
839 print_opts WRITE_DISJOINT clients wdisjoint_THREADS wdisjoint_REP \
841 local testdir=$DIR/d0.write_disjoint
843 setstripe_getstripe $testdir $wdisjoint_STRIPEPARAMS
845 # mpi_run uses mpiuser
848 local cmd="$WRITE_DISJOINT -f $testdir/file -n $wdisjoint_REP -m \
852 mpi_run ${MACHINEFILE_OPTION} ${MACHINEFILE} \
853 -np $((num_clients * $wdisjoint_THREADS)) $cmd
856 if [ $rc != 0 ] ; then
857 error "write_disjoint failed! $rc"
862 run_parallel_grouplock() {
863 PARALLEL_GROUPLOCK=${PARALLEL_GROUPLOCK:-$(which parallel_grouplock \
864 2> /dev/null || true)}
866 [ x$PARALLEL_GROUPLOCK = x ] && skip "PARALLEL_GROUPLOCK not found"
867 [ "$NFSCLIENT" ] && skip "skipped for NFSCLIENT mode"
869 parallel_grouplock_MINTASKS=${parallel_grouplock_MINTASKS:-5}
871 print_opts clients parallel_grouplock_MINTASKS MACHINEFILE
873 local testdir=$DIR/d0.parallel_grouplock
875 setstripe_getstripe $testdir $parallel_grouplock_STRIPEPARAMS
877 # mpi_run uses mpiuser
883 for i in $(seq 12); do
885 local cmd="$PARALLEL_GROUPLOCK -g -v -d $testdir $subtest"
888 mpi_run ${MACHINEFILE_OPTION} ${MACHINEFILE} \
889 -np $parallel_grouplock_MINTASKS $cmd
891 if [ $rc != 0 ] ; then
892 error_noexit "parallel_grouplock subtests $subtest " \
895 echo "parallel_grouplock subtests $subtest PASS"
897 let status=$((status + rc))
898 # clear debug to collect one log per one test
899 do_nodes $(comma_list $(nodes_list)) lctl clear
901 [ $status -eq 0 ] || error "parallel_grouplock status: $status"
905 cleanup_statahead () {
912 for i in $(seq 0 $num_mntpts);do
913 zconf_umount_clients $clients ${mntpt_root}$i ||
914 error_exit "Failed to umount lustre on ${mntpt_root}$i"
919 if [[ -n $NFSCLIENT ]]; then
920 skip "Statahead testing is not supported on NFS clients."
922 [ x$MDSRATE = x ] && skip_env "mdsrate not found"
924 statahead_NUMMNTPTS=${statahead_NUMMNTPTS:-5}
925 statahead_NUMFILES=${statahead_NUMFILES:-500000}
927 print_opts MDSRATE clients statahead_NUMMNTPTS statahead_NUMFILES
931 # do not use default "d[0-9]*" dir name
932 # to avoid of rm $statahead_NUMFILES (500k) files in t-f cleanup
934 local testdir=$DIR/$dir
936 # cleanup only if dir exists
937 # cleanup only $statahead_NUMFILES number of files
938 # ignore the other files created by someone else
940 mdsrate_cleanup $((num_clients * 32)) $MACHINEFILE \
941 $statahead_NUMFILES $testdir 'f%%d' --ignore
944 setstripe_getstripe $testdir $statahead_STRIPEPARAMS
946 # mpi_run uses mpiuser
949 local num_files=$statahead_NUMFILES
951 local IFree=$(inodes_available)
952 if [ $IFree -lt $num_files ]; then
958 local cmd1="${MDSRATE} ${MDSRATE_DEBUG} --mknod --dir $testdir"
959 local cmd2="--nfiles $num_files --filefmt 'f%%d'"
960 local cmd="$cmd1 $cmd2"
963 mpi_run ${MACHINEFILE_OPTION} ${MACHINEFILE} \
964 -np $((num_clients * 32)) $cmd
967 if [ $rc != 0 ] ; then
968 error "mdsrate failed to create $rc"
972 local num_mntpts=$statahead_NUMMNTPTS
973 local mntpt_root=$TMP/mntpt/lustre
974 local mntopts=$MNTOPTSTATAHEAD
976 echo "Mounting $num_mntpts lustre clients starts on $clients"
977 trap "cleanup_statahead $clients $mntpt_root $num_mntpts" EXIT ERR
978 for i in $(seq 0 $num_mntpts); do
979 zconf_mount_clients $clients ${mntpt_root}$i "$mntopts" ||
980 error_exit "Failed to mount lustre on ${mntpt_root}$i on $clients"
983 do_rpc_nodes $clients cancel_lru_locks mdc
985 do_rpc_nodes $clients do_ls $mntpt_root $num_mntpts $dir
987 mdsrate_cleanup $((num_clients * 32)) $MACHINEFILE \
988 $num_files $testdir 'f%%d' --ignore
990 # use rm instead of rmdir because of
991 # testdir could contain the files created by someone else,
992 # or by previous run where is num_files prev > num_files current
994 cleanup_statahead $clients $mntpt_root $num_mntpts
997 cleanup_rr_alloc () {
1000 local mntpt_root="$2"
1001 local rr_alloc_MNTPTS="$3"
1002 local mntpt_dir=$(dirname ${mntpt_root})
1004 for i in $(seq 0 $((rr_alloc_MNTPTS - 1))); do
1005 zconf_umount_clients $clients ${mntpt_root}$i ||
1006 error_exit "Failed to umount lustre on ${mntpt_root}$i"
1008 do_nodes $clients "rm -rf $mntpt_dir"
1012 remote_mds_nodsh && skip "remote MDS with nodsh"
1013 echo "===Test gives more reproduction percentage if number of "\
1014 "client and ost are more. Test with 44 or more clients "\
1015 "and 73 or more OSTs gives 100% reproduction rate=="
1017 RR_ALLOC=${RR_ALLOC:-$(which rr_alloc 2> /dev/null || true)}
1018 [ x$RR_ALLOC = x ] && skip_env "rr_alloc not found"
1019 declare -a diff_max_min_arr
1020 # foeo = file on each ost. calc = calculated.
1023 local qos_prec_objs="${TMP}/qos_and_precreated_objects"
1024 local rr_alloc_NFILES=${rr_alloc_NFILES:-555}
1025 local rr_alloc_MNTPTS=${rr_alloc_MNTPTS:-11}
1026 local total_MNTPTS=$((rr_alloc_MNTPTS * num_clients))
1027 local mntpt_root="${TMP}/rr_alloc_mntpt/lustre"
1028 test_mkdir $DIR/$tdir
1029 setstripe_getstripe $DIR/$tdir $rr_alloc_STRIPEPARAMS
1031 chmod 0777 $DIR/$tdir
1033 trap "cleanup_rr_alloc $clients $mntpt_root $rr_alloc_MNTPTS" EXIT ERR
1034 for i in $(seq 0 $((rr_alloc_MNTPTS - 1))); do
1035 zconf_mount_clients $clients ${mntpt_root}$i $MOUNT_OPTS ||
1036 error_exit "Failed to mount lustre on ${mntpt_root}$i $clients"
1039 local cmd="$RR_ALLOC $mntpt_root/$tdir/ash $rr_alloc_NFILES \
1042 # Save mdt values, set threshold to 100% i.e always Round Robin,
1043 # restore the saved values again after creating files...
1044 save_lustre_params mds1 \
1045 "lod.$FSNAME-MDT0000*.qos_threshold_rr" > $qos_prec_objs
1046 save_lustre_params mds1 \
1047 "osp.$FSNAME-OST*-osc-MDT0000.create_count" >> $qos_prec_objs
1049 local old_create_count=$(grep -e "create_count" $qos_prec_objs |
1050 cut -d'=' -f 2 | sort -nr | head -n1)
1052 # Make sure that every osp has enough precreated objects for the file
1055 # create_count is always set to the power of 2 only, so if the files
1056 # per OST are not multiple of that then it will be set to nearest
1057 # lower power of 2. So set 'create_count' to the upper power of 2.
1059 foeo_calc=$((rr_alloc_NFILES * total_MNTPTS / OSTCOUNT))
1060 local create_count=$((2 * foeo_calc))
1062 # create_count accepted values:
1063 # [OST_MIN_PRECREATE=32, OST_MAX_PRECREATE=20000]
1064 # values exceeding OST_MAX_PRECREATE are lowered to the maximum.
1065 [[ $create_count -lt 32 ]] && create_count=32
1067 for i in $(seq $MDSCOUNT); do
1068 do_facet mds$i "$LCTL set_param -n \
1069 lod.$FSNAME-MDT*.qos_threshold_rr=100 \
1070 osp.$FSNAME-OST*-osc-MDT*.create_count=$create_count"
1073 # Create few temporary files in order to increase the precreated objects
1074 # to a desired value, before starting 'rr_alloc' app. Due to default
1075 # value 32 of precreation count (OST_MIN_PRECREATE=32), precreated
1076 # objects available are 32 initially, these gets exhausted very soon,
1077 # which causes skip of some osps when very large number of files
1078 # is created per OSTs.
1079 createmany -o $DIR/$tdir/foo- $(((old_create_count + 1) * OSTCOUNT)) \
1081 rm -f /$DIR/$tdir/foo*
1083 # Check for enough precreated objects... We should not
1084 # fail here because code(osp_precreate.c) also takes care of it.
1085 # So we have good chances of passing test even if this check fails.
1087 for ost_idx in $(seq 0 $((OSTCOUNT - 1))); do
1088 [[ $(precreated_ost_obj_count $mdt_idx $ost_idx) -ge \
1089 $foeo_calc ]] || echo "Warning: test may fail because" \
1090 "of lack of precreated objects on OST${ost_idx}"
1093 if [[ $total_MNTPTS -ne 0 ]]; then
1094 # Now start the actual file creation app.
1095 mpi_run "-np $total_MNTPTS" $cmd || return
1097 error "No mount point"
1100 restore_lustre_params < $qos_prec_objs
1101 rm -f $qos_prec_objs
1103 diff_max_min_arr=($($LFS getstripe -r $DIR/$tdir/ |
1104 grep "lmm_stripe_offset:" | awk '{print $2}' | sort -n |
1105 uniq -c | awk 'NR==1 {min=max=$1} \
1106 { $1<min ? min=$1 : min; $1>max ? max=$1 : max} \
1107 END {print max-min, max, min}'))
1111 # In-case of fairly large number of file creation using RR (round-robin)
1112 # there can be two cases in which deviation will occur than the regular
1113 # RR algo behaviour-
1114 # 1- When rr_alloc does not start right with 'lqr_start_count' reseeded,
1115 # 2- When rr_alloc does not finish with 'lqr_start_count == 0'.
1116 # So the difference of files b/w any 2 OST should not be more than 2.
1117 [[ ${diff_max_min_arr[0]} -le 2 ]] ||
1118 error "Uneven distribution detected: difference between" \
1119 "maximum files per OST (${diff_max_min_arr[1]}) and" \
1120 "minimum files per OST (${diff_max_min_arr[2]}) must not be" \
1125 # fs_test.x is the default name for exe
1126 FS_TEST=${FS_TEST:=$(which fs_test.x 2> /dev/null || true)}
1128 local clients=${CLIENTS:-$(hostname)}
1129 local testdir=$DIR/d0.fs_test
1130 local file=${testdir}/fs_test
1131 fs_test_threads=${fs_test_threads:-2}
1132 fs_test_type=${fs_test_type:-1}
1133 fs_test_nobj=${fs_test_nobj:-10}
1134 fs_test_check=${fs_test_check:-3}
1135 fs_test_strided=${fs_test_strided:-1}
1136 fs_test_touch=${fs_test_touch:-3}
1137 fs_test_supersize=${fs_test_supersize:-1}
1138 fs_test_op=${fs_test_op:-write}
1139 fs_test_barriers=${fs_test_barriers:-bopen,bwrite,bclose}
1140 fs_test_io=${fs_test_io:-mpi}
1141 fs_test_objsize=${fs_test_objsize:-100}
1142 fs_test_objunit=${fs_test_objunit:-1048576} # 1 mb
1143 fs_test_ndirs=${fs_test_ndirs:-80000}
1145 [ x$FS_TEST = x ] && skip "FS_TEST not found"
1147 # Space estimation in bytes
1148 local space=$(df -B 1 -P $dir | tail -n 1 | awk '{ print $4 }')
1149 local total_threads=$((num_clients * fs_test_threads))
1150 echo "+ $fs_test_objsize * $fs_test_objunit * $total_threads "
1151 if [ $((space / 2)) -le \
1152 $((fs_test_objsize * fs_test_objunit * total_threads)) ]; then
1153 fs_test_objsize=$((space / 2 / fs_test_objunit / \
1155 [ $fs_test_objsize -eq 0 ] &&
1156 skip_env "Need free space more than \
1157 $((2 * total_threads * fs_test_objunit)) \
1158 : have $((space / fs_test_objunit))"
1160 echo "(reduced objsize to \
1161 $((fs_test_objsize * fs_test_objunit)) bytes)"
1164 print_opts FS_TEST clients fs_test_threads fs_test_objsize MACHINEFILE
1167 setstripe_getstripe $testdir $fs_test_STRIPEPARAMS
1169 # mpi_run uses mpiuser
1172 # --nodb Turn off the database code at runtime
1173 # -g --target The path to the data file
1174 # -t --type Whether to do N-N (1) or N-1 (2)
1175 # -n --nobj The number of objects written/read by each proc
1176 # -z --size The size of each object
1177 # -d ---num_nn_dirs Number of subdirectories for files
1178 # -C --check Check every byte using argument 3.
1179 # --collective Whether to use collective I/O (for N-1, mpi-io only)
1180 # -s --strided Whether to use a strided pattern (for N-1 only)
1181 # -T --touch Touch every byte using argument 3
1182 # -o --op Whether to read only (read) or write only (write)
1183 # -b --barriers When to barrier.
1184 # -i --io Use POSIX, MPI, or PLFS IO routines (mpi|posix|plfs)
1185 # -S --supersize Specify how many objects per superblock
1187 local cmd="$FS_TEST -nodb -g $file -t $fs_test_type -n $fs_test_nobj \
1188 -z $((fs_test_objsize * fs_test_objunit)) -d $fs_test_ndirs \
1189 -C $fs_test_check -collective -s $fs_test_strided \
1190 -T $fs_test_touch -o $fs_test_op -b $fs_test_barriers \
1191 -i $fs_test_io -S $fs_test_supersize"
1194 mpi_run "-np $((num_clients * fs_test_threads))" $cmd
1197 if [ $rc != 0 ] ; then
1198 error "fs_test failed! $rc"
1204 ior_mdtest_parallel() {
1212 run_mdtest $type || rc2=$?
1213 [[ $rc2 -ne 0 ]] && echo "mdtest failed with error $rc2"
1215 wait $pids || rc1=$?
1216 [[ $rc1 -ne 0 ]] && echo "ior failed with error $rc1"
1218 [[ $rc1 -ne 0 || $rc2 -ne 0 ]] && return 1
1223 FIO=${FIO:=$(which fio 2> /dev/null || true)}
1225 local clients=${CLIENTS:-$(hostname)}
1226 local fio_jobNum=${fio_jobNum:-4}
1227 local fio_jobFile=${fio_jobFile:-$TMP/fiojobfile.$(date +%s)}
1228 local fio_bs=${fio_bs:-1}
1229 local testdir=$DIR/d0.fio
1230 local file=${testdir}/fio
1232 local propagate=false
1234 [ "$SLOW" = "no" ] || runtime=600
1236 [ x$FIO = x ] && skip_env "FIO not found"
1239 setstripe_getstripe $testdir $fio_STRIPEPARAMS
1241 # use fio job file if exists,
1242 # create a simple one if missing
1243 if ! [ -f $fio_jobFile ]; then
1244 cat >> $fio_jobFile <<EOF
1250 filename=${file}_\$(hostname)
1252 # bs size increased by $i for each job
1253 for ((i=1; i<=fio_jobNum; i++)); do
1254 cat >> $fio_jobFile <<EOF
1257 bs=$(( fio_bs * i ))m
1260 # job file is created, should be propagated to all clients
1265 # propagate the job file if not all clients have it yet or
1266 # if the job file was created during the test run
1267 if ! do_nodesv $clients " [ -f $fio_jobFile ] " ||
1269 local cfg=$(cat $fio_jobFile)
1270 do_nodes $clients "echo \\\"$cfg\\\" > ${fio_jobFile}" ||
1271 error "job file $fio_jobFile is not propagated"
1272 do_nodesv $clients "cat ${fio_jobFile}"
1275 cmd="$FIO $fio_jobFile"
1278 log "clients: $clients $cmd"
1281 do_nodesv $clients "$cmd "
1284 [ $rc = 0 ] || error "fio failed: $rc"
1289 XDD=${XDD:=$(which xdd 2> /dev/null || true)}
1291 local clients=${CLIENTS:-$(hostname)}
1292 local testdir=$DIR/d0.xdd
1293 xdd_queuedepth=${xdd_queuedepth:-4}
1294 xdd_blocksize=${xdd_blocksize:-512}
1295 xdd_reqsize=${xdd_reqsize:-128}
1296 xdd_mbytes=${xdd_mbytes:-100}
1297 xdd_passes=${xdd_passes:-40}
1298 xdd_rwratio=${xdd_rwratio:-0}
1299 xdd_ntargets=${xdd_ntargets:-6}
1300 local xdd_custom_params=${xdd_custom_params:-"-dio -stoponerror \
1301 -maxpri -minall -noproclock -nomemlock"}
1303 [ x$XDD = x ] && skip "XDD not found"
1305 print_opts XDD clients xdd_queuedepth xdd_blocksize xdd_reqsize \
1306 xdd_mbytes xdd_passes xdd_rwratio
1309 setstripe_getstripe $testdir $xdd_STRIPEPARAMS
1312 # Target files creates based on the given number of targets
1313 for (( i=0; i < $xdd_ntargets; i++ ))
1315 files+="${testdir}/xdd"$i" "
1318 # -targets specifies the devices or files to perform operation
1319 # -reqsize number of 'blocks' per operation
1320 # -mbytes number of 1024*1024-byte blocks to transfer
1321 # -blocksize size of a single 'block'
1322 # -passes number of times to read mbytes
1323 # -queuedepth number of commands to queue on the target
1324 # -rwratio percentage of read to write operations
1325 # -verbose will print out statistics on each pass
1327 local cmd="$XDD -targets $xdd_ntargets $files -reqsize $xdd_reqsize \
1328 -mbytes $xdd_mbytes -blocksize $xdd_blocksize \
1329 -passes $xdd_passes -queuedepth $xdd_queuedepth \
1330 -rwratio $xdd_rwratio -verbose $xdd_custom_params"
1334 do_nodesv $clients "$cmd "
1337 [ $rc = 0 ] || error "xdd failed: $rc"