3 # Simple function used by run_*.sh scripts
9 if [ -z "${!name}" ]; then
10 echo "$0: $name must be set"
14 [ $failed ] && exit 1 || true
17 # lrepl - Lustre test Read-Eval-Print Loop.
19 # This function implements a REPL for the Lustre test framework. It
20 # doesn't exec an actual shell because the user may want to inspect
21 # variables and use functions from the test framework.
28 This is an interactive read-eval-print loop interactive shell
29 simulation that you can use to debug failing tests. You can
30 enter most bash command lines (see notes below).
32 Use this REPL to inspect variables, set them, call test
33 framework shell functions, etcetera.
35 'exit' or EOF to exit this shell.
37 set \$retcode to 0 to cause the assertion failure that
38 triggered this REPL to be ignored.
41 do_facet ost1 lctl get_param ost.*.ost.threads_*
42 do_rpc_nodes \$OSTNODES unload_modules
45 All but the last line of multi-line statements or blocks
46 must end in a backslash.
48 "Here documents" are not supported.
50 History is not supported, but command-line editing is.
54 # Prompt escapes don't work in read -p, sadly.
55 prompt=":${TESTNAME:-UNKNOWN}:$(uname -n):$(basename $PWD)% "
57 # We use read -r to get close to a shell experience
58 while read -e -r -p "$prompt" rawline; do
61 # Don't want to exit-exit, just exit the REPL
63 # We need to handle continuations, and read -r doesn't do
64 # that for us. Yet we need read -r.
66 # We also use case/esac to compare lines read to "*\\"
67 # because [ "$line" = *\\ ] and variants of that don't work.
69 while read -e -r -p '> ' rawline
71 line="$line"$'\n'"$rawline"
73 # We could check for here documents by matching
74 # against *<<*, but who cares.
87 # Finally! Time to eval.
91 echo $'\n\tExiting interactive shell...\n'
95 # lassert - Lustre test framework assert
97 # Arguments: failure code, failure message, expression/statement
99 # lassert evaluates the expression given, and, if false, calls
100 # error() to trigger test failure. If REPL_ON_LASSERT is true then
101 # lassert will call lrepl() to give the user an interactive shell.
102 # If the REPL sets retcode=0 then the assertion failure will be
109 echo "checking $* ($(eval echo \""$*"\"))..."
110 eval "$@" && return 0;
112 if ${REPL_ON_LASSERT:-false}; then
113 echo "Assertion $retcode failed: $*
114 (expanded: $(eval echo \""$*"\")) $msg"
118 error "Assertion $retcode failed: $*
119 (expanded: $(eval echo \""$*"\")) $msg"
123 # setmodopts- set module options for subsequent calls to load_modules
125 # Usage: setmodopts module_name new_value [var_in_which_to_save_old_value]
126 # setmodopts -a module_name new_value [var_in_which_to_save_old_value]
128 # In the second usage the new value is appended to the old.
132 if [ "$1" = -a ]; then
137 local _var=MODOPTS_$1
142 # Dynamic naming of variables is a pain in bash. In ksh93 we could
143 # write "nameref opts_var=${modname}_MODOPTS" then assign directly
144 # to opts_var. Associative arrays would also help, alternatively.
145 # Alas, we're stuck with eval until all distros move to a more recent
146 # version of bash. Fortunately we don't need to eval unset and export.
148 if [ -z "$_newvalue" ]; then
154 $_append && _newvalue="$_oldvalue $_newvalue"
155 export $_var="$_newvalue"
156 echo setmodopts: ${_var}=${_newvalue}
158 [ -n "$_savevar" ] && eval $_savevar=\""$_oldvalue"\"
161 echoerr () { echo "$@" 1>&2 ; }
164 echoerr "$(date +'%F %H:%M:%S'): client load was signaled to terminate"
166 local PGID=$(ps -eo "%c %p %r" | awk "/ $PPID / {print \$3}")
173 local mpirun="$MPIRUN $MPIRUN_OPTIONS"
174 local command="$mpirun $@"
175 local mpilog=$TMP/mpi.log
178 if [ -n "$MPI_USER" -a "$MPI_USER" != root -a -n "$mpirun" ]; then
179 echo "+ chmod 0777 $MOUNT"
181 command="su $MPI_USER bash -c \"$command \""
186 eval $command 2>&1 | tee $mpilog || true
189 if [ $rc -eq 0 ] && grep -q "p4_error:" $mpilog ; then
198 for i in ${1//,/ }; do
199 if [ "$list" = "" ]; then
202 list="$list$escape $i@$NETTYPE"
208 # FIXME: all setup/cleanup can be done without rpc.sh
211 [ x$1 = x--verbose ] && verbose=true
213 export LST_SESSION=`$LST show_session 2>/dev/null | awk '{print $5}'`
214 [ "$LST_SESSION" == "" ] && return
223 lst_session_cleanup_all () {
224 local list=$(comma_list $(nodes_list))
225 do_rpc_nodes $list lst_end_session
229 lsmod | grep -q lnet_selftest && \
230 rmmod lnet_selftest > /dev/null 2>&1 || true
234 local list=$(comma_list $(nodes_list))
236 # lst end_session needs to be executed only locally
237 # i.e. on node where lst new_session was called
238 lst_end_session --verbose
239 do_rpc_nodes $list lst_cleanup
243 load_module lnet_selftest
247 local list=$(comma_list $(nodes_list))
248 do_rpc_nodes $list lst_setup
254 # Passed a single argument, strips everything off following
255 # and includes the first period.
256 # client-20.lab.whamcloud.com becomes client-20
258 echo $(sed 's/\..*//' <<< $1)
264 # Find remote nodename, stripped of any domain, etc.
265 # 'hostname -s' is easy, but not implemented on all systems
267 local rname=$(do_node $1 "uname -n" || echo -1)
268 if [[ "$rname" = "-1" ]]; then
271 echo $(short_hostname $rname)
281 echo "${var}=${!var}"
283 [ -e $MACHINEFILE ] && cat $MACHINEFILE
287 [ "$(stat -f -c %T $1)" = "lustre" ]
290 setstripe_getstripe () {
295 is_lustre $file || return 0
297 if [ -n "$params" ]; then
298 $LFS setstripe $params $file ||
299 error "setstripe $params failed"
301 $LFS getstripe $file ||
302 error "getstripe $file failed"
307 local cbench_DIR=${cbench_DIR:-""}
308 local cbench_IDIRS=${cbench_IDIRS:-2}
309 local cbench_RUNS=${cbench_RUNS:-2}
311 print_opts cbench_DIR cbench_IDIRS cbench_RUNS
313 [ x$cbench_DIR = x ] &&
314 skip_env "compilebench not found"
316 [ -e $cbench_DIR/compilebench ] ||
317 skip_env "No compilebench build"
320 # compile dir kernel-0 ~1GB
321 # required space ~1GB * cbench_IDIRS
322 local space=$(df -P $dir | tail -n 1 | awk '{ print $4 }')
323 if [[ $space -le $((1024 * 1024 * cbench_IDIRS)) ]]; then
324 cbench_IDIRS=$((space / 1024 / 1024))
325 [[ $cbench_IDIRS -eq 0 ]] &&
326 skip_env "Need free space at least 1GB, have $space"
328 echo "reducing initial dirs to $cbench_IDIRS"
330 echo "free space = $space KB"
333 # t-f _base needs to be modifyed to set properly tdir
334 # for new "test_foo" functions names
335 # local testdir=$DIR/$tdir
336 local testdir=$dir/d0.compilebench.$$
337 test_mkdir -p $testdir
338 setstripe_getstripe $testdir $cbench_STRIPEPARAMS
342 local cmd="./compilebench -D $testdir -i $cbench_IDIRS \
343 -r $cbench_RUNS --makej"
352 [ $rc = 0 ] || error "compilebench failed: $rc"
358 local mntpt=${2:-$MOUNT}
359 METABENCH=${METABENCH:-$(which metabench 2> /dev/null || true)}
360 mbench_NFILES=${mbench_NFILES:-30400}
362 mbench_THREADS=${mbench_THREADS:-4}
363 mbench_OPTIONS=${mbench_OPTIONS:-}
364 mbench_CLEANUP=${mbench_CLEANUP:-true}
366 [ x$METABENCH = x ] && skip_env "metabench not found"
368 print_opts METABENCH clients mbench_NFILES mbench_THREADS
370 local testdir=$dir/d0.metabench
371 test_mkdir -p $testdir
372 setstripe_getstripe $testdir $mbench_STRIPEPARAMS
374 # mpi_run uses mpiuser
377 # -C Run the file creation tests. Creates zero byte files.
378 # -S Run the file stat tests.
379 # -c nfile Number of files to be used in each test.
380 # -k Cleanup files when finished.
381 local cmd="$METABENCH -w $testdir -c $mbench_NFILES -C -S $mbench_OPTIONS"
384 # find out if we need to use srun by checking $SRUN_PARTITION
385 if [ "$SRUN_PARTITION" ]; then
386 $SRUN $SRUN_OPTIONS -D $testdir -w $clients -N $num_clients \
387 -n $((num_clients * mbench_THREADS)) \
388 -p $SRUN_PARTITION -- $cmd
390 mpi_run ${MACHINEFILE_OPTION} ${MACHINEFILE} \
391 -np $((num_clients * $mbench_THREADS)) $cmd
395 if [ $rc != 0 ] ; then
396 error "metabench failed! $rc"
399 if $mbench_CLEANUP; then
402 mv $dir/d0.metabench $mntpt/_xxx.$(date +%s).d0.metabench
407 SIMUL=${SIMUL:=$(which simul 2> /dev/null || true)}
408 [ x$SIMUL = x ] && skip_env "simul not found"
409 [ "$NFSCLIENT" ] && skip "skipped for NFSCLIENT mode"
412 simul_THREADS=${simul_THREADS:-2}
413 simul_REP=${simul_REP:-20}
416 # Need space estimation here.
418 print_opts SIMUL clients simul_REP simul_THREADS
420 local testdir=$DIR/d0.simul
422 setstripe_getstripe $testdir $simul_STRIPEPARAMS
424 # mpi_run uses mpiuser
427 # -n # : repeat each test # times
428 # -N # : repeat the entire set of tests # times
430 local cmd="$SIMUL -d $testdir -n $simul_REP -N $simul_REP"
433 # find out if we need to use srun by checking $SRUN_PARTITION
434 if [ "$SRUN_PARTITION" ]; then
435 $SRUN $SRUN_OPTIONS -D $testdir -w $clients -N $num_clients \
436 -n $((num_clients * simul_THREADS)) -p $SRUN_PARTITION \
439 mpi_run ${MACHINEFILE_OPTION} ${MACHINEFILE} \
440 -np $((num_clients * simul_THREADS)) $cmd
445 if [ $rc != 0 ] ; then
446 error "simul failed! $rc"
452 MDTEST=${MDTEST:=$(which mdtest 2> /dev/null || true)}
453 [ x$MDTEST = x ] && skip_env "mdtest not found"
454 [ "$NFSCLIENT" ] && skip "skipped for NFSCLIENT mode"
457 mdtest_THREADS=${mdtest_THREADS:-2}
458 mdtest_nFiles=${mdtest_nFiles:-"100000"}
459 # We devide the files by number of core
460 mdtest_nFiles=$((mdtest_nFiles/mdtest_THREADS/num_clients))
461 mdtest_iteration=${mdtest_iteration:-1}
462 local mdtest_custom_params=${mdtest_custom_params:-""}
463 local type=${1:-"ssf"}
465 local mdtest_Nmntp=${mdtest_Nmntp:-1}
467 if [ $type = "ssf" ] && [ $mdtest_Nmntp -ne 1 ]; then
468 skip "shared directory mode is not compatible" \
469 "with multiple directory paths"
473 # Need space estimation here.
475 print_opts MDTEST mdtest_iteration mdtest_THREADS mdtest_nFiles
477 local testdir=$DIR/d0.mdtest
479 setstripe_getstripe $testdir $mdtest_STRIPEPARAMS
482 for ((i=1; i<mdtest_Nmntp; i++)); do
483 zconf_mount_clients $clients $MOUNT$i "$mntopts" ||
484 error_exit "Failed $clients on $MOUNT$i"
485 local dir=$DIR$i/d0.mdtest$i
487 setstripe_getstripe $dir $mdtest_SETSTRIPEPARAMS
489 testdir="$testdir@$dir"
491 # mpi_run uses mpiuser
493 # -i # : repeat each test # times
495 # -n # : number of file/dir to create/stat/remove
496 # -u : each process create/stat/remove individually
498 local cmd="$MDTEST -d $testdir -i $mdtest_iteration \
499 -n $mdtest_nFiles $mdtest_custom_params"
501 [ $type = "fpp" ] && cmd="$cmd -u"
504 # find out if we need to use srun by checking $SRUN_PARTITION
505 if [ "$SRUN_PARTITION" ]; then
506 $SRUN $SRUN_OPTIONS -D $testdir -w $clients -N $num_clients \
507 -n $((num_clients * mdtest_THREADS)) \
508 -p $SRUN_PARTITION -- $cmd
510 mpi_run ${MACHINEFILE_OPTION} ${MACHINEFILE} \
511 -np $((num_clients * mdtest_THREADS)) $cmd
516 if [ $rc != 0 ] ; then
517 error "mdtest failed! $rc"
520 for ((i=1; i<mdtest_Nmntp; i++)); do
521 local dir=$DIR$i/d0.mdtest$i
523 zconf_umount_clients $clients $MOUNT$i ||
524 error_exit "Failed umount $MOUNT$i on $clients"
530 cnt_DIR=${cnt_DIR:-""}
531 cnt_NRUN=${cnt_NRUN:-10}
533 print_opts cnt_DIR cnt_NRUN
535 [ x$cnt_DIR = x ] && skip_env "connectathon dir not found"
536 [ -e $cnt_DIR/runtests ] || skip_env "No connectathon runtests found"
539 # "special" tests create a 30 MB file + misc. small files
540 # required space ~40 MB
541 local space=$(df -P $dir | tail -n 1 | awk '{ print $4 }')
542 if [[ $space -le $((1024 * 40)) ]]; then
543 skip_env "Need free space at least 40MB, have $space KB"
545 echo "free space = $space KB"
547 local testdir=$dir/d0.connectathon
548 test_mkdir -p $testdir
549 setstripe_getstripe $testdir $cnt_STRIPEPARAMS
555 # To run connectathon:
556 # runtests [-a|-b|-g|-s|-l] [-f|-n|-t] [-N numpasses] [test-directory]
558 # One of the following test types
563 # -a all of the above
565 # -f a quick functional test
566 # -n suppress directory operations (mkdir and rmdir)
567 # -t run with time statistics (default for basic tests)
569 # -N numpasses - specifies the number of times to run
570 # the tests. Optional.
573 # Include lock tests unless we're running on nfsv4
574 local fstype=$(df -TP $testdir | awk 'NR==2 {print $2}')
575 echo "$testdir: $fstype"
576 if [[ $fstype != "nfs4" ]]; then
580 for test in $tests; do
581 local cmd="sh ./runtests -N $cnt_NRUN $test -f $testdir"
587 [ $rc = 0 ] || error "connectathon failed: $rc"
595 local type=${1:="ssf"}
597 local testdir=$dir/d0.ior.$type
598 local nfs_srvmntpt=$3
600 if [ "$NFSCLIENT" ]; then
601 [[ -n $nfs_srvmntpt ]] ||
602 { error "NFSCLIENT mode, but nfs exported dir"\
603 "is not set!" && return 1; }
606 IOR=${IOR:-$(which IOR 2> /dev/null || true)}
607 [ x$IOR = x ] && skip_env "IOR not found"
610 ior_THREADS=${ior_THREADS:-2}
611 ior_iteration=${ior_iteration:-1}
612 ior_blockSize=${ior_blockSize:-6}
613 ior_blockUnit=${ior_blockUnit:-M} # K, M, G
614 ior_xferSize=${ior_xferSize:-1M}
615 ior_type=${ior_type:-POSIX}
616 ior_DURATION=${ior_DURATION:-30} # minutes
617 ior_CLEANUP=${ior_CLEANUP:-true}
619 case ${ior_blockUnit} in
621 multiplier=$((1024 * 1024 * 1024))
624 multiplier=$((1024 * 1024))
629 *) error "Incorrect block unit should be one of [KMG]"
633 # calculate the space in bytes
634 local space=$(df -B 1 -P $dir | tail -n 1 | awk '{ print $4 }')
635 local total_threads=$((num_clients * ior_THREADS))
636 echo "+ $ior_blockSize * $multiplier * $total_threads "
637 if [ $((space / 2)) -le \
638 $((ior_blockSize * multiplier * total_threads)) ]; then
639 ior_blockSize=$((space / 2 / multiplier / total_threads))
640 [ $ior_blockSize -eq 0 ] &&
641 skip_env "Need free space more than $((2 * total_threads)) \
642 ${ior_blockUnit}: have $((space / multiplier))"
644 echo "(reduced blockSize to $ior_blockSize \
645 ${ior_blockUnit} bytes)"
648 print_opts IOR ior_THREADS ior_DURATION MACHINEFILE
650 test_mkdir -p $testdir
652 # mpi_run uses mpiuser
654 [[ "$ior_stripe_params" && -z "$ior_STRIPEPARAMS" ]] &&
655 ior_STRIPEPARAMS="$ior_stripe_params" &&
656 echo "got deprecated ior_stripe_params,"\
657 "use ior_STRIPEPARAMS instead"
658 setstripe_getstripe $testdir $ior_STRIPEPARAMS
662 # contiguous bytes to write per task (e.g.: 8, 4K, 2M, 1G)"
664 # -t N transferSize -- size of transfer in bytes (e.g.: 8, 4K, 2M, 1G)"
665 # -w writeFile -- write file"
666 # -r readFile -- read existing file"
667 # -W checkWrite -- check read after write"
668 # -C reorderTasks -- changes task ordering to n+1 ordering for readback
669 # -T maxTimeDuration -- max time in minutes to run tests"
670 # -k keepFile -- keep testFile(s) on program exit
673 if [ -n "$ior_custom_params" ]; then
674 cmd="$IOR -o $testdir/iorData $ior_custom_params"
676 cmd="$IOR -a $ior_type -b ${ior_blockSize}${ior_blockUnit} \
677 -o $testdir/iorData -t $ior_xferSize -v -C -w -r -W \
678 -i $ior_iteration -T $ior_DURATION -k"
681 [ $type = "fpp" ] && cmd="$cmd -F"
684 # find out if we need to use srun by checking $SRUN_PARTITION
685 if [ "$SRUN_PARTITION" ]; then
686 $SRUN $SRUN_OPTIONS -D $testdir -w $clients -N $num_clients \
687 -n $((num_clients * ior_THREADS)) -p $SRUN_PARTITION \
690 mpi_ior_custom_threads=${mpi_ior_custom_threads:-"$((num_clients * ior_THREADS))"}
691 mpi_run ${MACHINEFILE_OPTION} ${MACHINEFILE} \
692 -np $mpi_ior_custom_threads $cmd
697 if [ $rc != 0 ] ; then
698 error "ior failed! $rc"
700 $ior_CLEANUP && rm -rf $testdir || true
704 MIB=${MIB:=$(which mib 2> /dev/null || true)}
705 [ "$NFSCLIENT" ] && skip "skipped for NFSCLIENT mode"
706 [ x$MIB = x ] && skip_env "MIB not found"
709 mib_THREADS=${mib_THREADS:-2}
710 mib_xferSize=${mib_xferSize:-1m}
711 mib_xferLimit=${mib_xferLimit:-5000}
712 mib_timeLimit=${mib_timeLimit:-300}
713 mib_STRIPEPARAMS=${mib_STRIPEPARAMS:-"-c -1"}
715 print_opts MIB mib_THREADS mib_xferSize mib_xferLimit mib_timeLimit \
718 local testdir=$DIR/d0.mib
720 setstripe_getstripe $testdir $mib_STRIPEPARAMS
722 # mpi_run uses mpiuser
726 # -I Show intermediate values in output
727 # -H Show headers in output
728 # -L Do not issue new system calls after this many seconds
729 # -s Use system calls of this size
731 # -l Issue no more than this many system calls
732 local cmd="$MIB -t $testdir -s $mib_xferSize -l $mib_xferLimit \
733 -L $mib_timeLimit -HI -p mib.$(date +%Y%m%d%H%M%S)"
736 # find out if we need to use srun by checking $SRUN_PARTITION
737 if [ "$SRUN_PARTITION" ]; then
738 $SRUN $SRUN_OPTIONS -D $testdir -w $clients -N $num_clients \
739 -n $((num_clients * mib_THREADS)) -p $SRUN_PARTITION \
742 mpi_run ${MACHINEFILE_OPTION} ${MACHINEFILE} \
743 -np $((num_clients * mib_THREADS)) $cmd
748 if [ $rc != 0 ] ; then
749 error "mib failed! $rc"
755 CASC_RW=${CASC_RW:-$(which cascading_rw 2> /dev/null || true)}
756 [ x$CASC_RW = x ] && skip_env "cascading_rw not found"
757 [ "$NFSCLIENT" ] && skip "skipped for NFSCLIENT mode"
760 casc_THREADS=${casc_THREADS:-2}
761 casc_REP=${casc_REP:-300}
764 # Need space estimation here.
766 print_opts CASC_RW clients casc_THREADS casc_REP MACHINEFILE
768 local testdir=$DIR/d0.cascading_rw
770 setstripe_getstripe $testdir $casc_STRIPEPARAMS
772 # mpi_run uses mpiuser
776 # -n: repeat test # times
778 local cmd="$CASC_RW -g -d $testdir -n $casc_REP"
781 mpi_run ${MACHINEFILE_OPTION} ${MACHINEFILE} \
782 -np $((num_clients * $casc_THREADS)) $cmd
786 if [ $rc != 0 ] ; then
787 error "cascading_rw failed! $rc"
792 run_write_append_truncate() {
793 [ "$NFSCLIENT" ] && skip "skipped for NFSCLIENT mode"
794 # location is lustre/tests dir
795 if ! which write_append_truncate > /dev/null 2>&1 ; then
796 skip_env "write_append_truncate not found"
800 write_THREADS=${write_THREADS:-8}
801 write_REP=${write_REP:-10000}
804 # Need space estimation here.
806 local testdir=$DIR/d0.write_append_truncate
807 local file=$testdir/f0.wat
809 print_opts clients write_REP write_THREADS MACHINEFILE
812 # mpi_run uses mpiuser
813 setstripe_getstripe $testdir $write_STRIPEPARAMS
817 local cmd="write_append_truncate -n $write_REP $file"
820 mpi_run ${MACHINEFILE_OPTION} ${MACHINEFILE} \
821 -np $((num_clients * $write_THREADS)) $cmd
825 if [ $rc != 0 ] ; then
826 error "write_append_truncate failed! $rc"
832 run_write_disjoint() {
833 WRITE_DISJOINT=${WRITE_DISJOINT:-$(which write_disjoint 2> /dev/null ||
835 [ x$WRITE_DISJOINT = x ] && skip_env "write_disjoint not found"
836 [ "$NFSCLIENT" ] && skip "skipped for NFSCLIENT mode"
839 wdisjoint_THREADS=${wdisjoint_THREADS:-4}
840 wdisjoint_REP=${wdisjoint_REP:-10000}
844 # Need space estimation here.
846 print_opts WRITE_DISJOINT clients wdisjoint_THREADS wdisjoint_REP \
848 local testdir=$DIR/d0.write_disjoint
850 setstripe_getstripe $testdir $wdisjoint_STRIPEPARAMS
852 # mpi_run uses mpiuser
855 local cmd="$WRITE_DISJOINT -f $testdir/file -n $wdisjoint_REP -m \
859 mpi_run ${MACHINEFILE_OPTION} ${MACHINEFILE} \
860 -np $((num_clients * $wdisjoint_THREADS)) $cmd
864 if [ $rc != 0 ] ; then
865 error "write_disjoint failed! $rc"
870 run_parallel_grouplock() {
871 PARALLEL_GROUPLOCK=${PARALLEL_GROUPLOCK:-$(which parallel_grouplock \
872 2> /dev/null || true)}
874 [ x$PARALLEL_GROUPLOCK = x ] && skip "PARALLEL_GROUPLOCK not found"
875 [ "$NFSCLIENT" ] && skip "skipped for NFSCLIENT mode"
877 parallel_grouplock_MINTASKS=${parallel_grouplock_MINTASKS:-5}
879 print_opts clients parallel_grouplock_MINTASKS MACHINEFILE
881 local testdir=$DIR/d0.parallel_grouplock
883 setstripe_getstripe $testdir $parallel_grouplock_STRIPEPARAMS
885 # mpi_run uses mpiuser
891 for i in $(seq 12); do
893 local cmd="$PARALLEL_GROUPLOCK -g -v -d $testdir $subtest"
896 mpi_run ${MACHINEFILE_OPTION} ${MACHINEFILE} \
897 -np $parallel_grouplock_MINTASKS $cmd
899 if [ $rc != 0 ] ; then
900 error_noexit "parallel_grouplock subtests $subtest " \
903 echo "parallel_grouplock subtests $subtest PASS"
905 let status=$((status + rc))
906 # clear debug to collect one log per one test
907 do_nodes $(comma_list $(nodes_list)) lctl clear
909 [ $status -eq 0 ] || error "parallel_grouplock status: $status"
913 cleanup_statahead () {
920 for i in $(seq 0 $num_mntpts);do
921 zconf_umount_clients $clients ${mntpt_root}$i ||
922 error_exit "Failed to umount lustre on ${mntpt_root}$i"
927 if [[ -n $NFSCLIENT ]]; then
928 skip "Statahead testing is not supported on NFS clients."
930 [ x$MDSRATE = x ] && skip_env "mdsrate not found"
932 statahead_NUMMNTPTS=${statahead_NUMMNTPTS:-5}
933 statahead_NUMFILES=${statahead_NUMFILES:-500000}
935 print_opts MDSRATE clients statahead_NUMMNTPTS statahead_NUMFILES
939 # do not use default "d[0-9]*" dir name
940 # to avoid of rm $statahead_NUMFILES (500k) files in t-f cleanup
942 local testdir=$DIR/$dir
944 # cleanup only if dir exists
945 # cleanup only $statahead_NUMFILES number of files
946 # ignore the other files created by someone else
948 mdsrate_cleanup $((num_clients * 32)) $MACHINEFILE \
949 $statahead_NUMFILES $testdir 'f%%d' --ignore
952 setstripe_getstripe $testdir $statahead_STRIPEPARAMS
954 # mpi_run uses mpiuser
957 local num_files=$statahead_NUMFILES
959 local IFree=$(inodes_available)
960 if [ $IFree -lt $num_files ]; then
966 local cmd1="${MDSRATE} ${MDSRATE_DEBUG} --mknod --dir $testdir"
967 local cmd2="--nfiles $num_files --filefmt 'f%%d'"
968 local cmd="$cmd1 $cmd2"
971 mpi_run ${MACHINEFILE_OPTION} ${MACHINEFILE} \
972 -np $((num_clients * 32)) $cmd
976 if [ $rc != 0 ] ; then
977 error "mdsrate failed to create $rc"
981 local num_mntpts=$statahead_NUMMNTPTS
982 local mntpt_root=$TMP/mntpt/lustre
983 local mntopts=$MNTOPTSTATAHEAD
985 echo "Mounting $num_mntpts lustre clients starts on $clients"
986 trap "cleanup_statahead $clients $mntpt_root $num_mntpts" EXIT ERR
987 for i in $(seq 0 $num_mntpts); do
988 zconf_mount_clients $clients ${mntpt_root}$i "$mntopts" ||
989 error_exit "Failed to mount lustre on ${mntpt_root}$i on $clients"
992 do_rpc_nodes $clients cancel_lru_locks mdc
994 do_rpc_nodes $clients do_ls $mntpt_root $num_mntpts $dir
996 mdsrate_cleanup $((num_clients * 32)) $MACHINEFILE \
997 $num_files $testdir 'f%%d' --ignore
999 # use rm instead of rmdir because of
1000 # testdir could contain the files created by someone else,
1001 # or by previous run where is num_files prev > num_files current
1003 cleanup_statahead $clients $mntpt_root $num_mntpts
1006 cleanup_rr_alloc () {
1009 local mntpt_root="$2"
1010 local rr_alloc_MNTPTS="$3"
1011 local mntpt_dir=$(dirname ${mntpt_root})
1013 for i in $(seq 0 $((rr_alloc_MNTPTS - 1))); do
1014 zconf_umount_clients $clients ${mntpt_root}$i ||
1015 error_exit "Failed to umount lustre on ${mntpt_root}$i"
1017 do_nodes $clients "rm -rf $mntpt_dir"
1021 remote_mds_nodsh && skip "remote MDS with nodsh"
1023 echo "===Test gives more reproduction percentage if number of "
1024 echo " client and ost are more. Test with 44 or more clients "
1025 echo " and 73 or more OSTs gives 100% reproduction rate=="
1027 RR_ALLOC=${RR_ALLOC:-$(which rr_alloc 2> /dev/null || true)}
1028 [ x$RR_ALLOC = x ] && skip_env "rr_alloc not found"
1029 declare -a diff_max_min_arr
1031 local qos_prec_objs="${TMP}/qos_and_precreated_objects"
1032 local rr_alloc_NFILES=${rr_alloc_NFILES:-555}
1033 local rr_alloc_MNTPTS=${rr_alloc_MNTPTS:-11}
1034 local total_MNTPTS=$((rr_alloc_MNTPTS * num_clients))
1035 local mntpt_root="${TMP}/rr_alloc_mntpt/lustre"
1036 test_mkdir $DIR/$tdir
1037 setstripe_getstripe $DIR/$tdir $rr_alloc_STRIPEPARAMS
1039 chmod 0777 $DIR/$tdir
1041 trap "cleanup_rr_alloc $clients $mntpt_root $rr_alloc_MNTPTS" EXIT ERR
1042 for i in $(seq 0 $((rr_alloc_MNTPTS - 1))); do
1043 zconf_mount_clients $clients ${mntpt_root}$i $MOUNT_OPTS ||
1044 error_exit "Failed to mount lustre on ${mntpt_root}$i $clients"
1047 # Save mdt values, set threshold to 100% i.e always Round Robin,
1048 # restore the saved values again after creating files...
1049 save_lustre_params mds1 \
1050 "lod.$FSNAME-MDT0000*.qos_threshold_rr" > $qos_prec_objs
1051 save_lustre_params mds1 \
1052 "osp.$FSNAME-OST*-osc-MDT0000.create_count" >> $qos_prec_objs
1054 local old_create_count=$(grep -e "create_count" $qos_prec_objs |
1055 cut -d'=' -f 2 | sort -nr | head -n1)
1057 # Make sure that every osp has enough precreated objects for the file
1060 # create_count is always set to the power of 2 only, so if the files
1061 # per OST are not multiple of that then it will be set to nearest
1062 # lower power of 2. So set 'create_count' to the upper power of 2.
1064 # foeo = file on each ost. calc = calculated.
1065 local foeo_calc=$((rr_alloc_NFILES * total_MNTPTS / OSTCOUNT))
1066 local create_count=$((2 * foeo_calc))
1068 # create_count accepted values:
1069 # [OST_MIN_PRECREATE=32, OST_MAX_PRECREATE=20000]
1070 # values exceeding OST_MAX_PRECREATE are lowered to the maximum.
1071 [[ $create_count -lt 32 ]] && create_count=32
1073 for i in $(seq $MDSCOUNT); do
1074 do_facet mds$i "$LCTL set_param -n \
1075 lod.$FSNAME-MDT*.qos_threshold_rr=100 \
1076 osp.$FSNAME-OST*-osc-MDT*.create_count=$create_count"
1079 # Create few temporary files in order to increase the precreated objects
1080 # to a desired value, before starting 'rr_alloc' app. Due to default
1081 # value 32 of precreation count (OST_MIN_PRECREATE=32), precreated
1082 # objects available are 32 initially, these gets exhausted very soon,
1083 # which causes skip of some osps when very large number of files
1084 # is created per OSTs.
1085 createmany -o $DIR/$tdir/foo- $(((old_create_count + 1) * OSTCOUNT)) \
1087 unlinkmany $DIR/$tdir/foo- $(((old_create_count + 1) * OSTCOUNT))
1089 # Check for enough precreated objects... We should not
1090 # fail here because code(osp_precreate.c) also takes care of it.
1091 # So we have good chances of passing test even if this check fails.
1093 for ((ost_idx = 0; ost_idx < $OSTCOUNT; ost_idx++ )); do
1094 (($(precreated_ost_obj_count $mdt_idx $ost_idx) >= foeo_calc))||
1095 echo "Warning: test may fail from too few objs on OST$ost_idx"
1098 local cmd="$RR_ALLOC $mntpt_root/$tdir/f $rr_alloc_NFILES $num_clients"
1100 if [[ $total_MNTPTS -ne 0 ]]; then
1101 # Now start the actual file creation app.
1102 mpi_run "-np $total_MNTPTS" $cmd || return
1104 error "No mount point"
1107 restore_lustre_params < $qos_prec_objs
1108 rm -f $qos_prec_objs
1110 diff_max_min_arr=($($LFS getstripe -r $DIR/$tdir/ |
1111 awk '/lmm_stripe_offset:/ {print $2}' |
1113 awk 'NR==1 {min=max=$1} \
1114 { $1<min ? min=$1:min; $1>max ? max=$1:max} \
1115 END {print max-min, max, min}'))
1117 $LFS find $DIR/$tdir -type f | xargs -n1 -P8 unlink
1120 # In-case of fairly large number of file creation using RR (round-robin)
1121 # there can be two cases in which deviation will occur than the regular
1122 # RR algo behaviour-
1123 # 1- When rr_alloc does not start right with 'lqr_start_count' reseeded,
1124 # 2- When rr_alloc does not finish with 'lqr_start_count == 0'.
1125 # So the difference of files b/w any 2 OST should not be more than 2.
1126 # In some cases it may be more, but shouldn't be > 0.3% of the files.
1127 local max_diff=$((create_count > 600 ? create_count / 300 : 2))
1129 (( ${diff_max_min_arr[0]} <= $max_diff )) || {
1130 $LFS getstripe -r $DIR/$tdir |
1131 awk '/lmm_stripe_offset:/ {print $2}' | sort | uniq -c
1133 error "max/min OST objects (${diff_max_min_arr[1]} : ${diff_max_min_arr[2]}) too different"
1138 # fs_test.x is the default name for exe
1139 FS_TEST=${FS_TEST:=$(which fs_test.x 2> /dev/null || true)}
1141 local clients=${CLIENTS:-$(hostname)}
1142 local testdir=$DIR/d0.fs_test
1143 local file=${testdir}/fs_test
1144 fs_test_threads=${fs_test_threads:-2}
1145 fs_test_type=${fs_test_type:-1}
1146 fs_test_nobj=${fs_test_nobj:-10}
1147 fs_test_check=${fs_test_check:-3}
1148 fs_test_strided=${fs_test_strided:-1}
1149 fs_test_touch=${fs_test_touch:-3}
1150 fs_test_supersize=${fs_test_supersize:-1}
1151 fs_test_op=${fs_test_op:-write}
1152 fs_test_barriers=${fs_test_barriers:-bopen,bwrite,bclose}
1153 fs_test_io=${fs_test_io:-mpi}
1154 fs_test_objsize=${fs_test_objsize:-100}
1155 fs_test_objunit=${fs_test_objunit:-1048576} # 1 mb
1156 fs_test_ndirs=${fs_test_ndirs:-80000}
1158 [ x$FS_TEST = x ] && skip "FS_TEST not found"
1160 # Space estimation in bytes
1161 local space=$(df -B 1 -P $dir | tail -n 1 | awk '{ print $4 }')
1162 local total_threads=$((num_clients * fs_test_threads))
1163 echo "+ $fs_test_objsize * $fs_test_objunit * $total_threads "
1164 if [ $((space / 2)) -le \
1165 $((fs_test_objsize * fs_test_objunit * total_threads)) ]; then
1166 fs_test_objsize=$((space / 2 / fs_test_objunit /
1168 [ $fs_test_objsize -eq 0 ] &&
1169 skip_env "Need free space more than \
1170 $((2 * total_threads * fs_test_objunit)) \
1171 : have $((space / fs_test_objunit))"
1173 echo "(reduced objsize to \
1174 $((fs_test_objsize * fs_test_objunit)) bytes)"
1177 print_opts FS_TEST clients fs_test_threads fs_test_objsize MACHINEFILE
1180 setstripe_getstripe $testdir $fs_test_STRIPEPARAMS
1182 # mpi_run uses mpiuser
1185 # --nodb Turn off the database code at runtime
1186 # -g --target The path to the data file
1187 # -t --type Whether to do N-N (1) or N-1 (2)
1188 # -n --nobj The number of objects written/read by each proc
1189 # -z --size The size of each object
1190 # -d ---num_nn_dirs Number of subdirectories for files
1191 # -C --check Check every byte using argument 3.
1192 # --collective Whether to use collective I/O (for N-1, mpi-io only)
1193 # -s --strided Whether to use a strided pattern (for N-1 only)
1194 # -T --touch Touch every byte using argument 3
1195 # -o --op Whether to read only (read) or write only (write)
1196 # -b --barriers When to barrier.
1197 # -i --io Use POSIX, MPI, or PLFS IO routines (mpi|posix|plfs)
1198 # -S --supersize Specify how many objects per superblock
1200 local cmd="$FS_TEST -nodb -g $file -t $fs_test_type -n $fs_test_nobj \
1201 -z $((fs_test_objsize * fs_test_objunit)) -d $fs_test_ndirs \
1202 -C $fs_test_check -collective -s $fs_test_strided \
1203 -T $fs_test_touch -o $fs_test_op -b $fs_test_barriers \
1204 -i $fs_test_io -S $fs_test_supersize"
1207 mpi_run "-np $((num_clients * fs_test_threads))" $cmd
1210 if [ $rc != 0 ] ; then
1211 error "fs_test failed! $rc"
1217 ior_mdtest_parallel() {
1225 run_mdtest $type || rc2=$?
1226 [[ $rc2 -ne 0 ]] && echo "mdtest failed with error $rc2"
1228 wait $pids || rc1=$?
1229 [[ $rc1 -ne 0 ]] && echo "ior failed with error $rc1"
1231 [[ $rc1 -ne 0 || $rc2 -ne 0 ]] && return 1
1236 FIO=${FIO:=$(which fio 2> /dev/null || true)}
1238 local clients=${CLIENTS:-$(hostname)}
1239 local fio_jobNum=${fio_jobNum:-4}
1240 local fio_jobFile=${fio_jobFile:-$TMP/fiojobfile.$(date +%s)}
1241 local fio_bs=${fio_bs:-1}
1242 local testdir=$DIR/d0.fio
1243 local file=${testdir}/fio
1245 local propagate=false
1247 [ "$SLOW" = "no" ] || runtime=600
1249 [ x$FIO = x ] && skip_env "FIO not found"
1252 setstripe_getstripe $testdir $fio_STRIPEPARAMS
1254 # use fio job file if exists,
1255 # create a simple one if missing
1256 if ! [ -f $fio_jobFile ]; then
1257 cat >> $fio_jobFile <<EOF
1263 filename=${file}_\$(hostname)
1265 # bs size increased by $i for each job
1266 for ((i=1; i<=fio_jobNum; i++)); do
1267 cat >> $fio_jobFile <<EOF
1270 bs=$(( fio_bs * i ))m
1273 # job file is created, should be propagated to all clients
1278 # propagate the job file if not all clients have it yet or
1279 # if the job file was created during the test run
1280 if ! do_nodesv $clients " [ -f $fio_jobFile ] " ||
1282 local cfg=$(cat $fio_jobFile)
1283 do_nodes $clients "echo \\\"$cfg\\\" > ${fio_jobFile}" ||
1284 error "job file $fio_jobFile is not propagated"
1285 do_nodesv $clients "cat ${fio_jobFile}"
1288 cmd="$FIO $fio_jobFile"
1291 log "clients: $clients $cmd"
1294 do_nodesv $clients "$cmd "
1297 [ $rc = 0 ] || error "fio failed: $rc"
1302 XDD=${XDD:=$(which xdd 2> /dev/null || true)}
1304 local clients=${CLIENTS:-$(hostname)}
1305 local testdir=$DIR/d0.xdd
1306 xdd_queuedepth=${xdd_queuedepth:-4}
1307 xdd_blocksize=${xdd_blocksize:-512}
1308 xdd_reqsize=${xdd_reqsize:-128}
1309 xdd_mbytes=${xdd_mbytes:-100}
1310 xdd_passes=${xdd_passes:-40}
1311 xdd_rwratio=${xdd_rwratio:-0}
1312 xdd_ntargets=${xdd_ntargets:-6}
1313 local xdd_custom_params=${xdd_custom_params:-"-dio -stoponerror \
1314 -maxpri -minall -noproclock -nomemlock"}
1316 [ x$XDD = x ] && skip "XDD not found"
1318 print_opts XDD clients xdd_queuedepth xdd_blocksize xdd_reqsize \
1319 xdd_mbytes xdd_passes xdd_rwratio
1322 setstripe_getstripe $testdir $xdd_STRIPEPARAMS
1325 # Target files creates based on the given number of targets
1326 for (( i=0; i < $xdd_ntargets; i++ ))
1328 files+="${testdir}/xdd"$i" "
1331 # -targets specifies the devices or files to perform operation
1332 # -reqsize number of 'blocks' per operation
1333 # -mbytes number of 1024*1024-byte blocks to transfer
1334 # -blocksize size of a single 'block'
1335 # -passes number of times to read mbytes
1336 # -queuedepth number of commands to queue on the target
1337 # -rwratio percentage of read to write operations
1338 # -verbose will print out statistics on each pass
1340 local cmd="$XDD -targets $xdd_ntargets $files -reqsize $xdd_reqsize \
1341 -mbytes $xdd_mbytes -blocksize $xdd_blocksize \
1342 -passes $xdd_passes -queuedepth $xdd_queuedepth \
1343 -rwratio $xdd_rwratio -verbose $xdd_custom_params"
1347 do_nodesv $clients "$cmd "
1350 [ $rc = 0 ] || error "xdd failed: $rc"