3 # Simple function used by run_*.sh scripts
8 if [ -z "${!name}" ]; then
9 echo "$0: $name must be set"
13 [ $failed ] && exit 1 || true
16 # lrepl - Lustre test Read-Eval-Print Loop.
18 # This function implements a REPL for the Lustre test framework. It
19 # doesn't exec an actual shell because the user may want to inspect
20 # variables and use functions from the test framework.
27 This is an interactive read-eval-print loop interactive shell
28 simulation that you can use to debug failing tests. You can
29 enter most bash command lines (see notes below).
31 Use this REPL to inspect variables, set them, call test
32 framework shell functions, etcetera.
34 'exit' or EOF to exit this shell.
36 set \$retcode to 0 to cause the assertion failure that
37 triggered this REPL to be ignored.
40 do_facet ost1 lctl get_param ost.*.ost.threads_*
41 do_rpc_nodes \$OSTNODES unload_modules
44 All but the last line of multi-line statements or blocks
45 must end in a backslash.
47 "Here documents" are not supported.
49 History is not supported, but command-line editing is.
53 # Prompt escapes don't work in read -p, sadly.
54 prompt=":test_${testnum:-UNKNOWN}:$(uname -n):$(basename $PWD)% "
56 # We use read -r to get close to a shell experience
57 while read -e -r -p "$prompt" rawline; do
60 # Don't want to exit-exit, just exit the REPL
62 # We need to handle continuations, and read -r doesn't do
63 # that for us. Yet we need read -r.
65 # We also use case/esac to compare lines read to "*\\"
66 # because [ "$line" = *\\ ] and variants of that don't work.
68 while read -e -r -p '> ' rawline
70 line="$line"$'\n'"$rawline"
72 # We could check for here documents by matching
73 # against *<<*, but who cares.
86 # Finally! Time to eval.
90 echo $'\n\tExiting interactive shell...\n'
94 # lassert - Lustre test framework assert
96 # Arguments: failure code, failure message, expression/statement
98 # lassert evaluates the expression given, and, if false, calls
99 # error() to trigger test failure. If REPL_ON_LASSERT is true then
100 # lassert will call lrepl() to give the user an interactive shell.
101 # If the REPL sets retcode=0 then the assertion failure will be
108 echo "checking $* ($(eval echo \""$*"\"))..."
109 eval "$@" && return 0;
111 if ${REPL_ON_LASSERT:-false}; then
112 echo "Assertion $retcode failed: $* (expanded: $(eval echo \""$*"\"))
117 error "Assertion $retcode failed: $* (expanded: $(eval echo \""$*"\"))
122 # setmodopts- set module options for subsequent calls to load_modules
124 # Usage: setmodopts module_name new_value [var_in_which_to_save_old_value]
125 # setmodopts -a module_name new_value [var_in_which_to_save_old_value]
127 # In the second usage the new value is appended to the old.
131 if [ "$1" = -a ]; then
136 local _var=MODOPTS_$1
141 # Dynamic naming of variables is a pain in bash. In ksh93 we could
142 # write "nameref opts_var=${modname}_MODOPTS" then assign directly
143 # to opts_var. Associative arrays would also help, alternatively.
144 # Alas, we're stuck with eval until all distros move to a more recent
145 # version of bash. Fortunately we don't need to eval unset and export.
147 if [ -z "$_newvalue" ]; then
153 $_append && _newvalue="$_oldvalue $_newvalue"
154 export $_var="$_newvalue"
155 echo setmodopts: ${_var}=${_newvalue}
157 [ -n "$_savevar" ] && eval $_savevar=\""$_oldvalue"\"
160 echoerr () { echo "$@" 1>&2 ; }
163 echoerr "$(date +'%F %H:%M:%S'): client load was signaled to terminate"
165 local PGID=$(ps -eo "%c %p %r" | awk "/ $PPID / {print \$3}")
172 local mpirun="$MPIRUN $MPIRUN_OPTIONS --oversubscribe"
173 local command="$mpirun $@"
174 local mpilog=$TMP/mpi.log
177 if [ -n "$MPI_USER" -a "$MPI_USER" != root -a -n "$mpirun" ]; then
178 echo "+ chmod 0777 $MOUNT"
180 command="su $MPI_USER sh -c \"$command \""
185 eval $command 2>&1 | tee $mpilog || true
188 if [ $rc -eq 0 ] && grep -q "p4_error:" $mpilog ; then
197 for i in ${1//,/ }; do
198 if [ "$list" = "" ]; then
201 list="$list$escape $i@$NETTYPE"
207 # FIXME: all setup/cleanup can be done without rpc.sh
210 [ x$1 = x--verbose ] && verbose=true
212 export LST_SESSION=`$LST show_session 2>/dev/null | awk -F " " '{print $5}'`
213 [ "$LST_SESSION" == "" ] && return
222 lst_session_cleanup_all () {
223 local list=$(comma_list $(nodes_list))
224 do_rpc_nodes $list lst_end_session
228 lsmod | grep -q lnet_selftest && \
229 rmmod lnet_selftest > /dev/null 2>&1 || true
233 local list=$(comma_list $(nodes_list))
235 # lst end_session needs to be executed only locally
236 # i.e. on node where lst new_session was called
237 lst_end_session --verbose
238 do_rpc_nodes $list lst_cleanup
242 load_module lnet_selftest
246 local list=$(comma_list $(nodes_list))
247 do_rpc_nodes $list lst_setup
253 # Passed a single argument, strips everything off following
254 # and includes the first period.
255 # client-20.lab.whamcloud.com becomes client-20
257 echo $(sed 's/\..*//' <<< $1)
263 # Find remote nodename, stripped of any domain, etc.
264 # 'hostname -s' is easy, but not implemented on all systems
266 local rname=$(do_node $1 "uname -n" || echo -1)
267 if [[ "$rname" = "-1" ]]; then
270 echo $(short_hostname $rname)
280 echo "${var}=${!var}"
282 [ -e $MACHINEFILE ] && cat $MACHINEFILE
287 local cbench_DIR=${cbench_DIR:-""}
288 local cbench_IDIRS=${cbench_IDIRS:-2}
289 local cbench_RUNS=${cbench_RUNS:-2}
291 print_opts cbench_DIR cbench_IDIRS cbench_RUNS
293 [ x$cbench_DIR = x ] &&
294 skip_env "compilebench not found"
296 [ -e $cbench_DIR/compilebench ] ||
297 skip_env "No compilebench build"
300 # compile dir kernel-0 ~1GB
301 # required space ~1GB * cbench_IDIRS
302 local space=$(df -P $dir | tail -n 1 | awk '{ print $4 }')
303 if [[ $space -le $((1024 * 1024 * cbench_IDIRS)) ]]; then
304 cbench_IDIRS=$((space / 1024 / 1024))
305 [[ $cbench_IDIRS -eq 0 ]] &&
306 skip_env "Need free space at least 1GB, have $space"
308 echo "reducing initial dirs to $cbench_IDIRS"
310 echo "free space = $space KB"
313 # t-f _base needs to be modifyed to set properly tdir
314 # for new "test_foo" functions names
315 # local testdir=$DIR/$tdir
316 local testdir=$dir/d0.compilebench.$$
321 local cmd="./compilebench -D $testdir -i $cbench_IDIRS \
322 -r $cbench_RUNS --makej"
331 [ $rc = 0 ] || error "compilebench failed: $rc"
337 local mntpt=${2:-$MOUNT}
338 METABENCH=${METABENCH:-$(which metabench 2> /dev/null || true)}
339 mbench_NFILES=${mbench_NFILES:-30400}
341 mbench_THREADS=${mbench_THREADS:-4}
342 mbench_OPTIONS=${mbench_OPTIONS:-}
343 mbench_CLEANUP=${mbench_CLEANUP:-true}
345 [ x$METABENCH = x ] && skip_env "metabench not found"
347 print_opts METABENCH clients mbench_NFILES mbench_THREADS
349 local testdir=$dir/d0.metabench
351 # mpi_run uses mpiuser
354 # -C Run the file creation tests. Creates zero byte files.
355 # -S Run the file stat tests.
356 # -c nfile Number of files to be used in each test.
357 # -k Cleanup files when finished.
358 local cmd="$METABENCH -w $testdir -c $mbench_NFILES -C -S $mbench_OPTIONS"
361 # find out if we need to use srun by checking $SRUN_PARTITION
362 if [ "$SRUN_PARTITION" ]; then
363 $SRUN $SRUN_OPTIONS -D $testdir -w $clients -N $num_clients \
364 -n $((num_clients * mbench_THREADS)) \
365 -p $SRUN_PARTITION -- $cmd
367 mpi_run ${MACHINEFILE_OPTION} ${MACHINEFILE} \
368 -np $((num_clients * $mbench_THREADS)) $cmd
372 if [ $rc != 0 ] ; then
373 error "metabench failed! $rc"
376 if $mbench_CLEANUP; then
379 mv $dir/d0.metabench $mntpt/_xxx.$(date +%s).d0.metabench
384 SIMUL=${SIMUL:=$(which simul 2> /dev/null || true)}
385 [ x$SIMUL = x ] && skip_env "simul not found"
386 [ "$NFSCLIENT" ] && skip "skipped for NFSCLIENT mode"
389 simul_THREADS=${simul_THREADS:-2}
390 simul_REP=${simul_REP:-20}
393 # Need space estimation here.
395 print_opts SIMUL clients simul_REP simul_THREADS
397 local testdir=$DIR/d0.simul
399 # mpi_run uses mpiuser
402 # -n # : repeat each test # times
403 # -N # : repeat the entire set of tests # times
405 local cmd="$SIMUL -d $testdir -n $simul_REP -N $simul_REP"
408 # find out if we need to use srun by checking $SRUN_PARTITION
409 if [ "$SRUN_PARTITION" ]; then
410 $SRUN $SRUN_OPTIONS -D $testdir -w $clients -N $num_clients \
411 -n $((num_clients * simul_THREADS)) -p $SRUN_PARTITION \
414 mpi_run ${MACHINEFILE_OPTION} ${MACHINEFILE} \
415 -np $((num_clients * simul_THREADS)) $cmd
419 if [ $rc != 0 ] ; then
420 error "simul failed! $rc"
426 MDTEST=${MDTEST:=$(which mdtest 2> /dev/null || true)}
427 [ x$MDTEST = x ] && skip_env "mdtest not found"
428 [ "$NFSCLIENT" ] && skip "skipped for NFSCLIENT mode"
431 mdtest_THREADS=${mdtest_THREADS:-2}
432 mdtest_nFiles=${mdtest_nFiles:-"100000"}
433 # We devide the files by number of core
434 mdtest_nFiles=$((mdtest_nFiles/mdtest_THREADS/num_clients))
435 mdtest_iteration=${mdtest_iteration:-1}
436 local mdtest_custom_params=${mdtest_custom_params:-""}
437 local type=${1:-"ssf"}
440 # Need space estimation here.
442 print_opts MDTEST mdtest_iteration mdtest_THREADS mdtest_nFiles
444 local testdir=$DIR/d0.mdtest
446 # mpi_run uses mpiuser
449 # -i # : repeat each test # times
451 # -n # : number of file/dir to create/stat/remove
452 # -u : each process create/stat/remove individually
454 local cmd="$MDTEST -d $testdir -i $mdtest_iteration \
455 -n $mdtest_nFiles $mdtest_custom_params"
457 [ $type = "fpp" ] && cmd="$cmd -u"
460 # find out if we need to use srun by checking $SRUN_PARTITION
461 if [ "$SRUN_PARTITION" ]; then
462 $SRUN $SRUN_OPTIONS -D $testdir -w $clients -N $num_clients \
463 -n $((num_clients * mdtest_THREADS)) \
464 -p $SRUN_PARTITION -- $cmd
466 mpi_run ${MACHINEFILE_OPTION} ${MACHINEFILE} \
467 -np $((num_clients * mdtest_THREADS)) $cmd
471 if [ $rc != 0 ] ; then
472 error "mdtest failed! $rc"
479 cnt_DIR=${cnt_DIR:-""}
480 cnt_NRUN=${cnt_NRUN:-10}
482 print_opts cnt_DIR cnt_NRUN
484 [ x$cnt_DIR = x ] && skip_env "connectathon dir not found"
485 [ -e $cnt_DIR/runtests ] || skip_env "No connectathon runtests found"
488 # "special" tests create a 30 MB file + misc. small files
489 # required space ~40 MB
490 local space=$(df -P $dir | tail -n 1 | awk '{ print $4 }')
491 if [[ $space -le $((1024 * 40)) ]]; then
492 skip_env "Need free space at least 40MB, have $space KB"
494 echo "free space = $space KB"
496 local testdir=$dir/d0.connectathon
503 # To run connectathon:
504 # runtests [-a|-b|-g|-s|-l] [-f|-n|-t] [-N numpasses] [test-directory]
506 # One of the following test types
511 # -a all of the above
513 # -f a quick functional test
514 # -n suppress directory operations (mkdir and rmdir)
515 # -t run with time statistics (default for basic tests)
517 # -N numpasses - specifies the number of times to run
518 # the tests. Optional.
521 # Include lock tests unless we're running on nfsv4
522 local fstype=$(df -TP $testdir | awk 'NR==2 {print $2}')
523 echo "$testdir: $fstype"
524 if [[ $fstype != "nfs4" ]]; then
528 for test in $tests; do
529 local cmd="sh ./runtests -N $cnt_NRUN $test -f $testdir"
535 [ $rc = 0 ] || error "connectathon failed: $rc"
543 local type=${1:="ssf"}
545 local testdir=$dir/d0.ior.$type
546 local nfs_srvmntpt=$3
548 if [ "$NFSCLIENT" ]; then
549 [[ -n $nfs_srvmntpt ]] ||
550 { error "NFSCLIENT mode, but nfs exported dir"\
551 "is not set!" && return 1; }
554 IOR=${IOR:-$(which IOR 2> /dev/null || true)}
555 [ x$IOR = x ] && skip_env "IOR not found"
558 ior_THREADS=${ior_THREADS:-2}
559 ior_iteration=${ior_iteration:-1}
560 ior_blockSize=${ior_blockSize:-6}
561 ior_blockUnit=${ior_blockUnit:-M} # K, M, G
562 ior_xferSize=${ior_xferSize:-1M}
563 ior_type=${ior_type:-POSIX}
564 ior_DURATION=${ior_DURATION:-30} # minutes
566 case ${ior_blockUnit} in
568 multiplier=$((1024 * 1024 * 1024))
571 multiplier=$((1024 * 1024))
576 *) error "Incorrect block unit should be one of [KMG]"
580 # calculate the space in bytes
581 local space=$(df -B 1 -P $dir | tail -n 1 | awk '{ print $4 }')
582 local total_threads=$((num_clients * ior_THREADS))
583 echo "+ $ior_blockSize * $multiplier * $total_threads "
584 if [ $((space / 2)) -le \
585 $((ior_blockSize * multiplier * total_threads)) ]; then
586 ior_blockSize=$((space / 2 / multiplier / total_threads))
587 [ $ior_blockSize -eq 0 ] &&
588 skip_env "Need free space more than $((2 * total_threads)) \
589 ${ior_blockUnit}: have $((space / multiplier))"
591 echo "(reduced blockSize to $ior_blockSize \
592 ${ior_blockUnit} bytes)"
595 print_opts IOR ior_THREADS ior_DURATION MACHINEFILE
598 # mpi_run uses mpiuser
600 if [ -z "$NFSCLIENT" ]; then
601 ior_stripe_params=${ior_stripe_params:-"-c -1"}
602 $LFS setstripe $testdir $ior_stripe_params ||
603 { error "setstripe failed" && return 2; }
608 # contiguous bytes to write per task (e.g.: 8, 4K, 2M, 1G)"
610 # -t N transferSize -- size of transfer in bytes (e.g.: 8, 4K, 2M, 1G)"
611 # -w writeFile -- write file"
612 # -r readFile -- read existing file"
613 # -W checkWrite -- check read after write"
614 # -C reorderTasks -- changes task ordering to n+1 ordering for readback
615 # -T maxTimeDuration -- max time in minutes to run tests"
616 # -k keepFile -- keep testFile(s) on program exit
619 if [ -n "$ior_custom_params" ]; then
620 cmd="$IOR $ior_custom_params -o $testdir/iorData"
622 cmd="$IOR -a $ior_type -b ${ior_blockSize}${ior_blockUnit} \
623 -o $testdir/iorData -t $ior_xferSize -v -C -w -r -W \
624 -i $ior_iteration -T $ior_DURATION -k"
627 [ $type = "fpp" ] && cmd="$cmd -F"
630 # find out if we need to use srun by checking $SRUN_PARTITION
631 if [ "$SRUN_PARTITION" ]; then
632 $SRUN $SRUN_OPTIONS -D $testdir -w $clients -N $num_clients \
633 -n $((num_clients * ior_THREADS)) -p $SRUN_PARTITION \
636 mpi_ior_custom_threads=${mpi_ior_custom_threads:-"$((num_clients * ior_THREADS))"}
637 mpi_run ${MACHINEFILE_OPTION} ${MACHINEFILE} \
638 -np $mpi_ior_custom_threads $cmd
642 if [ $rc != 0 ] ; then
643 error "ior failed! $rc"
649 MIB=${MIB:=$(which mib 2> /dev/null || true)}
650 [ "$NFSCLIENT" ] && skip "skipped for NFSCLIENT mode"
651 [ x$MIB = x ] && skip_env "MIB not found"
654 mib_THREADS=${mib_THREADS:-2}
655 mib_xferSize=${mib_xferSize:-1m}
656 mib_xferLimit=${mib_xferLimit:-5000}
657 mib_timeLimit=${mib_timeLimit:-300}
659 print_opts MIB mib_THREADS mib_xferSize mib_xferLimit mib_timeLimit \
662 local testdir=$DIR/d0.mib
664 # mpi_run uses mpiuser
666 $LFS setstripe $testdir -c -1 ||
667 error "setstripe failed"
669 # -I Show intermediate values in output
670 # -H Show headers in output
671 # -L Do not issue new system calls after this many seconds
672 # -s Use system calls of this size
674 # -l Issue no more than this many system calls
675 local cmd="$MIB -t $testdir -s $mib_xferSize -l $mib_xferLimit \
676 -L $mib_timeLimit -HI -p mib.$(date +%Y%m%d%H%M%S)"
679 # find out if we need to use srun by checking $SRUN_PARTITION
680 if [ "$SRUN_PARTITION" ]; then
681 $SRUN $SRUN_OPTIONS -D $testdir -w $clients -N $num_clients \
682 -n $((num_clients * mib_THREADS)) -p $SRUN_PARTITION \
685 mpi_run ${MACHINEFILE_OPTION} ${MACHINEFILE} \
686 -np $((num_clients * mib_THREADS)) $cmd
690 if [ $rc != 0 ] ; then
691 error "mib failed! $rc"
697 CASC_RW=${CASC_RW:-$(which cascading_rw 2> /dev/null || true)}
698 [ x$CASC_RW = x ] && skip_env "cascading_rw not found"
699 [ "$NFSCLIENT" ] && skip "skipped for NFSCLIENT mode"
702 casc_THREADS=${casc_THREADS:-2}
703 casc_REP=${casc_REP:-300}
706 # Need space estimation here.
708 print_opts CASC_RW clients casc_THREADS casc_REP MACHINEFILE
710 local testdir=$DIR/d0.cascading_rw
712 # mpi_run uses mpiuser
716 # -n: repeat test # times
718 local cmd="$CASC_RW -g -d $testdir -n $casc_REP"
721 mpi_run ${MACHINEFILE_OPTION} ${MACHINEFILE} \
722 -np $((num_clients * $casc_THREADS)) $cmd
725 if [ $rc != 0 ] ; then
726 error "cascading_rw failed! $rc"
731 run_write_append_truncate() {
732 [ "$NFSCLIENT" ] && skip "skipped for NFSCLIENT mode"
733 # location is lustre/tests dir
734 if ! which write_append_truncate > /dev/null 2>&1 ; then
735 skip_env "write_append_truncate not found"
739 write_THREADS=${write_THREADS:-8}
740 write_REP=${write_REP:-10000}
743 # Need space estimation here.
745 local testdir=$DIR/d0.write_append_truncate
746 local file=$testdir/f0.wat
748 print_opts clients write_REP write_THREADS MACHINEFILE
751 # mpi_run uses mpiuser
754 local cmd="write_append_truncate -n $write_REP $file"
757 mpi_run ${MACHINEFILE_OPTION} ${MACHINEFILE} \
758 -np $((num_clients * $write_THREADS)) $cmd
761 if [ $rc != 0 ] ; then
762 error "write_append_truncate failed! $rc"
768 run_write_disjoint() {
769 WRITE_DISJOINT=${WRITE_DISJOINT:-$(which write_disjoint 2> /dev/null ||
771 [ x$WRITE_DISJOINT = x ] && skip_env "write_disjoint not found"
772 [ "$NFSCLIENT" ] && skip "skipped for NFSCLIENT mode"
775 wdisjoint_THREADS=${wdisjoint_THREADS:-4}
776 wdisjoint_REP=${wdisjoint_REP:-10000}
780 # Need space estimation here.
782 print_opts WRITE_DISJOINT clients wdisjoint_THREADS wdisjoint_REP \
784 local testdir=$DIR/d0.write_disjoint
786 # mpi_run uses mpiuser
789 local cmd="$WRITE_DISJOINT -f $testdir/file -n $wdisjoint_REP -m \
793 mpi_run ${MACHINEFILE_OPTION} ${MACHINEFILE} \
794 -np $((num_clients * $wdisjoint_THREADS)) $cmd
797 if [ $rc != 0 ] ; then
798 error "write_disjoint failed! $rc"
803 run_parallel_grouplock() {
804 PARALLEL_GROUPLOCK=${PARALLEL_GROUPLOCK:-$(which parallel_grouplock \
805 2> /dev/null || true)}
807 [ x$PARALLEL_GROUPLOCK = x ] && skip "PARALLEL_GROUPLOCK not found"
808 [ "$NFSCLIENT" ] && skip "skipped for NFSCLIENT mode"
810 parallel_grouplock_MINTASKS=${parallel_grouplock_MINTASKS:-5}
812 print_opts clients parallel_grouplock_MINTASKS MACHINEFILE
814 local testdir=$DIR/d0.parallel_grouplock
816 # mpi_run uses mpiuser
822 for i in $(seq 12); do
824 local cmd="$PARALLEL_GROUPLOCK -g -v -d $testdir $subtest"
827 mpi_run ${MACHINEFILE_OPTION} ${MACHINEFILE} \
828 -np $parallel_grouplock_MINTASKS $cmd
830 if [ $rc != 0 ] ; then
831 error_noexit "parallel_grouplock subtests $subtest " \
834 echo "parallel_grouplock subtests $subtest PASS"
836 let status=$((status + rc))
837 # clear debug to collect one log per one test
838 do_nodes $(comma_list $(nodes_list)) lctl clear
840 [ $status -eq 0 ] || error "parallel_grouplock status: $status"
844 cleanup_statahead () {
851 for i in $(seq 0 $num_mntpts);do
852 zconf_umount_clients $clients ${mntpt_root}$i ||
853 error_exit "Failed to umount lustre on ${mntpt_root}$i"
858 if [[ -n $NFSCLIENT ]]; then
859 skip "Statahead testing is not supported on NFS clients."
861 [ x$MDSRATE = x ] && skip_env "mdsrate not found"
863 statahead_NUMMNTPTS=${statahead_NUMMNTPTS:-5}
864 statahead_NUMFILES=${statahead_NUMFILES:-500000}
866 print_opts MDSRATE clients statahead_NUMMNTPTS statahead_NUMFILES
870 # do not use default "d[0-9]*" dir name
871 # to avoid of rm $statahead_NUMFILES (500k) files in t-f cleanup
873 local testdir=$DIR/$dir
875 # cleanup only if dir exists
876 # cleanup only $statahead_NUMFILES number of files
877 # ignore the other files created by someone else
879 mdsrate_cleanup $((num_clients * 32)) $MACHINEFILE \
880 $statahead_NUMFILES $testdir 'f%%d' --ignore
883 # mpi_run uses mpiuser
886 local num_files=$statahead_NUMFILES
888 local IFree=$(inodes_available)
889 if [ $IFree -lt $num_files ]; then
895 local cmd1="${MDSRATE} ${MDSRATE_DEBUG} --mknod --dir $testdir"
896 local cmd2="--nfiles $num_files --filefmt 'f%%d'"
897 local cmd="$cmd1 $cmd2"
900 mpi_run ${MACHINEFILE_OPTION} ${MACHINEFILE} \
901 -np $((num_clients * 32)) $cmd
904 if [ $rc != 0 ] ; then
905 error "mdsrate failed to create $rc"
909 local num_mntpts=$statahead_NUMMNTPTS
910 local mntpt_root=$TMP/mntpt/lustre
911 local mntopts=$MNTOPTSTATAHEAD
913 echo "Mounting $num_mntpts lustre clients starts on $clients"
914 trap "cleanup_statahead $clients $mntpt_root $num_mntpts" EXIT ERR
915 for i in $(seq 0 $num_mntpts); do
916 zconf_mount_clients $clients ${mntpt_root}$i "$mntopts" ||
917 error_exit "Failed to mount lustre on ${mntpt_root}$i on $clients"
920 do_rpc_nodes $clients cancel_lru_locks mdc
922 do_rpc_nodes $clients do_ls $mntpt_root $num_mntpts $dir
924 mdsrate_cleanup $((num_clients * 32)) $MACHINEFILE \
925 $num_files $testdir 'f%%d' --ignore
927 # use rm instead of rmdir because of
928 # testdir could contain the files created by someone else,
929 # or by previous run where is num_files prev > num_files current
931 cleanup_statahead $clients $mntpt_root $num_mntpts
934 cleanup_rr_alloc () {
937 local mntpt_root="$2"
938 local rr_alloc_MNTPTS="$3"
939 local mntpt_dir=$(dirname ${mntpt_root})
941 for i in $(seq 0 $((rr_alloc_MNTPTS - 1))); do
942 zconf_umount_clients $clients ${mntpt_root}$i ||
943 error_exit "Failed to umount lustre on ${mntpt_root}$i"
945 do_nodes $clients "rm -rf $mntpt_dir"
949 remote_mds_nodsh && skip "remote MDS with nodsh"
950 echo "===Test gives more reproduction percentage if number of "\
951 "client and ost are more. Test with 44 or more clients "\
952 "and 73 or more OSTs gives 100% reproduction rate=="
954 RR_ALLOC=${RR_ALLOC:-$(which rr_alloc 2> /dev/null || true)}
955 [ x$RR_ALLOC = x ] && skip_env "rr_alloc not found"
956 declare -a diff_max_min_arr
957 # foeo = file on each ost. calc = calculated.
960 local qos_prec_objs="${TMP}/qos_and_precreated_objects"
961 local rr_alloc_NFILES=${rr_alloc_NFILES:-555}
962 local rr_alloc_MNTPTS=${rr_alloc_MNTPTS:-11}
963 local total_MNTPTS=$((rr_alloc_MNTPTS * num_clients))
964 local mntpt_root="${TMP}/rr_alloc_mntpt/lustre"
965 if [ $MDSCOUNT -lt 2 ]; then
966 [ -e $DIR/$tdir ] || mkdir -p $DIR/$tdir
968 [ -e $DIR/$tdir ] || $LFS mkdir -i 0 $DIR/$tdir
970 chmod 0777 $DIR/$tdir
971 $LFS setstripe -c 1 /$DIR/$tdir
973 trap "cleanup_rr_alloc $clients $mntpt_root $rr_alloc_MNTPTS" EXIT ERR
974 for i in $(seq 0 $((rr_alloc_MNTPTS - 1))); do
975 zconf_mount_clients $clients ${mntpt_root}$i $MOUNT_OPTS ||
976 error_exit "Failed to mount lustre on ${mntpt_root}$i $clients"
979 local cmd="$RR_ALLOC $mntpt_root/$tdir/ash $rr_alloc_NFILES \
982 # Save mdt values, set threshold to 100% i.e always Round Robin,
983 # restore the saved values again after creating files...
984 save_lustre_params mds1 \
985 "lov.$FSNAME-MDT0000*.qos_threshold_rr" > $qos_prec_objs
986 save_lustre_params mds1 \
987 "osp.$FSNAME-OST*-osc-MDT0000.create_count" >> $qos_prec_objs
989 local old_create_count=$(grep -e "create_count" $qos_prec_objs |
990 cut -d'=' -f 2 | sort -nr | head -n1)
992 # Make sure that every osp has enough precreated objects for the file
995 # create_count is always set to the power of 2 only, so if the files
996 # per OST are not multiple of that then it will be set to nearest
997 # lower power of 2. So set 'create_count' to the upper power of 2.
999 foeo_calc=$((rr_alloc_NFILES * total_MNTPTS / OSTCOUNT))
1000 local create_count=$((2 * foeo_calc))
1001 do_facet mds1 "$LCTL set_param -n \
1002 lov.$FSNAME-MDT0000*.qos_threshold_rr 100 \
1003 osp.$FSNAME-OST*-osc-MDT0000.create_count $create_count" ||
1004 error "failed while setting qos_threshold_rr & creat_count"
1006 # Create few temporary files in order to increase the precreated objects
1007 # to a desired value, before starting 'rr_alloc' app. Due to default
1008 # value 32 of precreation count (OST_MIN_PRECREATE=32), precreated
1009 # objects available are 32 initially, these gets exhausted very soon,
1010 # which causes skip of some osps when very large number of files
1011 # is created per OSTs.
1012 createmany -o $DIR/$tdir/foo- $(((old_create_count + 1) * OSTCOUNT)) \
1014 rm -f /$DIR/$tdir/foo*
1016 # Check for enough precreated objects... We should not
1017 # fail here because code(osp_precreate.c) also takes care of it.
1018 # So we have good chances of passing test even if this check fails.
1020 for ost_idx in $(seq 0 $((OSTCOUNT - 1))); do
1021 [[ $(precreated_ost_obj_count $mdt_idx $ost_idx) -ge \
1022 $foeo_calc ]] || echo "Warning: test may fail because" \
1023 "of lack of precreated objects on OST${ost_idx}"
1026 if [[ $total_MNTPTS -ne 0 ]]; then
1027 # Now start the actual file creation app.
1028 mpi_run "-np $total_MNTPTS" $cmd || return
1030 error "No mount point"
1033 restore_lustre_params < $qos_prec_objs
1034 rm -f $qos_prec_objs
1036 diff_max_min_arr=($($LFS getstripe -r $DIR/$tdir/ |
1037 grep "lmm_stripe_offset:" | awk '{print $2}' | sort -n |
1038 uniq -c | awk 'NR==1 {min=max=$1} \
1039 { $1<min ? min=$1 : min; $1>max ? max=$1 : max} \
1040 END {print max-min, max, min}'))
1044 # In-case of fairly large number of file creation using RR (round-robin)
1045 # there can be two cases in which deviation will occur than the regular
1046 # RR algo behaviour-
1047 # 1- When rr_alloc does not start right with 'lqr_start_count' reseeded,
1048 # 2- When rr_alloc does not finish with 'lqr_start_count == 0'.
1049 # So the difference of files b/w any 2 OST should not be more than 2.
1050 [[ ${diff_max_min_arr[0]} -le 2 ]] ||
1051 error "Uneven distribution detected: difference between" \
1052 "maximum files per OST (${diff_max_min_arr[1]}) and" \
1053 "minimum files per OST (${diff_max_min_arr[2]}) must not be" \
1058 # fs_test.x is the default name for exe
1059 FS_TEST=${FS_TEST:=$(which fs_test.x 2> /dev/null || true)}
1061 local clients=${CLIENTS:-$(hostname)}
1062 local testdir=$DIR/d0.fs_test
1063 local file=${testdir}/fs_test
1064 fs_test_threads=${fs_test_threads:-2}
1065 fs_test_type=${fs_test_type:-1}
1066 fs_test_nobj=${fs_test_nobj:-10}
1067 fs_test_check=${fs_test_check:-3}
1068 fs_test_strided=${fs_test_strided:-1}
1069 fs_test_touch=${fs_test_touch:-3}
1070 fs_test_supersize=${fs_test_supersize:-1}
1071 fs_test_op=${fs_test_op:-write}
1072 fs_test_barriers=${fs_test_barriers:-bopen,bwrite,bclose}
1073 fs_test_io=${fs_test_io:-mpi}
1074 fs_test_objsize=${fs_test_objsize:-100}
1075 fs_test_objunit=${fs_test_objunit:-1048576} # 1 mb
1076 fs_test_ndirs=${fs_test_ndirs:-80000}
1078 [ x$FS_TEST = x ] && skip "FS_TEST not found"
1080 # Space estimation in bytes
1081 local space=$(df -B 1 -P $dir | tail -n 1 | awk '{ print $4 }')
1082 local total_threads=$((num_clients * fs_test_threads))
1083 echo "+ $fs_test_objsize * $fs_test_objunit * $total_threads "
1084 if [ $((space / 2)) -le \
1085 $((fs_test_objsize * fs_test_objunit * total_threads)) ]; then
1086 fs_test_objsize=$((space / 2 / fs_test_objunit / \
1088 [ $fs_test_objsize -eq 0 ] &&
1089 skip_env "Need free space more than \
1090 $((2 * total_threads * fs_test_objunit)) \
1091 : have $((space / fs_test_objunit))"
1093 echo "(reduced objsize to \
1094 $((fs_test_objsize * fs_test_objunit)) bytes)"
1097 print_opts FS_TEST clients fs_test_threads fs_test_objsize MACHINEFILE
1100 # mpi_run uses mpiuser
1103 # --nodb Turn off the database code at runtime
1104 # -g --target The path to the data file
1105 # -t --type Whether to do N-N (1) or N-1 (2)
1106 # -n --nobj The number of objects written/read by each proc
1107 # -z --size The size of each object
1108 # -d ---num_nn_dirs Number of subdirectories for files
1109 # -C --check Check every byte using argument 3.
1110 # --collective Whether to use collective I/O (for N-1, mpi-io only)
1111 # -s --strided Whether to use a strided pattern (for N-1 only)
1112 # -T --touch Touch every byte using argument 3
1113 # -o --op Whether to read only (read) or write only (write)
1114 # -b --barriers When to barrier.
1115 # -i --io Use POSIX, MPI, or PLFS IO routines (mpi|posix|plfs)
1116 # -S --supersize Specify how many objects per superblock
1118 local cmd="$FS_TEST -nodb -g $file -t $fs_test_type -n $fs_test_nobj \
1119 -z $((fs_test_objsize * fs_test_objunit)) -d $fs_test_ndirs \
1120 -C $fs_test_check -collective -s $fs_test_strided \
1121 -T $fs_test_touch -o $fs_test_op -b $fs_test_barriers \
1122 -i $fs_test_io -S $fs_test_supersize"
1125 mpi_run "-np $((num_clients * fs_test_threads))" $cmd
1128 if [ $rc != 0 ] ; then
1129 error "fs_test failed! $rc"
1135 ior_mdtest_parallel() {
1143 run_mdtest $type || rc2=$?
1144 [[ $rc2 -ne 0 ]] && echo "mdtest failed with error $rc2"
1146 wait $pids || rc1=$?
1147 [[ $rc1 -ne 0 ]] && echo "ior failed with error $rc1"
1149 [[ $rc1 -ne 0 || $rc2 -ne 0 ]] && return 1
1154 FIO=${FIO:=$(which fio 2> /dev/null || true)}
1156 local clients=${CLIENTS:-$(hostname)}
1157 local fio_jobNum=${fio_jobNum:-4}
1158 local fio_jobFile=${fio_jobFile:-$TMP/fiojobfile.$(date +%s)}
1159 local fio_bs=${fio_bs:-1}
1160 local testdir=$DIR/d0.fio
1161 local file=${testdir}/fio
1163 local propagate=false
1165 [ "$SLOW" = "no" ] || runtime=600
1167 [ x$FIO = x ] && skip_env "FIO not found"
1171 # use fio job file if exists,
1172 # create a simple one if missing
1173 if ! [ -f $fio_jobFile ]; then
1174 cat >> $fio_jobFile <<EOF
1180 filename=${file}_\$(hostname)
1182 # bs size increased by $i for each job
1183 for ((i=1; i<=fio_jobNum; i++)); do
1184 cat >> $fio_jobFile <<EOF
1187 bs=$(( fio_bs * i ))m
1190 # job file is created, should be propagated to all clients
1195 # propagate the job file if not all clients have it yet or
1196 # if the job file was created during the test run
1197 if ! do_nodesv $clients " [ -f $fio_jobFile ] " ||
1199 local cfg=$(cat $fio_jobFile)
1200 do_nodes $clients "echo \\\"$cfg\\\" > ${fio_jobFile}" ||
1201 error "job file $fio_jobFile is not propagated"
1202 do_nodesv $clients "cat ${fio_jobFile}"
1205 cmd="$FIO $fio_jobFile"
1208 log "clients: $clients $cmd"
1211 do_nodesv $clients "$cmd "
1214 [ $rc = 0 ] || error "fio failed: $rc"
1219 XDD=${XDD:=$(which xdd 2> /dev/null || true)}
1221 local clients=${CLIENTS:-$(hostname)}
1222 local testdir=$DIR/d0.xdd
1223 xdd_queuedepth=${xdd_queuedepth:-4}
1224 xdd_blocksize=${xdd_blocksize:-512}
1225 xdd_reqsize=${xdd_reqsize:-128}
1226 xdd_mbytes=${xdd_mbytes:-100}
1227 xdd_passes=${xdd_passes:-40}
1228 xdd_rwratio=${xdd_rwratio:-0}
1229 xdd_ntargets=${xdd_ntargets:-6}
1230 local xdd_custom_params=${xdd_custom_params:-"-dio -stoponerror \
1231 -maxpri -minall -noproclock -nomemlock"}
1233 [ x$XDD = x ] && skip "XDD not found"
1235 print_opts XDD clients xdd_queuedepth xdd_blocksize xdd_reqsize \
1236 xdd_mbytes xdd_passes xdd_rwratio
1241 # Target files creates based on the given number of targets
1242 for (( i=0; i < $xdd_ntargets; i++ ))
1244 files+="${testdir}/xdd"$i" "
1247 # -targets specifies the devices or files to perform operation
1248 # -reqsize number of 'blocks' per operation
1249 # -mbytes number of 1024*1024-byte blocks to transfer
1250 # -blocksize size of a single 'block'
1251 # -passes number of times to read mbytes
1252 # -queuedepth number of commands to queue on the target
1253 # -rwratio percentage of read to write operations
1254 # -verbose will print out statistics on each pass
1256 local cmd="$XDD -targets $xdd_ntargets $files -reqsize $xdd_reqsize \
1257 -mbytes $xdd_mbytes -blocksize $xdd_blocksize \
1258 -passes $xdd_passes -queuedepth $xdd_queuedepth \
1259 -rwratio $xdd_rwratio -verbose $xdd_custom_params"
1263 do_nodesv $clients "$cmd "
1266 [ $rc = 0 ] || error "xdd failed: $rc"