3 # Simple function used by run_*.sh scripts
8 if [ -z "${!name}" ]; then
9 echo "$0: $name must be set"
13 [ $failed ] && exit 1 || true
16 # lrepl - Lustre test Read-Eval-Print Loop.
18 # This function implements a REPL for the Lustre test framework. It
19 # doesn't exec an actual shell because the user may want to inspect
20 # variables and use functions from the test framework.
27 This is an interactive read-eval-print loop interactive shell
28 simulation that you can use to debug failing tests. You can
29 enter most bash command lines (see notes below).
31 Use this REPL to inspect variables, set them, call test
32 framework shell functions, etcetera.
34 'exit' or EOF to exit this shell.
36 set \$retcode to 0 to cause the assertion failure that
37 triggered this REPL to be ignored.
40 do_facet ost1 lctl get_param ost.*.ost.threads_*
41 do_rpc_nodes \$OSTNODES unload_modules
44 All but the last line of multi-line statements or blocks
45 must end in a backslash.
47 "Here documents" are not supported.
49 History is not supported, but command-line editing is.
53 # Prompt escapes don't work in read -p, sadly.
54 prompt=":test_${testnum:-UNKNOWN}:$(uname -n):$(basename $PWD)% "
56 # We use read -r to get close to a shell experience
57 while read -e -r -p "$prompt" rawline; do
60 # Don't want to exit-exit, just exit the REPL
62 # We need to handle continuations, and read -r doesn't do
63 # that for us. Yet we need read -r.
65 # We also use case/esac to compare lines read to "*\\"
66 # because [ "$line" = *\\ ] and variants of that don't work.
68 while read -e -r -p '> ' rawline
70 line="$line"$'\n'"$rawline"
72 # We could check for here documents by matching
73 # against *<<*, but who cares.
86 # Finally! Time to eval.
90 echo $'\n\tExiting interactive shell...\n'
94 # lassert - Lustre test framework assert
96 # Arguments: failure code, failure message, expression/statement
98 # lassert evaluates the expression given, and, if false, calls
99 # error() to trigger test failure. If REPL_ON_LASSERT is true then
100 # lassert will call lrepl() to give the user an interactive shell.
101 # If the REPL sets retcode=0 then the assertion failure will be
108 echo "checking $* ($(eval echo \""$*"\"))..."
109 eval "$@" && return 0;
111 if ${REPL_ON_LASSERT:-false}; then
112 echo "Assertion $retcode failed: $* (expanded: $(eval echo \""$*"\"))
117 error "Assertion $retcode failed: $* (expanded: $(eval echo \""$*"\"))
122 # setmodopts- set module options for subsequent calls to load_modules
124 # Usage: setmodopts module_name new_value [var_in_which_to_save_old_value]
125 # setmodopts -a module_name new_value [var_in_which_to_save_old_value]
127 # In the second usage the new value is appended to the old.
131 if [ "$1" = -a ]; then
136 local _var=MODOPTS_$1
141 # Dynamic naming of variables is a pain in bash. In ksh93 we could
142 # write "nameref opts_var=${modname}_MODOPTS" then assign directly
143 # to opts_var. Associative arrays would also help, alternatively.
144 # Alas, we're stuck with eval until all distros move to a more recent
145 # version of bash. Fortunately we don't need to eval unset and export.
147 if [ -z "$_newvalue" ]; then
153 $_append && _newvalue="$_oldvalue $_newvalue"
154 export $_var="$_newvalue"
155 echo setmodopts: ${_var}=${_newvalue}
157 [ -n "$_savevar" ] && eval $_savevar=\""$_oldvalue"\"
160 echoerr () { echo "$@" 1>&2 ; }
163 echoerr "$(date +'%F %H:%M:%S'): client load was signaled to terminate"
165 local PGID=$(ps -eo "%c %p %r" | awk "/ $PPID / {print \$3}")
172 local mpirun="$MPIRUN $MPIRUN_OPTIONS"
173 local command="$mpirun $@"
174 local mpilog=$TMP/mpi.log
177 if [ -n "$MPI_USER" -a "$MPI_USER" != root -a -n "$mpirun" ]; then
178 echo "+ chmod 0777 $MOUNT"
180 command="su $MPI_USER sh -c \"$command \""
185 eval $command 2>&1 | tee $mpilog || true
188 if [ $rc -eq 0 ] && grep -q "p4_error:" $mpilog ; then
197 for i in ${1//,/ }; do
198 if [ "$list" = "" ]; then
201 list="$list$escape $i@$NETTYPE"
207 # FIXME: all setup/cleanup can be done without rpc.sh
210 [ x$1 = x--verbose ] && verbose=true
212 export LST_SESSION=`$LST show_session 2>/dev/null | awk -F " " '{print $5}'`
213 [ "$LST_SESSION" == "" ] && return
222 lst_session_cleanup_all () {
223 local list=$(comma_list $(nodes_list))
224 do_rpc_nodes $list lst_end_session
228 lsmod | grep -q lnet_selftest && \
229 rmmod lnet_selftest > /dev/null 2>&1 || true
233 local list=$(comma_list $(nodes_list))
235 # lst end_session needs to be executed only locally
236 # i.e. on node where lst new_session was called
237 lst_end_session --verbose
238 do_rpc_nodes $list lst_cleanup
242 load_module lnet_selftest
246 local list=$(comma_list $(nodes_list))
247 do_rpc_nodes $list lst_setup
253 # Passed a single argument, strips everything off following
254 # and includes the first period.
255 # client-20.lab.whamcloud.com becomes client-20
257 echo $(sed 's/\..*//' <<< $1)
263 # Find remote nodename, stripped of any domain, etc.
264 # 'hostname -s' is easy, but not implemented on all systems
266 local rname=$(do_node $1 "uname -n" || echo -1)
267 if [[ "$rname" = "-1" ]]; then
270 echo $(short_hostname $rname)
280 echo "${var}=${!var}"
282 [ -e $MACHINEFILE ] && cat $MACHINEFILE
287 local cbench_DIR=${cbench_DIR:-""}
288 local cbench_IDIRS=${cbench_IDIRS:-2}
289 local cbench_RUNS=${cbench_RUNS:-2}
291 print_opts cbench_DIR cbench_IDIRS cbench_RUNS
293 [ x$cbench_DIR = x ] &&
294 { skip_env "compilebench not found" && return; }
296 [ -e $cbench_DIR/compilebench ] ||
297 { skip_env "No compilebench build" && return; }
300 # compile dir kernel-0 ~1GB
301 # required space ~1GB * cbench_IDIRS
302 local space=$(df -P $dir | tail -n 1 | awk '{ print $4 }')
303 if [[ $space -le $((1024 * 1024 * cbench_IDIRS)) ]]; then
304 cbench_IDIRS=$((space / 1024 / 1024))
305 [[ $cbench_IDIRS -eq 0 ]] &&
306 skip_env "Need free space at least 1GB, have $space" &&
309 echo "reducing initial dirs to $cbench_IDIRS"
311 echo "free space = $space KB"
314 # t-f _base needs to be modifyed to set properly tdir
315 # for new "test_foo" functions names
316 # local testdir=$DIR/$tdir
317 local testdir=$dir/d0.compilebench.$$
322 local cmd="./compilebench -D $testdir -i $cbench_IDIRS \
323 -r $cbench_RUNS --makej"
332 [ $rc = 0 ] || error "compilebench failed: $rc"
338 local mntpt=${2:-$MOUNT}
339 METABENCH=${METABENCH:-$(which metabench 2> /dev/null || true)}
340 mbench_NFILES=${mbench_NFILES:-30400}
342 mbench_THREADS=${mbench_THREADS:-4}
343 mbench_OPTIONS=${mbench_OPTIONS:-}
344 mbench_CLEANUP=${mbench_CLEANUP:-true}
346 [ x$METABENCH = x ] &&
347 { skip_env "metabench not found" && return; }
349 print_opts METABENCH clients mbench_NFILES mbench_THREADS
351 local testdir=$dir/d0.metabench
353 # mpi_run uses mpiuser
356 # -C Run the file creation tests. Creates zero byte files.
357 # -S Run the file stat tests.
358 # -c nfile Number of files to be used in each test.
359 # -k Cleanup files when finished.
360 local cmd="$METABENCH -w $testdir -c $mbench_NFILES -C -S $mbench_OPTIONS"
363 # find out if we need to use srun by checking $SRUN_PARTITION
364 if [ "$SRUN_PARTITION" ]; then
365 $SRUN $SRUN_OPTIONS -D $testdir -w $clients -N $num_clients \
366 -n $((num_clients * mbench_THREADS)) \
367 -p $SRUN_PARTITION -- $cmd
369 mpi_run ${MACHINEFILE_OPTION} ${MACHINEFILE} \
370 -np $((num_clients * $mbench_THREADS)) $cmd
374 if [ $rc != 0 ] ; then
375 error "metabench failed! $rc"
378 if $mbench_CLEANUP; then
381 mv $dir/d0.metabench $mntpt/_xxx.$(date +%s).d0.metabench
387 SIMUL=${SIMUL:=$(which simul 2> /dev/null || true)}
389 simul_THREADS=${simul_THREADS:-2}
390 simul_REP=${simul_REP:-20}
392 if [ "$NFSCLIENT" ]; then
393 skip "skipped for NFSCLIENT mode"
398 { skip_env "simul not found" && return; }
401 # Need space estimation here.
403 print_opts SIMUL clients simul_REP simul_THREADS
405 local testdir=$DIR/d0.simul
407 # mpi_run uses mpiuser
410 # -n # : repeat each test # times
411 # -N # : repeat the entire set of tests # times
413 local cmd="$SIMUL -d $testdir -n $simul_REP -N $simul_REP"
416 # find out if we need to use srun by checking $SRUN_PARTITION
417 if [ "$SRUN_PARTITION" ]; then
418 $SRUN $SRUN_OPTIONS -D $testdir -w $clients -N $num_clients \
419 -n $((num_clients * simul_THREADS)) -p $SRUN_PARTITION \
422 mpi_run ${MACHINEFILE_OPTION} ${MACHINEFILE} \
423 -np $((num_clients * simul_THREADS)) $cmd
427 if [ $rc != 0 ] ; then
428 error "simul failed! $rc"
435 MDTEST=${MDTEST:=$(which mdtest 2> /dev/null || true)}
437 mdtest_THREADS=${mdtest_THREADS:-2}
438 mdtest_nFiles=${mdtest_nFiles:-"100000"}
439 # We devide the files by number of core
440 mdtest_nFiles=$((mdtest_nFiles/mdtest_THREADS/num_clients))
441 mdtest_iteration=${mdtest_iteration:-1}
442 local mdtest_custom_params=${mdtest_custom_params:-""}
444 local type=${1:-"ssf"}
446 if [ "$NFSCLIENT" ]; then
447 skip "skipped for NFSCLIENT mode"
452 { skip_env "mdtest not found" && return; }
455 # Need space estimation here.
457 print_opts MDTEST mdtest_iteration mdtest_THREADS mdtest_nFiles
459 local testdir=$DIR/d0.mdtest
461 # mpi_run uses mpiuser
464 # -i # : repeat each test # times
466 # -n # : number of file/dir to create/stat/remove
467 # -u : each process create/stat/remove individually
469 local cmd="$MDTEST -d $testdir -i $mdtest_iteration \
470 -n $mdtest_nFiles $mdtest_custom_params"
472 [ $type = "fpp" ] && cmd="$cmd -u"
475 # find out if we need to use srun by checking $SRUN_PARTITION
476 if [ "$SRUN_PARTITION" ]; then
477 $SRUN $SRUN_OPTIONS -D $testdir -w $clients -N $num_clients \
478 -n $((num_clients * mdtest_THREADS)) \
479 -p $SRUN_PARTITION -- $cmd
481 mpi_run ${MACHINEFILE_OPTION} ${MACHINEFILE} \
482 -np $((num_clients * mdtest_THREADS)) $cmd
486 if [ $rc != 0 ] ; then
487 error "mdtest failed! $rc"
494 cnt_DIR=${cnt_DIR:-""}
495 cnt_NRUN=${cnt_NRUN:-10}
497 print_opts cnt_DIR cnt_NRUN
500 { skip_env "connectathon dir not found" && return; }
502 [ -e $cnt_DIR/runtests ] ||
503 { skip_env "No connectathon runtests found" && return; }
506 # "special" tests create a 30 MB file + misc. small files
507 # required space ~40 MB
508 local space=$(df -P $dir | tail -n 1 | awk '{ print $4 }')
509 if [[ $space -le $((1024 * 40)) ]]; then
510 skip_env "Need free space at least 40MB, have $space KB" &&
513 echo "free space = $space KB"
515 local testdir=$dir/d0.connectathon
522 # To run connectathon:
523 # runtests [-a|-b|-g|-s|-l] [-f|-n|-t] [-N numpasses] [test-directory]
525 # One of the following test types
530 # -a all of the above
532 # -f a quick functional test
533 # -n suppress directory operations (mkdir and rmdir)
534 # -t run with time statistics (default for basic tests)
536 # -N numpasses - specifies the number of times to run
537 # the tests. Optional.
540 # Include lock tests unless we're running on nfsv4
541 local fstype=$(df -TP $testdir | awk 'NR==2 {print $2}')
542 echo "$testdir: $fstype"
543 if [[ $fstype != "nfs4" ]]; then
547 for test in $tests; do
548 local cmd="./runtests -N $cnt_NRUN $test -f $testdir"
554 [ $rc = 0 ] || error "connectathon failed: $rc"
562 local type=${1:="ssf"}
564 local testdir=$dir/d0.ior.$type
565 local nfs_srvmntpt=$3
567 if [ "$NFSCLIENT" ]; then
568 [[ -n $nfs_srvmntpt ]] ||
569 { error "NFSCLIENT mode, but nfs exported dir"\
570 "is not set!" && return 1; }
573 IOR=${IOR:-$(which IOR 2> /dev/null || true)}
575 { skip_env "IOR not found" && return; }
578 ior_THREADS=${ior_THREADS:-2}
579 ior_iteration=${ior_iteration:-1}
580 ior_blockSize=${ior_blockSize:-6}
581 ior_blockUnit=${ior_blockUnit:-M} # K, M, G
582 ior_xferSize=${ior_xferSize:-1M}
583 ior_type=${ior_type:-POSIX}
584 ior_DURATION=${ior_DURATION:-30} # minutes
586 case ${ior_blockUnit} in
588 multiplier=$((1024 * 1024 * 1024))
591 multiplier=$((1024 * 1024))
596 *) error "Incorrect block unit should be one of [KMG]"
600 # calculate the space in bytes
601 local space=$(df -B 1 -P $dir | tail -n 1 | awk '{ print $4 }')
602 local total_threads=$((num_clients * ior_THREADS))
603 echo "+ $ior_blockSize * $multiplier * $total_threads "
604 if [ $((space / 2)) -le \
605 $((ior_blockSize * multiplier * total_threads)) ]; then
606 ior_blockSize=$((space / 2 / multiplier / total_threads))
607 [ $ior_blockSize -eq 0 ] && \
608 skip_env "Need free space more than $((2 * total_threads)) \
609 ${ior_blockUnit}: have $((space / multiplier))" &&
612 echo "(reduced blockSize to $ior_blockSize \
613 ${ior_blockUnit} bytes)"
616 print_opts IOR ior_THREADS ior_DURATION MACHINEFILE
619 # mpi_run uses mpiuser
621 if [ -z "$NFSCLIENT" ]; then
622 ior_stripe_params=${ior_stripe_params:-"-c -1"}
623 $LFS setstripe $testdir $ior_stripe_params ||
624 { error "setstripe failed" && return 2; }
629 # contiguous bytes to write per task (e.g.: 8, 4K, 2M, 1G)"
631 # -t N transferSize -- size of transfer in bytes (e.g.: 8, 4K, 2M, 1G)"
632 # -w writeFile -- write file"
633 # -r readFile -- read existing file"
634 # -W checkWrite -- check read after write"
635 # -C reorderTasks -- changes task ordering to n+1 ordering for readback
636 # -T maxTimeDuration -- max time in minutes to run tests"
637 # -k keepFile -- keep testFile(s) on program exit
640 if [ -n "$ior_custom_params" ]; then
641 cmd="$IOR $ior_custom_params -o $testdir/iorData"
643 cmd="$IOR -a $ior_type -b ${ior_blockSize}${ior_blockUnit} \
644 -o $testdir/iorData -t $ior_xferSize -v -C -w -r -W \
645 -i $ior_iteration -T $ior_DURATION -k"
648 [ $type = "fpp" ] && cmd="$cmd -F"
651 # find out if we need to use srun by checking $SRUN_PARTITION
652 if [ "$SRUN_PARTITION" ]; then
653 $SRUN $SRUN_OPTIONS -D $testdir -w $clients -N $num_clients \
654 -n $((num_clients * ior_THREADS)) -p $SRUN_PARTITION \
657 mpi_ior_custom_threads=${mpi_ior_custom_threads:-"$((num_clients * ior_THREADS))"}
658 mpi_run ${MACHINEFILE_OPTION} ${MACHINEFILE} \
659 -np $mpi_ior_custom_threads $cmd
663 if [ $rc != 0 ] ; then
664 error "ior failed! $rc"
671 MIB=${MIB:=$(which mib 2> /dev/null || true)}
673 mib_THREADS=${mib_THREADS:-2}
674 mib_xferSize=${mib_xferSize:-1m}
675 mib_xferLimit=${mib_xferLimit:-5000}
676 mib_timeLimit=${mib_timeLimit:-300}
678 if [ "$NFSCLIENT" ]; then
679 skip "skipped for NFSCLIENT mode"
684 { skip_env "MIB not found" && return; }
686 print_opts MIB mib_THREADS mib_xferSize mib_xferLimit mib_timeLimit \
689 local testdir=$DIR/d0.mib
691 # mpi_run uses mpiuser
693 $LFS setstripe $testdir -c -1 ||
694 { error "setstripe failed" && return 2; }
696 # -I Show intermediate values in output
697 # -H Show headers in output
698 # -L Do not issue new system calls after this many seconds
699 # -s Use system calls of this size
701 # -l Issue no more than this many system calls
702 local cmd="$MIB -t $testdir -s $mib_xferSize -l $mib_xferLimit \
703 -L $mib_timeLimit -HI -p mib.$(date +%Y%m%d%H%M%S)"
706 # find out if we need to use srun by checking $SRUN_PARTITION
707 if [ "$SRUN_PARTITION" ]; then
708 $SRUN $SRUN_OPTIONS -D $testdir -w $clients -N $num_clients \
709 -n $((num_clients * mib_THREADS)) -p $SRUN_PARTITION \
712 mpi_run ${MACHINEFILE_OPTION} ${MACHINEFILE} \
713 -np $((num_clients * mib_THREADS)) $cmd
717 if [ $rc != 0 ] ; then
718 error "mib failed! $rc"
725 CASC_RW=${CASC_RW:-$(which cascading_rw 2> /dev/null || true)}
727 casc_THREADS=${casc_THREADS:-2}
728 casc_REP=${casc_REP:-300}
730 if [ "$NFSCLIENT" ]; then
731 skip "skipped for NFSCLIENT mode"
736 { skip_env "cascading_rw not found" && return; }
739 # Need space estimation here.
741 print_opts CASC_RW clients casc_THREADS casc_REP MACHINEFILE
743 local testdir=$DIR/d0.cascading_rw
745 # mpi_run uses mpiuser
749 # -n: repeat test # times
751 local cmd="$CASC_RW -g -d $testdir -n $casc_REP"
754 mpi_run ${MACHINEFILE_OPTION} ${MACHINEFILE} \
755 -np $((num_clients * $casc_THREADS)) $cmd
758 if [ $rc != 0 ] ; then
759 error "cascading_rw failed! $rc"
764 run_write_append_truncate() {
767 write_THREADS=${write_THREADS:-8}
768 write_REP=${write_REP:-10000}
770 if [ "$NFSCLIENT" ]; then
771 skip "skipped for NFSCLIENT mode"
775 # location is lustre/tests dir
776 if ! which write_append_truncate > /dev/null 2>&1 ; then
777 skip_env "write_append_truncate not found"
782 # Need space estimation here.
784 local testdir=$DIR/d0.write_append_truncate
785 local file=$testdir/f0.wat
787 print_opts clients write_REP write_THREADS MACHINEFILE
790 # mpi_run uses mpiuser
793 local cmd="write_append_truncate -n $write_REP $file"
796 mpi_run ${MACHINEFILE_OPTION} ${MACHINEFILE} \
797 -np $((num_clients * $write_THREADS)) $cmd
800 if [ $rc != 0 ] ; then
801 error "write_append_truncate failed! $rc"
807 run_write_disjoint() {
808 if [ "$NFSCLIENT" ]; then
809 skip "skipped for NFSCLIENT mode"
813 WRITE_DISJOINT=${WRITE_DISJOINT:-$(which write_disjoint 2> /dev/null ||
816 [ x$WRITE_DISJOINT = x ] &&
817 { skip_env "write_disjoint not found" && return; }
820 wdisjoint_THREADS=${wdisjoint_THREADS:-4}
821 wdisjoint_REP=${wdisjoint_REP:-10000}
825 # Need space estimation here.
827 print_opts WRITE_DISJOINT clients wdisjoint_THREADS wdisjoint_REP \
829 local testdir=$DIR/d0.write_disjoint
831 # mpi_run uses mpiuser
834 local cmd="$WRITE_DISJOINT -f $testdir/file -n $wdisjoint_REP -m \
838 mpi_run ${MACHINEFILE_OPTION} ${MACHINEFILE} \
839 -np $((num_clients * $wdisjoint_THREADS)) $cmd
842 if [ $rc != 0 ] ; then
843 error "write_disjoint failed! $rc"
848 run_parallel_grouplock() {
850 PARALLEL_GROUPLOCK=${PARALLEL_GROUPLOCK:-$(which parallel_grouplock \
851 2> /dev/null || true)}
852 parallel_grouplock_MINTASKS=${parallel_grouplock_MINTASKS:-5}
854 if [ "$NFSCLIENT" ]; then
855 skip "skipped for NFSCLIENT mode"
859 [ x$PARALLEL_GROUPLOCK = x ] &&
860 { skip "PARALLEL_GROUPLOCK not found" && return; }
862 print_opts clients parallel_grouplock_MINTASKS MACHINEFILE
864 local testdir=$DIR/d0.parallel_grouplock
866 # mpi_run uses mpiuser
872 for i in $(seq 12); do
874 local cmd="$PARALLEL_GROUPLOCK -g -v -d $testdir $subtest"
877 mpi_run ${MACHINEFILE_OPTION} ${MACHINEFILE} \
878 -np $parallel_grouplock_MINTASKS $cmd
880 if [ $rc != 0 ] ; then
881 error_noexit "parallel_grouplock subtests $subtest " \
884 echo "parallel_grouplock subtests $subtest PASS"
886 let status=$((status + rc))
887 # clear debug to collect one log per one test
888 do_nodes $(comma_list $(nodes_list)) lctl clear
890 [ $status -eq 0 ] || error "parallel_grouplock status: $status"
894 cleanup_statahead () {
901 for i in $(seq 0 $num_mntpts);do
902 zconf_umount_clients $clients ${mntpt_root}$i ||
903 error_exit "Failed to umount lustre on ${mntpt_root}$i"
909 statahead_NUMMNTPTS=${statahead_NUMMNTPTS:-5}
910 statahead_NUMFILES=${statahead_NUMFILES:-500000}
912 if [[ -n $NFSCLIENT ]]; then
913 skip "Statahead testing is not supported on NFS clients."
918 { skip_env "mdsrate not found" && return; }
920 print_opts MDSRATE clients statahead_NUMMNTPTS statahead_NUMFILES
924 # do not use default "d[0-9]*" dir name
925 # to avoid of rm $statahead_NUMFILES (500k) files in t-f cleanup
927 local testdir=$DIR/$dir
929 # cleanup only if dir exists
930 # cleanup only $statahead_NUMFILES number of files
931 # ignore the other files created by someone else
933 mdsrate_cleanup $((num_clients * 32)) $MACHINEFILE \
934 $statahead_NUMFILES $testdir 'f%%d' --ignore
937 # mpi_run uses mpiuser
940 local num_files=$statahead_NUMFILES
942 local IFree=$(inodes_available)
943 if [ $IFree -lt $num_files ]; then
949 local cmd1="${MDSRATE} ${MDSRATE_DEBUG} --mknod --dir $testdir"
950 local cmd2="--nfiles $num_files --filefmt 'f%%d'"
951 local cmd="$cmd1 $cmd2"
954 mpi_run ${MACHINEFILE_OPTION} ${MACHINEFILE} \
955 -np $((num_clients * 32)) $cmd
958 if [ $rc != 0 ] ; then
959 error "mdsrate failed to create $rc"
963 local num_mntpts=$statahead_NUMMNTPTS
964 local mntpt_root=$TMP/mntpt/lustre
965 local mntopts=$MNTOPTSTATAHEAD
967 echo "Mounting $num_mntpts lustre clients starts on $clients"
968 trap "cleanup_statahead $clients $mntpt_root $num_mntpts" EXIT ERR
969 for i in $(seq 0 $num_mntpts); do
970 zconf_mount_clients $clients ${mntpt_root}$i "$mntopts" ||
971 error_exit "Failed to mount lustre on ${mntpt_root}$i on $clients"
974 do_rpc_nodes $clients cancel_lru_locks mdc
976 do_rpc_nodes $clients do_ls $mntpt_root $num_mntpts $dir
978 mdsrate_cleanup $((num_clients * 32)) $MACHINEFILE \
979 $num_files $testdir 'f%%d' --ignore
981 # use rm instead of rmdir because of
982 # testdir could contain the files created by someone else,
983 # or by previous run where is num_files prev > num_files current
985 cleanup_statahead $clients $mntpt_root $num_mntpts
988 cleanup_rr_alloc () {
991 local mntpt_root="$2"
992 local rr_alloc_MNTPTS="$3"
993 local mntpt_dir=$(dirname ${mntpt_root})
995 for i in $(seq 0 $((rr_alloc_MNTPTS - 1))); do
996 zconf_umount_clients $clients ${mntpt_root}$i ||
997 error_exit "Failed to umount lustre on ${mntpt_root}$i"
999 do_nodes $clients "rm -rf $mntpt_dir"
1003 remote_mds_nodsh && skip "remote MDS with nodsh" && return
1004 echo "===Test gives more reproduction percentage if number of "\
1005 "client and ost are more. Test with 44 or more clients "\
1006 "and 73 or more OSTs gives 100% reproduction rate=="
1008 RR_ALLOC=${RR_ALLOC:-$(which rr_alloc 2> /dev/null || true)}
1009 [ x$RR_ALLOC = x ] && { skip_env "rr_alloc not found" && return; }
1010 declare -a diff_max_min_arr
1011 # foeo = file on each ost. calc = calculated.
1014 local qos_prec_objs="${TMP}/qos_and_precreated_objects"
1015 local rr_alloc_NFILES=${rr_alloc_NFILES:-555}
1016 local rr_alloc_MNTPTS=${rr_alloc_MNTPTS:-11}
1017 local total_MNTPTS=$((rr_alloc_MNTPTS * num_clients))
1018 local mntpt_root="${TMP}/rr_alloc_mntpt/lustre"
1019 if [ $MDSCOUNT -lt 2 ]; then
1020 [ -e $DIR/$tdir ] || mkdir -p $DIR/$tdir
1022 [ -e $DIR/$tdir ] || $LFS mkdir -i 0 $DIR/$tdir
1024 chmod 0777 $DIR/$tdir
1025 $SETSTRIPE -c 1 /$DIR/$tdir
1027 trap "cleanup_rr_alloc $clients $mntpt_root $rr_alloc_MNTPTS" EXIT ERR
1028 for i in $(seq 0 $((rr_alloc_MNTPTS - 1))); do
1029 zconf_mount_clients $clients ${mntpt_root}$i $MOUNT_OPTS ||
1030 error_exit "Failed to mount lustre on ${mntpt_root}$i $clients"
1033 local cmd="$RR_ALLOC $mntpt_root/$tdir/ash $rr_alloc_NFILES \
1036 # Save mdt values, set threshold to 100% i.e always Round Robin,
1037 # restore the saved values again after creating files...
1038 save_lustre_params mds1 \
1039 "lov.$FSNAME-MDT0000*.qos_threshold_rr" > $qos_prec_objs
1040 save_lustre_params mds1 \
1041 "osp.$FSNAME-OST*-osc-MDT0000.create_count" >> $qos_prec_objs
1043 local old_create_count=$(grep -e "create_count" $qos_prec_objs |
1044 cut -d'=' -f 2 | sort -nr | head -n1)
1046 # Make sure that every osp has enough precreated objects for the file
1049 # create_count is always set to the power of 2 only, so if the files
1050 # per OST are not multiple of that then it will be set to nearest
1051 # lower power of 2. So set 'create_count' to the upper power of 2.
1053 foeo_calc=$((rr_alloc_NFILES * total_MNTPTS / OSTCOUNT))
1054 local create_count=$((2 * foeo_calc))
1055 do_facet mds1 "$LCTL set_param -n \
1056 lov.$FSNAME-MDT0000*.qos_threshold_rr 100 \
1057 osp.$FSNAME-OST*-osc-MDT0000.create_count $create_count" ||
1058 error "failed while setting qos_threshold_rr & creat_count"
1060 # Create few temporary files in order to increase the precreated objects
1061 # to a desired value, before starting 'rr_alloc' app. Due to default
1062 # value 32 of precreation count (OST_MIN_PRECREATE=32), precreated
1063 # objects available are 32 initially, these gets exhausted very soon,
1064 # which causes skip of some osps when very large number of files
1065 # is created per OSTs.
1066 createmany -o $DIR/$tdir/foo- $(((old_create_count + 1) * OSTCOUNT)) \
1068 rm -f /$DIR/$tdir/foo*
1070 # Check for enough precreated objects... We should not
1071 # fail here because code(osp_precreate.c) also takes care of it.
1072 # So we have good chances of passing test even if this check fails.
1074 for ost_idx in $(seq 0 $((OSTCOUNT - 1))); do
1075 [[ $(precreated_ost_obj_count $mdt_idx $ost_idx) -ge \
1076 $foeo_calc ]] || echo "Warning: test may fail because" \
1077 "of lack of precreated objects on OST${ost_idx}"
1080 if [[ $total_MNTPTS -ne 0 ]]; then
1081 # Now start the actual file creation app.
1082 mpi_run "-np $total_MNTPTS" $cmd || return
1084 error "No mount point"
1087 restore_lustre_params < $qos_prec_objs
1088 rm -f $qos_prec_objs
1090 diff_max_min_arr=($($GETSTRIPE -r $DIR/$tdir/ |
1091 grep "lmm_stripe_offset:" | awk '{print $2}' | sort -n |
1092 uniq -c | awk 'NR==1 {min=max=$1} \
1093 { $1<min ? min=$1 : min; $1>max ? max=$1 : max} \
1094 END {print max-min, max, min}'))
1098 # In-case of fairly large number of file creation using RR (round-robin)
1099 # there can be two cases in which deviation will occur than the regular
1100 # RR algo behaviour-
1101 # 1- When rr_alloc does not start right with 'lqr_start_count' reseeded,
1102 # 2- When rr_alloc does not finish with 'lqr_start_count == 0'.
1103 # So the difference of files b/w any 2 OST should not be more than 2.
1104 [[ ${diff_max_min_arr[0]} -le 2 ]] ||
1105 error "Uneven distribution detected: difference between" \
1106 "maximum files per OST (${diff_max_min_arr[1]}) and" \
1107 "minimum files per OST (${diff_max_min_arr[2]}) must not be" \
1112 # fs_test.x is the default name for exe
1113 FS_TEST=${FS_TEST:=$(which fs_test.x 2> /dev/null || true)}
1115 local clients=${CLIENTS:-$(hostname)}
1116 local testdir=$DIR/d0.fs_test
1117 local file=${testdir}/fs_test
1118 fs_test_threads=${fs_test_threads:-2}
1119 fs_test_type=${fs_test_type:-1}
1120 fs_test_nobj=${fs_test_nobj:-10}
1121 fs_test_check=${fs_test_check:-3}
1122 fs_test_strided=${fs_test_strided:-1}
1123 fs_test_touch=${fs_test_touch:-3}
1124 fs_test_supersize=${fs_test_supersize:-1}
1125 fs_test_op=${fs_test_op:-write}
1126 fs_test_barriers=${fs_test_barriers:-bopen,bwrite,bclose}
1127 fs_test_io=${fs_test_io:-mpi}
1128 fs_test_objsize=${fs_test_objsize:-100}
1129 fs_test_objunit=${fs_test_objunit:-1048576} # 1 mb
1130 fs_test_ndirs=${fs_test_ndirs:-80000}
1132 [ x$FS_TEST = x ] &&
1133 { skip "FS_TEST not found" && return; }
1135 # Space estimation in bytes
1136 local space=$(df -B 1 -P $dir | tail -n 1 | awk '{ print $4 }')
1137 local total_threads=$((num_clients * fs_test_threads))
1138 echo "+ $fs_test_objsize * $fs_test_objunit * $total_threads "
1139 if [ $((space / 2)) -le \
1140 $((fs_test_objsize * fs_test_objunit * total_threads)) ]; then
1141 fs_test_objsize=$((space / 2 / fs_test_objunit / \
1143 [ $fs_test_objsize -eq 0 ] && \
1144 skip_env "Need free space more than \
1145 $((2 * total_threads * fs_test_objunit)) \
1146 : have $((space / fs_test_objunit))" &&
1149 echo "(reduced objsize to \
1150 $((fs_test_objsize * fs_test_objunit)) bytes)"
1153 print_opts FS_TEST clients fs_test_threads fs_test_objsize MACHINEFILE
1156 # mpi_run uses mpiuser
1159 # --nodb Turn off the database code at runtime
1160 # -g --target The path to the data file
1161 # -t --type Whether to do N-N (1) or N-1 (2)
1162 # -n --nobj The number of objects written/read by each proc
1163 # -z --size The size of each object
1164 # -d ---num_nn_dirs Number of subdirectories for files
1165 # -C --check Check every byte using argument 3.
1166 # --collective Whether to use collective I/O (for N-1, mpi-io only)
1167 # -s --strided Whether to use a strided pattern (for N-1 only)
1168 # -T --touch Touch every byte using argument 3
1169 # -o --op Whether to read only (read) or write only (write)
1170 # -b --barriers When to barrier.
1171 # -i --io Use POSIX, MPI, or PLFS IO routines (mpi|posix|plfs)
1172 # -S --supersize Specify how many objects per superblock
1174 local cmd="$FS_TEST -nodb -g $file -t $fs_test_type -n $fs_test_nobj \
1175 -z $((fs_test_objsize * fs_test_objunit)) -d $fs_test_ndirs \
1176 -C $fs_test_check -collective -s $fs_test_strided \
1177 -T $fs_test_touch -o $fs_test_op -b $fs_test_barriers \
1178 -i $fs_test_io -S $fs_test_supersize"
1181 mpi_run "-np $((num_clients * fs_test_threads))" $cmd
1184 if [ $rc != 0 ] ; then
1185 error "fs_test failed! $rc"
1191 ior_mdtest_parallel() {
1199 run_mdtest $type || rc2=$?
1200 [[ $rc2 -ne 0 ]] && echo "mdtest failed with error $rc2"
1202 wait $pids || rc1=$?
1203 [[ $rc1 -ne 0 ]] && echo "ior failed with error $rc1"
1205 [[ $rc1 -ne 0 || $rc2 -ne 0 ]] && return 1
1210 FIO=${FIO:=$(which fio 2> /dev/null || true)}
1212 local clients=${CLIENTS:-$(hostname)}
1213 local fio_jobNum=${fio_jobNum:-4}
1214 local fio_jobFile=${fio_jobFile:-$TMP/fiojobfile.$(date +%s)}
1215 local fio_bs=${fio_bs:-1}
1216 local testdir=$DIR/d0.fio
1217 local file=${testdir}/fio
1219 local propagate=false
1221 [ "$SLOW" = "no" ] || runtime=600
1224 { skip_env "FIO not found" && return; }
1228 # use fio job file if exists,
1229 # create a simple one if missing
1230 if ! [ -f $fio_jobFile ]; then
1231 cat >> $fio_jobFile <<EOF
1237 filename=${file}_\$(hostname)
1239 # bs size increased by $i for each job
1240 for ((i=1; i<=fio_jobNum; i++)); do
1241 cat >> $fio_jobFile <<EOF
1244 bs=$(( fio_bs * i ))m
1247 # job file is created, should be propagated to all clients
1252 # propagate the job file if not all clients have it yet or
1253 # if the job file was created during the test run
1254 if ! do_nodesv $clients " [ -f $fio_jobFile ] " ||
1256 local cfg=$(cat $fio_jobFile)
1257 do_nodes $clients "echo \\\"$cfg\\\" > ${fio_jobFile}" ||
1258 error "job file $fio_jobFile is not propagated"
1259 do_nodesv $clients "cat ${fio_jobFile}"
1262 cmd="$FIO $fio_jobFile"
1265 log "clients: $clients $cmd"
1268 do_nodesv $clients "$cmd "
1271 [ $rc = 0 ] || error "fio failed: $rc"
1276 XDD=${XDD:=$(which xdd 2> /dev/null || true)}
1278 local clients=${CLIENTS:-$(hostname)}
1279 local testdir=$DIR/d0.xdd
1280 xdd_queuedepth=${xdd_queuedepth:-4}
1281 xdd_blocksize=${xdd_blocksize:-512}
1282 xdd_reqsize=${xdd_reqsize:-128}
1283 xdd_mbytes=${xdd_mbytes:-100}
1284 xdd_passes=${xdd_passes:-40}
1285 xdd_rwratio=${xdd_rwratio:-0}
1286 xdd_ntargets=${xdd_ntargets:-6}
1287 local xdd_custom_params=${xdd_custom_params:-"-dio -stoponerror \
1288 -maxpri -minall -noproclock -nomemlock"}
1291 { skip "XDD not found" && return; }
1293 print_opts XDD clients xdd_queuedepth xdd_blocksize xdd_reqsize \
1294 xdd_mbytes xdd_passes xdd_rwratio
1299 # Target files creates based on the given number of targets
1300 for (( i=0; i < $xdd_ntargets; i++ ))
1302 files+="${testdir}/xdd"$i" "
1305 # -targets specifies the devices or files to perform operation
1306 # -reqsize number of 'blocks' per operation
1307 # -mbytes number of 1024*1024-byte blocks to transfer
1308 # -blocksize size of a single 'block'
1309 # -passes number of times to read mbytes
1310 # -queuedepth number of commands to queue on the target
1311 # -rwratio percentage of read to write operations
1312 # -verbose will print out statistics on each pass
1314 local cmd="$XDD -targets $xdd_ntargets $files -reqsize $xdd_reqsize \
1315 -mbytes $xdd_mbytes -blocksize $xdd_blocksize \
1316 -passes $xdd_passes -queuedepth $xdd_queuedepth \
1317 -rwratio $xdd_rwratio -verbose $xdd_custom_params"
1321 do_nodesv $clients "$cmd "
1324 [ $rc = 0 ] || error "xdd failed: $rc"