3 # Simple function used by run_*.sh scripts
8 if [ -z "${!name}" ]; then
9 echo "$0: $name must be set"
13 [ $failed ] && exit 1 || true
16 # lrepl - Lustre test Read-Eval-Print Loop.
18 # This function implements a REPL for the Lustre test framework. It
19 # doesn't exec an actual shell because the user may want to inspect
20 # variables and use functions from the test framework.
27 This is an interactive read-eval-print loop interactive shell
28 simulation that you can use to debug failing tests. You can
29 enter most bash command lines (see notes below).
31 Use this REPL to inspect variables, set them, call test
32 framework shell functions, etcetera.
34 'exit' or EOF to exit this shell.
36 set \$retcode to 0 to cause the assertion failure that
37 triggered this REPL to be ignored.
40 do_facet ost1 lctl get_param ost.*.ost.threads_*
41 do_rpc_nodes \$OSTNODES unload_modules
44 All but the last line of multi-line statements or blocks
45 must end in a backslash.
47 "Here documents" are not supported.
49 History is not supported, but command-line editing is.
53 # Prompt escapes don't work in read -p, sadly.
54 prompt=":test_${testnum:-UNKNOWN}:$(uname -n):$(basename $PWD)% "
56 # We use read -r to get close to a shell experience
57 while read -e -r -p "$prompt" rawline; do
60 # Don't want to exit-exit, just exit the REPL
62 # We need to handle continuations, and read -r doesn't do
63 # that for us. Yet we need read -r.
65 # We also use case/esac to compare lines read to "*\\"
66 # because [ "$line" = *\\ ] and variants of that don't work.
68 while read -e -r -p '> ' rawline
70 line="$line"$'\n'"$rawline"
72 # We could check for here documents by matching
73 # against *<<*, but who cares.
86 # Finally! Time to eval.
90 echo $'\n\tExiting interactive shell...\n'
94 # lassert - Lustre test framework assert
96 # Arguments: failure code, failure message, expression/statement
98 # lassert evaluates the expression given, and, if false, calls
99 # error() to trigger test failure. If REPL_ON_LASSERT is true then
100 # lassert will call lrepl() to give the user an interactive shell.
101 # If the REPL sets retcode=0 then the assertion failure will be
108 echo "checking $* ($(eval echo \""$*"\"))..."
109 eval "$@" && return 0;
111 if ${REPL_ON_LASSERT:-false}; then
112 echo "Assertion $retcode failed: $* (expanded: $(eval echo \""$*"\"))
117 error "Assertion $retcode failed: $* (expanded: $(eval echo \""$*"\"))
122 # setmodopts- set module options for subsequent calls to load_modules
124 # Usage: setmodopts module_name new_value [var_in_which_to_save_old_value]
125 # setmodopts -a module_name new_value [var_in_which_to_save_old_value]
127 # In the second usage the new value is appended to the old.
131 if [ "$1" = -a ]; then
136 local _var=MODOPTS_$1
141 # Dynamic naming of variables is a pain in bash. In ksh93 we could
142 # write "nameref opts_var=${modname}_MODOPTS" then assign directly
143 # to opts_var. Associative arrays would also help, alternatively.
144 # Alas, we're stuck with eval until all distros move to a more recent
145 # version of bash. Fortunately we don't need to eval unset and export.
147 if [ -z "$_newvalue" ]; then
153 $_append && _newvalue="$_oldvalue $_newvalue"
154 export $_var="$_newvalue"
155 echo setmodopts: ${_var}=${_newvalue}
157 [ -n "$_savevar" ] && eval $_savevar=\""$_oldvalue"\"
160 echoerr () { echo "$@" 1>&2 ; }
163 echoerr "$(date +'%F %H:%M:%S'): client load was signaled to terminate"
165 local PGID=$(ps -eo "%c %p %r" | awk "/ $PPID / {print \$3}")
172 local mpirun="$MPIRUN $MPIRUN_OPTIONS"
173 local command="$mpirun $@"
174 local mpilog=$TMP/mpi.log
177 if [ -n "$MPI_USER" -a "$MPI_USER" != root -a -n "$mpirun" ]; then
178 echo "+ chmod 0777 $MOUNT"
180 command="su $MPI_USER sh -c \"$command \""
185 eval $command 2>&1 | tee $mpilog || true
188 if [ $rc -eq 0 ] && grep -q "p4_error:" $mpilog ; then
197 for i in ${1//,/ }; do
198 if [ "$list" = "" ]; then
201 list="$list$escape $i@$NETTYPE"
207 # FIXME: all setup/cleanup can be done without rpc.sh
210 [ x$1 = x--verbose ] && verbose=true
212 export LST_SESSION=`$LST show_session 2>/dev/null | awk -F " " '{print $5}'`
213 [ "$LST_SESSION" == "" ] && return
222 lst_session_cleanup_all () {
223 local list=$(comma_list $(nodes_list))
224 do_rpc_nodes $list lst_end_session
228 lsmod | grep -q lnet_selftest && \
229 rmmod lnet_selftest > /dev/null 2>&1 || true
233 local list=$(comma_list $(nodes_list))
235 # lst end_session needs to be executed only locally
236 # i.e. on node where lst new_session was called
237 lst_end_session --verbose
238 do_rpc_nodes $list lst_cleanup
242 load_module lnet_selftest
246 local list=$(comma_list $(nodes_list))
247 do_rpc_nodes $list lst_setup
253 # Passed a single argument, strips everything off following
254 # and includes the first period.
255 # client-20.lab.whamcloud.com becomes client-20
257 echo $(sed 's/\..*//' <<< $1)
263 # Find remote nodename, stripped of any domain, etc.
264 # 'hostname -s' is easy, but not implemented on all systems
266 local rname=$(do_node $1 "uname -n" || echo -1)
267 if [[ "$rname" = "-1" ]]; then
270 echo $(short_hostname $rname)
280 echo "${var}=${!var}"
282 [ -e $MACHINEFILE ] && cat $MACHINEFILE
287 local cbench_DIR=${cbench_DIR:-""}
288 local cbench_IDIRS=${cbench_IDIRS:-2}
289 local cbench_RUNS=${cbench_RUNS:-2}
291 print_opts cbench_DIR cbench_IDIRS cbench_RUNS
293 [ x$cbench_DIR = x ] &&
294 { skip_env "compilebench not found" && return; }
296 [ -e $cbench_DIR/compilebench ] ||
297 { skip_env "No compilebench build" && return; }
300 # compile dir kernel-0 ~1GB
301 # required space ~1GB * cbench_IDIRS
302 local space=$(df -P $dir | tail -n 1 | awk '{ print $4 }')
303 if [[ $space -le $((1024 * 1024 * cbench_IDIRS)) ]]; then
304 cbench_IDIRS=$((space / 1024 / 1024))
305 [[ $cbench_IDIRS -eq 0 ]] &&
306 skip_env "Need free space at least 1GB, have $space" &&
309 echo "reducing initial dirs to $cbench_IDIRS"
311 echo "free space = $space KB"
314 # t-f _base needs to be modifyed to set properly tdir
315 # for new "test_foo" functions names
316 # local testdir=$DIR/$tdir
317 local testdir=$dir/d0.compilebench.$$
322 local cmd="./compilebench -D $testdir -i $cbench_IDIRS \
323 -r $cbench_RUNS --makej"
332 [ $rc = 0 ] || error "compilebench failed: $rc"
338 local mntpt=${2:-$MOUNT}
339 METABENCH=${METABENCH:-$(which metabench 2> /dev/null || true)}
340 mbench_NFILES=${mbench_NFILES:-30400}
342 mbench_THREADS=${mbench_THREADS:-4}
343 mbench_OPTIONS=${mbench_OPTIONS:-}
344 mbench_CLEANUP=${mbench_CLEANUP:-true}
346 [ x$METABENCH = x ] &&
347 { skip_env "metabench not found" && return; }
349 print_opts METABENCH clients mbench_NFILES mbench_THREADS
351 local testdir=$dir/d0.metabench
353 # mpi_run uses mpiuser
356 # -C Run the file creation tests. Creates zero byte files.
357 # -S Run the file stat tests.
358 # -c nfile Number of files to be used in each test.
359 # -k Cleanup files when finished.
360 local cmd="$METABENCH -w $testdir -c $mbench_NFILES -C -S $mbench_OPTIONS"
363 # find out if we need to use srun by checking $SRUN_PARTITION
364 if [ "$SRUN_PARTITION" ]; then
365 $SRUN $SRUN_OPTIONS -D $testdir -w $clients -N $num_clients \
366 -n $((num_clients * mbench_THREADS)) \
367 -p $SRUN_PARTITION -- $cmd
369 mpi_run ${MACHINEFILE_OPTION} ${MACHINEFILE} \
370 -np $((num_clients * $mbench_THREADS)) $cmd
374 if [ $rc != 0 ] ; then
375 error "metabench failed! $rc"
378 if $mbench_CLEANUP; then
381 mv $dir/d0.metabench $mntpt/_xxx.$(date +%s).d0.metabench
387 SIMUL=${SIMUL:=$(which simul 2> /dev/null || true)}
389 simul_THREADS=${simul_THREADS:-2}
390 simul_REP=${simul_REP:-20}
392 if [ "$NFSCLIENT" ]; then
393 skip "skipped for NFSCLIENT mode"
398 { skip_env "simul not found" && return; }
401 # Need space estimation here.
403 print_opts SIMUL clients simul_REP simul_THREADS
405 local testdir=$DIR/d0.simul
407 # mpi_run uses mpiuser
410 # -n # : repeat each test # times
411 # -N # : repeat the entire set of tests # times
413 local cmd="$SIMUL -d $testdir -n $simul_REP -N $simul_REP"
416 # find out if we need to use srun by checking $SRUN_PARTITION
417 if [ "$SRUN_PARTITION" ]; then
418 $SRUN $SRUN_OPTIONS -D $testdir -w $clients -N $num_clients \
419 -n $((num_clients * simul_THREADS)) -p $SRUN_PARTITION \
422 mpi_run ${MACHINEFILE_OPTION} ${MACHINEFILE} \
423 -np $((num_clients * simul_THREADS)) $cmd
427 if [ $rc != 0 ] ; then
428 error "simul failed! $rc"
435 MDTEST=${MDTEST:=$(which mdtest 2> /dev/null || true)}
437 mdtest_THREADS=${mdtest_THREADS:-2}
438 mdtest_nFiles=${mdtest_nFiles:-"100000"}
439 # We devide the files by number of core
440 mdtest_nFiles=$((mdtest_nFiles/mdtest_THREADS/num_clients))
441 mdtest_iteration=${mdtest_iteration:-1}
442 local mdtest_custom_params=${mdtest_custom_params:-""}
444 local type=${1:-"ssf"}
446 if [ "$NFSCLIENT" ]; then
447 skip "skipped for NFSCLIENT mode"
452 { skip_env "mdtest not found" && return; }
455 # Need space estimation here.
457 print_opts MDTEST mdtest_iteration mdtest_THREADS mdtest_nFiles
459 local testdir=$DIR/d0.mdtest
461 # mpi_run uses mpiuser
464 # -i # : repeat each test # times
466 # -n # : number of file/dir to create/stat/remove
467 # -u : each process create/stat/remove individually
469 local cmd="$MDTEST -d $testdir -i $mdtest_iteration \
470 -n $mdtest_nFiles $mdtest_custom_params"
472 [ $type = "fpp" ] && cmd="$cmd -u"
475 # find out if we need to use srun by checking $SRUN_PARTITION
476 if [ "$SRUN_PARTITION" ]; then
477 $SRUN $SRUN_OPTIONS -D $testdir -w $clients -N $num_clients \
478 -n $((num_clients * mdtest_THREADS)) \
479 -p $SRUN_PARTITION -- $cmd
481 mpi_run ${MACHINEFILE_OPTION} ${MACHINEFILE} \
482 -np $((num_clients * mdtest_THREADS)) $cmd
486 if [ $rc != 0 ] ; then
487 error "mdtest failed! $rc"
494 cnt_DIR=${cnt_DIR:-""}
495 cnt_NRUN=${cnt_NRUN:-10}
497 print_opts cnt_DIR cnt_NRUN
500 { skip_env "connectathon dir not found" && return; }
502 [ -e $cnt_DIR/runtests ] ||
503 { skip_env "No connectathon runtests found" && return; }
506 # "special" tests create a 30 MB file + misc. small files
507 # required space ~40 MB
508 local space=$(df -P $dir | tail -n 1 | awk '{ print $4 }')
509 if [[ $space -le $((1024 * 40)) ]]; then
510 skip_env "Need free space at least 40MB, have $space KB" &&
513 echo "free space = $space KB"
515 local testdir=$dir/d0.connectathon
522 # To run connectathon:
523 # runtests [-a|-b|-g|-s|-l] [-f|-n|-t] [-N numpasses] [test-directory]
525 # One of the following test types
530 # -a all of the above
532 # -f a quick functional test
533 # -n suppress directory operations (mkdir and rmdir)
534 # -t run with time statistics (default for basic tests)
536 # -N numpasses - specifies the number of times to run
537 # the tests. Optional.
540 # Include lock tests unless we're running on nfsv4
541 local fstype=$(df -TP $testdir | awk 'NR==2 {print $2}')
542 echo "$testdir: $fstype"
543 if [[ $fstype != "nfs4" ]]; then
547 for test in $tests; do
548 local cmd="./runtests -N $cnt_NRUN $test -f $testdir"
554 [ $rc = 0 ] || error "connectathon failed: $rc"
562 local type=${1:="ssf"}
564 local testdir=$dir/d0.ior.$type
565 local nfs_srvmntpt=$3
567 if [ "$NFSCLIENT" ]; then
568 [[ -n $nfs_srvmntpt ]] ||
569 { error "NFSCLIENT mode, but nfs exported dir"\
570 "is not set!" && return 1; }
573 IOR=${IOR:-$(which IOR 2> /dev/null || true)}
575 { skip_env "IOR not found" && return; }
578 ior_THREADS=${ior_THREADS:-2}
579 ior_iteration=${ior_iteration:-1}
580 ior_blockSize=${ior_blockSize:-6}
581 ior_blockUnit=${ior_blockUnit:-M} # K, M, G
582 ior_xferSize=${ior_xferSize:-1M}
583 ior_type=${ior_type:-POSIX}
584 ior_DURATION=${ior_DURATION:-30} # minutes
586 case ${ior_blockUnit} in
588 multiplier=$((1024 * 1024 * 1024))
591 multiplier=$((1024 * 1024))
596 *) error "Incorrect block unit should be one of [KMG]"
600 # calculate the space in bytes
601 local space=$(df -B 1 -P $dir | tail -n 1 | awk '{ print $4 }')
602 local total_threads=$((num_clients * ior_THREADS))
603 echo "+ $ior_blockSize * $multiplier * $total_threads "
604 if [ $((space / 2)) -le \
605 $((ior_blockSize * multiplier * total_threads)) ]; then
606 ior_blockSize=$((space / 2 / multiplier / total_threads))
607 [ $ior_blockSize -eq 0 ] && \
608 skip_env "Need free space more than $((2 * total_threads)) \
609 ${ior_blockUnit}: have $((space / multiplier))" &&
612 echo "(reduced blockSize to $ior_blockSize \
613 ${ior_blockUnit} bytes)"
616 print_opts IOR ior_THREADS ior_DURATION MACHINEFILE
619 # mpi_run uses mpiuser
621 if [ -z "$NFSCLIENT" ]; then
622 $LFS setstripe $testdir -c -1 ||
623 { error "setstripe failed" && return 2; }
628 # contiguous bytes to write per task (e.g.: 8, 4K, 2M, 1G)"
630 # -t N transferSize -- size of transfer in bytes (e.g.: 8, 4K, 2M, 1G)"
631 # -w writeFile -- write file"
632 # -r readFile -- read existing file"
633 # -W checkWrite -- check read after write"
634 # -C reorderTasks -- changes task ordering to n+1 ordering for readback
635 # -T maxTimeDuration -- max time in minutes to run tests"
636 # -k keepFile -- keep testFile(s) on program exit
638 local cmd="$IOR -a $ior_type -b ${ior_blockSize}${ior_blockUnit} \
639 -o $testdir/iorData -t $ior_xferSize -v -C -w -r -W \
640 -i $ior_iteration -T $ior_DURATION -k"
642 [ $type = "fpp" ] && cmd="$cmd -F"
645 # find out if we need to use srun by checking $SRUN_PARTITION
646 if [ "$SRUN_PARTITION" ]; then
647 $SRUN $SRUN_OPTIONS -D $testdir -w $clients -N $num_clients \
648 -n $((num_clients * ior_THREADS)) -p $SRUN_PARTITION \
651 mpi_run ${MACHINEFILE_OPTION} ${MACHINEFILE} \
652 -np $((num_clients * $ior_THREADS)) $cmd
656 if [ $rc != 0 ] ; then
657 error "ior failed! $rc"
664 MIB=${MIB:=$(which mib 2> /dev/null || true)}
666 mib_THREADS=${mib_THREADS:-2}
667 mib_xferSize=${mib_xferSize:-1m}
668 mib_xferLimit=${mib_xferLimit:-5000}
669 mib_timeLimit=${mib_timeLimit:-300}
671 if [ "$NFSCLIENT" ]; then
672 skip "skipped for NFSCLIENT mode"
677 { skip_env "MIB not found" && return; }
679 print_opts MIB mib_THREADS mib_xferSize mib_xferLimit mib_timeLimit \
682 local testdir=$DIR/d0.mib
684 # mpi_run uses mpiuser
686 $LFS setstripe $testdir -c -1 ||
687 { error "setstripe failed" && return 2; }
689 # -I Show intermediate values in output
690 # -H Show headers in output
691 # -L Do not issue new system calls after this many seconds
692 # -s Use system calls of this size
694 # -l Issue no more than this many system calls
695 local cmd="$MIB -t $testdir -s $mib_xferSize -l $mib_xferLimit \
696 -L $mib_timeLimit -HI -p mib.$(date +%Y%m%d%H%M%S)"
699 # find out if we need to use srun by checking $SRUN_PARTITION
700 if [ "$SRUN_PARTITION" ]; then
701 $SRUN $SRUN_OPTIONS -D $testdir -w $clients -N $num_clients \
702 -n $((num_clients * mib_THREADS)) -p $SRUN_PARTITION \
705 mpi_run ${MACHINEFILE_OPTION} ${MACHINEFILE} \
706 -np $((num_clients * mib_THREADS)) $cmd
710 if [ $rc != 0 ] ; then
711 error "mib failed! $rc"
718 CASC_RW=${CASC_RW:-$(which cascading_rw 2> /dev/null || true)}
720 casc_THREADS=${casc_THREADS:-2}
721 casc_REP=${casc_REP:-300}
723 if [ "$NFSCLIENT" ]; then
724 skip "skipped for NFSCLIENT mode"
729 { skip_env "cascading_rw not found" && return; }
732 # Need space estimation here.
734 print_opts CASC_RW clients casc_THREADS casc_REP MACHINEFILE
736 local testdir=$DIR/d0.cascading_rw
738 # mpi_run uses mpiuser
742 # -n: repeat test # times
744 local cmd="$CASC_RW -g -d $testdir -n $casc_REP"
747 mpi_run ${MACHINEFILE_OPTION} ${MACHINEFILE} \
748 -np $((num_clients * $casc_THREADS)) $cmd
751 if [ $rc != 0 ] ; then
752 error "cascading_rw failed! $rc"
757 run_write_append_truncate() {
760 write_THREADS=${write_THREADS:-8}
761 write_REP=${write_REP:-10000}
763 if [ "$NFSCLIENT" ]; then
764 skip "skipped for NFSCLIENT mode"
768 # location is lustre/tests dir
769 if ! which write_append_truncate > /dev/null 2>&1 ; then
770 skip_env "write_append_truncate not found"
775 # Need space estimation here.
777 local testdir=$DIR/d0.write_append_truncate
778 local file=$testdir/f0.wat
780 print_opts clients write_REP write_THREADS MACHINEFILE
783 # mpi_run uses mpiuser
786 local cmd="write_append_truncate -n $write_REP $file"
789 mpi_run ${MACHINEFILE_OPTION} ${MACHINEFILE} \
790 -np $((num_clients * $write_THREADS)) $cmd
793 if [ $rc != 0 ] ; then
794 error "write_append_truncate failed! $rc"
800 run_write_disjoint() {
801 if [ "$NFSCLIENT" ]; then
802 skip "skipped for NFSCLIENT mode"
806 WRITE_DISJOINT=${WRITE_DISJOINT:-$(which write_disjoint 2> /dev/null ||
809 [ x$WRITE_DISJOINT = x ] &&
810 { skip_env "write_disjoint not found" && return; }
813 wdisjoint_THREADS=${wdisjoint_THREADS:-4}
814 wdisjoint_REP=${wdisjoint_REP:-10000}
818 # Need space estimation here.
820 print_opts WRITE_DISJOINT clients wdisjoint_THREADS wdisjoint_REP \
822 local testdir=$DIR/d0.write_disjoint
824 # mpi_run uses mpiuser
827 local cmd="$WRITE_DISJOINT -f $testdir/file -n $wdisjoint_REP -m \
831 mpi_run ${MACHINEFILE_OPTION} ${MACHINEFILE} \
832 -np $((num_clients * $wdisjoint_THREADS)) $cmd
835 if [ $rc != 0 ] ; then
836 error "write_disjoint failed! $rc"
841 run_parallel_grouplock() {
843 PARALLEL_GROUPLOCK=${PARALLEL_GROUPLOCK:-$(which parallel_grouplock \
844 2> /dev/null || true)}
845 parallel_grouplock_MINTASKS=${parallel_grouplock_MINTASKS:-5}
847 if [ "$NFSCLIENT" ]; then
848 skip "skipped for NFSCLIENT mode"
852 [ x$PARALLEL_GROUPLOCK = x ] &&
853 { skip "PARALLEL_GROUPLOCK not found" && return; }
855 print_opts clients parallel_grouplock_MINTASKS MACHINEFILE
857 local testdir=$DIR/d0.parallel_grouplock
859 # mpi_run uses mpiuser
865 for i in $(seq 12); do
867 local cmd="$PARALLEL_GROUPLOCK -g -v -d $testdir $subtest"
870 mpi_run ${MACHINEFILE_OPTION} ${MACHINEFILE} \
871 -np $parallel_grouplock_MINTASKS $cmd
873 if [ $rc != 0 ] ; then
874 error_noexit "parallel_grouplock subtests $subtest " \
877 echo "parallel_grouplock subtests $subtest PASS"
879 let status=$((status + rc))
880 # clear debug to collect one log per one test
881 do_nodes $(comma_list $(nodes_list)) lctl clear
883 [ $status -eq 0 ] || error "parallel_grouplock status: $status"
887 cleanup_statahead () {
894 for i in $(seq 0 $num_mntpts);do
895 zconf_umount_clients $clients ${mntpt_root}$i ||
896 error_exit "Failed to umount lustre on ${mntpt_root}$i"
902 statahead_NUMMNTPTS=${statahead_NUMMNTPTS:-5}
903 statahead_NUMFILES=${statahead_NUMFILES:-500000}
905 if [[ -n $NFSCLIENT ]]; then
906 skip "Statahead testing is not supported on NFS clients."
911 { skip_env "mdsrate not found" && return; }
913 print_opts MDSRATE clients statahead_NUMMNTPTS statahead_NUMFILES
917 # do not use default "d[0-9]*" dir name
918 # to avoid of rm $statahead_NUMFILES (500k) files in t-f cleanup
920 local testdir=$DIR/$dir
922 # cleanup only if dir exists
923 # cleanup only $statahead_NUMFILES number of files
924 # ignore the other files created by someone else
926 mdsrate_cleanup $((num_clients * 32)) $MACHINEFILE \
927 $statahead_NUMFILES $testdir 'f%%d' --ignore
930 # mpi_run uses mpiuser
933 local num_files=$statahead_NUMFILES
935 local IFree=$(inodes_available)
936 if [ $IFree -lt $num_files ]; then
942 local cmd1="${MDSRATE} ${MDSRATE_DEBUG} --mknod --dir $testdir"
943 local cmd2="--nfiles $num_files --filefmt 'f%%d'"
944 local cmd="$cmd1 $cmd2"
947 mpi_run ${MACHINEFILE_OPTION} ${MACHINEFILE} \
948 -np $((num_clients * 32)) $cmd
951 if [ $rc != 0 ] ; then
952 error "mdsrate failed to create $rc"
956 local num_mntpts=$statahead_NUMMNTPTS
957 local mntpt_root=$TMP/mntpt/lustre
958 local mntopts=$MNTOPTSTATAHEAD
960 echo "Mounting $num_mntpts lustre clients starts on $clients"
961 trap "cleanup_statahead $clients $mntpt_root $num_mntpts" EXIT ERR
962 for i in $(seq 0 $num_mntpts); do
963 zconf_mount_clients $clients ${mntpt_root}$i "$mntopts" ||
964 error_exit "Failed to mount lustre on ${mntpt_root}$i on $clients"
967 do_rpc_nodes $clients cancel_lru_locks mdc
969 do_rpc_nodes $clients do_ls $mntpt_root $num_mntpts $dir
971 mdsrate_cleanup $((num_clients * 32)) $MACHINEFILE \
972 $num_files $testdir 'f%%d' --ignore
974 # use rm instead of rmdir because of
975 # testdir could contain the files created by someone else,
976 # or by previous run where is num_files prev > num_files current
978 cleanup_statahead $clients $mntpt_root $num_mntpts
981 cleanup_rr_alloc () {
984 local mntpt_root="$2"
985 local rr_alloc_MNTPTS="$3"
986 local mntpt_dir=$(dirname ${mntpt_root})
988 for i in $(seq 0 $((rr_alloc_MNTPTS - 1))); do
989 zconf_umount_clients $clients ${mntpt_root}$i ||
990 error_exit "Failed to umount lustre on ${mntpt_root}$i"
992 do_nodes $clients "rm -rf $mntpt_dir"
996 remote_mds_nodsh && skip "remote MDS with nodsh" && return
997 echo "===Test gives more reproduction percentage if number of "\
998 "client and ost are more. Test with 44 or more clients "\
999 "and 73 or more OSTs gives 100% reproduction rate=="
1001 RR_ALLOC=${RR_ALLOC:-$(which rr_alloc 2> /dev/null || true)}
1002 [ x$RR_ALLOC = x ] && { skip_env "rr_alloc not found" && return; }
1003 declare -a diff_max_min_arr
1004 # foeo = file on each ost. calc = calculated.
1007 local qos_prec_objs="${TMP}/qos_and_precreated_objects"
1008 local rr_alloc_NFILES=${rr_alloc_NFILES:-555}
1009 local rr_alloc_MNTPTS=${rr_alloc_MNTPTS:-11}
1010 local total_MNTPTS=$((rr_alloc_MNTPTS * num_clients))
1011 local mntpt_root="${TMP}/rr_alloc_mntpt/lustre"
1012 if [ $MDSCOUNT -lt 2 ]; then
1013 [ -e $DIR/$tdir ] || mkdir -p $DIR/$tdir
1015 [ -e $DIR/$tdir ] || $LFS mkdir -i 0 $DIR/$tdir
1017 chmod 0777 $DIR/$tdir
1018 $SETSTRIPE -c 1 /$DIR/$tdir
1020 trap "cleanup_rr_alloc $clients $mntpt_root $rr_alloc_MNTPTS" EXIT ERR
1021 for i in $(seq 0 $((rr_alloc_MNTPTS - 1))); do
1022 zconf_mount_clients $clients ${mntpt_root}$i $MOUNT_OPTS ||
1023 error_exit "Failed to mount lustre on ${mntpt_root}$i $clients"
1026 local cmd="$RR_ALLOC $mntpt_root/$tdir/ash $rr_alloc_NFILES \
1029 # Save mdt values, set threshold to 100% i.e always Round Robin,
1030 # restore the saved values again after creating files...
1031 save_lustre_params mds1 \
1032 "lov.lustre-MDT0000*.qos_threshold_rr" > $qos_prec_objs
1033 save_lustre_params mds1 \
1034 "osp.lustre-OST*-osc-MDT0000.create_count" >> $qos_prec_objs
1036 local old_create_count=$(grep -e "create_count" $qos_prec_objs |
1037 cut -d'=' -f 2 | sort -nr | head -n1)
1039 # Make sure that every osp has enough precreated objects for the file
1042 # create_count is always set to the power of 2 only, so if the files
1043 # per OST are not multiple of that then it will be set to nearest
1044 # lower power of 2. So set 'create_count' to the upper power of 2.
1046 foeo_calc=$((rr_alloc_NFILES * total_MNTPTS / OSTCOUNT))
1047 local create_count=$((2 * foeo_calc))
1048 do_facet mds1 "$LCTL set_param -n \
1049 lov.lustre-MDT0000*.qos_threshold_rr 100 \
1050 osp.lustre-OST*-osc-MDT0000.create_count $create_count" ||
1051 error "failed while setting qos_threshold_rr & creat_count"
1053 # Create few temporary files in order to increase the precreated objects
1054 # to a desired value, before starting 'rr_alloc' app. Due to default
1055 # value 32 of precreation count (OST_MIN_PRECREATE=32), precreated
1056 # objects available are 32 initially, these gets exhausted very soon,
1057 # which causes skip of some osps when very large number of files
1058 # is created per OSTs.
1059 createmany -o $DIR/$tdir/foo- $(((old_create_count + 1) * OSTCOUNT)) \
1061 rm -f /$DIR/$tdir/foo*
1063 # Check for enough precreated objects... We should not
1064 # fail here because code(osp_precreate.c) also takes care of it.
1065 # So we have good chances of passing test even if this check fails.
1067 for ost_idx in $(seq 0 $((OSTCOUNT - 1))); do
1068 [[ $(precreated_ost_obj_count $mdt_idx $ost_idx) -ge \
1069 $foeo_calc ]] || echo "Warning: test may fail because" \
1070 "of lack of precreated objects on OST${ost_idx}"
1073 if [[ $total_MNTPTS -ne 0 ]]; then
1074 # Now start the actual file creation app.
1075 mpi_run "-np $total_MNTPTS" $cmd || return
1077 error "No mount point"
1080 restore_lustre_params < $qos_prec_objs
1081 rm -f $qos_prec_objs
1083 diff_max_min_arr=($($GETSTRIPE -r $DIR/$tdir/ |
1084 grep "lmm_stripe_offset:" | awk '{print $2}' | sort -n |
1085 uniq -c | awk 'NR==1 {min=max=$1} \
1086 { $1<min ? min=$1 : min; $1>max ? max=$1 : max} \
1087 END {print max-min, max, min}'))
1091 # In-case of fairly large number of file creation using RR (round-robin)
1092 # there can be two cases in which deviation will occur than the regular
1093 # RR algo behaviour-
1094 # 1- When rr_alloc does not start right with 'lqr_start_count' reseeded,
1095 # 2- When rr_alloc does not finish with 'lqr_start_count == 0'.
1096 # So the difference of files b/w any 2 OST should not be more than 2.
1097 [[ ${diff_max_min_arr[0]} -le 2 ]] ||
1098 error "Uneven distribution detected: difference between" \
1099 "maximum files per OST (${diff_max_min_arr[1]}) and" \
1100 "minimum files per OST (${diff_max_min_arr[2]}) must not be" \
1105 # fs_test.x is the default name for exe
1106 FS_TEST=${FS_TEST:=$(which fs_test.x 2> /dev/null || true)}
1108 local clients=${CLIENTS:-$(hostname)}
1109 local testdir=$DIR/d0.fs_test
1110 local file=${testdir}/fs_test
1111 fs_test_threads=${fs_test_threads:-2}
1112 fs_test_type=${fs_test_type:-1}
1113 fs_test_nobj=${fs_test_nobj:-10}
1114 fs_test_check=${fs_test_check:-3}
1115 fs_test_strided=${fs_test_strided:-1}
1116 fs_test_touch=${fs_test_touch:-3}
1117 fs_test_supersize=${fs_test_supersize:-1}
1118 fs_test_op=${fs_test_op:-write}
1119 fs_test_barriers=${fs_test_barriers:-bopen,bwrite,bclose}
1120 fs_test_io=${fs_test_io:-mpi}
1121 fs_test_objsize=${fs_test_objsize:-100}
1122 fs_test_objunit=${fs_test_objunit:-1048576} # 1 mb
1123 fs_test_ndirs=${fs_test_ndirs:-80000}
1125 [ x$FS_TEST = x ] &&
1126 { skip "FS_TEST not found" && return; }
1128 # Space estimation in bytes
1129 local space=$(df -B 1 -P $dir | tail -n 1 | awk '{ print $4 }')
1130 local total_threads=$((num_clients * fs_test_threads))
1131 echo "+ $fs_test_objsize * $fs_test_objunit * $total_threads "
1132 if [ $((space / 2)) -le \
1133 $((fs_test_objsize * fs_test_objunit * total_threads)) ]; then
1134 fs_test_objsize=$((space / 2 / fs_test_objunit / \
1136 [ $fs_test_objsize -eq 0 ] && \
1137 skip_env "Need free space more than \
1138 $((2 * total_threads * fs_test_objunit)) \
1139 : have $((space / fs_test_objunit))" &&
1142 echo "(reduced objsize to \
1143 $((fs_test_objsize * fs_test_objunit)) bytes)"
1146 print_opts FS_TEST clients fs_test_threads fs_test_objsize MACHINEFILE
1149 # mpi_run uses mpiuser
1152 # --nodb Turn off the database code at runtime
1153 # -g --target The path to the data file
1154 # -t --type Whether to do N-N (1) or N-1 (2)
1155 # -n --nobj The number of objects written/read by each proc
1156 # -z --size The size of each object
1157 # -d ---num_nn_dirs Number of subdirectories for files
1158 # -C --check Check every byte using argument 3.
1159 # --collective Whether to use collective I/O (for N-1, mpi-io only)
1160 # -s --strided Whether to use a strided pattern (for N-1 only)
1161 # -T --touch Touch every byte using argument 3
1162 # -o --op Whether to read only (read) or write only (write)
1163 # -b --barriers When to barrier.
1164 # -i --io Use POSIX, MPI, or PLFS IO routines (mpi|posix|plfs)
1165 # -S --supersize Specify how many objects per superblock
1167 local cmd="$FS_TEST -nodb -g $file -t $fs_test_type -n $fs_test_nobj \
1168 -z $((fs_test_objsize * fs_test_objunit)) -d $fs_test_ndirs \
1169 -C $fs_test_check -collective -s $fs_test_strided \
1170 -T $fs_test_touch -o $fs_test_op -b $fs_test_barriers \
1171 -i $fs_test_io -S $fs_test_supersize"
1174 mpi_run "-np $((num_clients * fs_test_threads))" $cmd
1177 if [ $rc != 0 ] ; then
1178 error "fs_test failed! $rc"
1184 ior_mdtest_parallel() {
1192 run_mdtest $type || rc2=$?
1193 [[ $rc2 -ne 0 ]] && echo "mdtest failed with error $rc2"
1195 wait $pids || rc1=$?
1196 [[ $rc1 -ne 0 ]] && echo "ior failed with error $rc1"
1198 [[ $rc1 -ne 0 || $rc2 -ne 0 ]] && return 1
1203 FIO=${FIO:=$(which fio 2> /dev/null || true)}
1205 local clients=${CLIENTS:-$(hostname)}
1206 local fio_jobNum=${fio_jobNum:-4}
1207 local fio_jobFile=${fio_jobFile:-$TMP/fiojobfile.$(date +%s)}
1208 local fio_bs=${fio_bs:-1}
1209 local testdir=$DIR/d0.fio
1210 local file=${testdir}/fio
1212 local propagate=false
1214 [ "$SLOW" = "no" ] || runtime=600
1217 { skip_env "FIO not found" && return; }
1221 # use fio job file if exists,
1222 # create a simple one if missing
1223 if ! [ -f $fio_jobFile ]; then
1224 cat >> $fio_jobFile <<EOF
1230 filename=${file}_\$(hostname)
1232 # bs size increased by $i for each job
1233 for ((i=1; i<=fio_jobNum; i++)); do
1234 cat >> $fio_jobFile <<EOF
1237 bs=$(( fio_bs * i ))m
1240 # job file is created, should be propagated to all clients
1245 # propagate the job file if not all clients have it yet or
1246 # if the job file was created during the test run
1247 if ! do_nodesv $clients " [ -f $fio_jobFile ] " ||
1249 local cfg=$(cat $fio_jobFile)
1250 do_nodes $clients "echo \\\"$cfg\\\" > ${fio_jobFile}" ||
1251 error "job file $fio_jobFile is not propagated"
1252 do_nodesv $clients "cat ${fio_jobFile}"
1255 cmd="$FIO $fio_jobFile"
1258 log "clients: $clients $cmd"
1261 do_nodesv $clients "$cmd "
1264 [ $rc = 0 ] || error "fio failed: $rc"
1269 XDD=${XDD:=$(which xdd 2> /dev/null || true)}
1271 local clients=${CLIENTS:-$(hostname)}
1272 local testdir=$DIR/d0.xdd
1273 xdd_queuedepth=${xdd_queuedepth:-4}
1274 xdd_blocksize=${xdd_blocksize:-512}
1275 xdd_reqsize=${xdd_reqsize:-128}
1276 xdd_mbytes=${xdd_mbytes:-100}
1277 xdd_passes=${xdd_passes:-40}
1278 xdd_rwratio=${xdd_rwratio:-0}
1279 xdd_ntargets=${xdd_ntargets:-6}
1280 local xdd_custom_params=${xdd_custom_params:-"-dio -stoponerror \
1281 -maxpri -minall -noproclock -nomemlock"}
1284 { skip "XDD not found" && return; }
1286 print_opts XDD clients xdd_queuedepth xdd_blocksize xdd_reqsize \
1287 xdd_mbytes xdd_passes xdd_rwratio
1292 # Target files creates based on the given number of targets
1293 for (( i=0; i < $xdd_ntargets; i++ ))
1295 files+="${testdir}/xdd"$i" "
1298 # -targets specifies the devices or files to perform operation
1299 # -reqsize number of 'blocks' per operation
1300 # -mbytes number of 1024*1024-byte blocks to transfer
1301 # -blocksize size of a single 'block'
1302 # -passes number of times to read mbytes
1303 # -queuedepth number of commands to queue on the target
1304 # -rwratio percentage of read to write operations
1305 # -verbose will print out statistics on each pass
1307 local cmd="$XDD -targets $xdd_ntargets $files -reqsize $xdd_reqsize \
1308 -mbytes $xdd_mbytes -blocksize $xdd_blocksize \
1309 -passes $xdd_passes -queuedepth $xdd_queuedepth \
1310 -rwratio $xdd_rwratio -verbose $xdd_custom_params"
1314 do_nodesv $clients "$cmd "
1317 [ $rc = 0 ] || error "xdd failed: $rc"