2 # -*- mode: Bash; tab-width: 4; indent-tabs-mode: t; -*-
3 # vim:shiftwidth=4:softtabstop=4:tabstop=4:
5 # Simple function used by run_*.sh scripts
10 if [ -z "${!name}" ]; then
11 echo "$0: $name must be set"
15 [ $failed ] && exit 1 || true
18 # lrepl - Lustre test Read-Eval-Print Loop.
20 # This function implements a REPL for the Lustre test framework. It
21 # doesn't exec an actual shell because the user may want to inspect
22 # variables and use functions from the test framework.
29 This is an interactive read-eval-print loop interactive shell
30 simulation that you can use to debug failing tests. You can
31 enter most bash command lines (see notes below).
33 Use this REPL to inspect variables, set them, call test
34 framework shell functions, etcetera.
36 'exit' or EOF to exit this shell.
38 set \$retcode to 0 to cause the assertion failure that
39 triggered this REPL to be ignored.
42 do_facet ost1 lctl get_param ost.*.ost.threads_*
43 do_rpc_nodes \$OSTNODES unload_modules
46 All but the last line of multi-line statements or blocks
47 must end in a backslash.
49 "Here documents" are not supported.
51 History is not supported, but command-line editing is.
55 # Prompt escapes don't work in read -p, sadly.
56 prompt=":test_${testnum:-UNKNOWN}:$(uname -n):$(basename $PWD)% "
58 # We use read -r to get close to a shell experience
59 while read -e -r -p "$prompt" rawline; do
62 # Don't want to exit-exit, just exit the REPL
64 # We need to handle continuations, and read -r doesn't do
65 # that for us. Yet we need read -r.
67 # We also use case/esac to compare lines read to "*\\"
68 # because [ "$line" = *\\ ] and variants of that don't work.
70 while read -e -r -p '> ' rawline
72 line="$line"$'\n'"$rawline"
74 # We could check for here documents by matching
75 # against *<<*, but who cares.
88 # Finally! Time to eval.
92 echo $'\n\tExiting interactive shell...\n'
96 # lassert - Lustre test framework assert
98 # Arguments: failure code, failure message, expression/statement
100 # lassert evaluates the expression given, and, if false, calls
101 # error() to trigger test failure. If REPL_ON_LASSERT is true then
102 # lassert will call lrepl() to give the user an interactive shell.
103 # If the REPL sets retcode=0 then the assertion failure will be
110 echo "checking $* ($(eval echo \""$*"\"))..."
111 eval "$@" && return 0;
113 if ${REPL_ON_LASSERT:-false}; then
114 echo "Assertion $retcode failed: $* (expanded: $(eval echo \""$*"\"))
119 error "Assertion $retcode failed: $* (expanded: $(eval echo \""$*"\"))
124 # setmodopts- set module options for subsequent calls to load_modules
126 # Usage: setmodopts module_name new_value [var_in_which_to_save_old_value]
127 # setmodopts -a module_name new_value [var_in_which_to_save_old_value]
129 # In the second usage the new value is appended to the old.
133 if [ "$1" = -a ]; then
138 local _var=MODOPTS_$1
143 # Dynamic naming of variables is a pain in bash. In ksh93 we could
144 # write "nameref opts_var=${modname}_MODOPTS" then assign directly
145 # to opts_var. Associative arrays would also help, alternatively.
146 # Alas, we're stuck with eval until all distros move to a more recent
147 # version of bash. Fortunately we don't need to eval unset and export.
149 if [ -z "$_newvalue" ]; then
155 $_append && _newvalue="$_oldvalue $_newvalue"
156 export $_var="$_newvalue"
157 echo setmodopts: ${_var}=${_newvalue}
159 [ -n "$_savevar" ] && eval $_savevar=\""$_oldvalue"\"
162 echoerr () { echo "$@" 1>&2 ; }
165 echoerr "$(date +'%F %H:%M:%S'): client load was signaled to terminate"
167 local PGID=$(ps -eo "%c %p %r" | awk "/ $PPID / {print \$3}")
174 local mpirun="$MPIRUN $MPIRUN_OPTIONS"
175 local command="$mpirun $@"
176 local mpilog=$TMP/mpi.log
179 if [ -n "$MPI_USER" -a "$MPI_USER" != root -a -n "$mpirun" ]; then
180 echo "+ chmod 0777 $MOUNT"
182 command="su $MPI_USER sh -c \"$command \""
187 eval $command 2>&1 | tee $mpilog || true
190 if [ $rc -eq 0 ] && grep -q "p4_error:" $mpilog ; then
199 for i in ${1//,/ }; do
200 if [ "$list" = "" ]; then
203 list="$list$escape $i@$NETTYPE"
209 # FIXME: all setup/cleanup can be done without rpc.sh
212 [ x$1 = x--verbose ] && verbose=true
214 export LST_SESSION=`$LST show_session 2>/dev/null | awk -F " " '{print $5}'`
215 [ "$LST_SESSION" == "" ] && return
224 lst_session_cleanup_all () {
225 local list=$(comma_list $(nodes_list))
226 do_rpc_nodes $list lst_end_session
230 lsmod | grep -q lnet_selftest && \
231 rmmod lnet_selftest > /dev/null 2>&1 || true
235 local list=$(comma_list $(nodes_list))
237 # lst end_session needs to be executed only locally
238 # i.e. on node where lst new_session was called
239 lst_end_session --verbose
240 do_rpc_nodes $list lst_cleanup
244 load_module lnet_selftest
248 local list=$(comma_list $(nodes_list))
249 do_rpc_nodes $list lst_setup
255 # Passed a single argument, strips everything off following
256 # and includes the first period.
257 # client-20.lab.whamcloud.com becomes client-20
259 echo $(sed 's/\..*//' <<< $1)
265 # Find remote nodename, stripped of any domain, etc.
266 # 'hostname -s' is easy, but not implemented on all systems
268 local rname=$(do_node $1 "uname -n" || echo -1)
269 if [[ "$rname" = "-1" ]]; then
272 echo $(short_hostname $rname)
282 echo "${var}=${!var}"
284 [ -e $MACHINEFILE ] && cat $MACHINEFILE
289 # compile dir kernel-0 ~1GB
290 # required space ~1GB * cbench_IDIRS
294 cbench_DIR=${cbench_DIR:-""}
295 cbench_IDIRS=${cbench_IDIRS:-2}
296 cbench_RUNS=${cbench_RUNS:-2}
298 print_opts cbench_DIR cbench_IDIRS cbench_RUNS
300 [ x$cbench_DIR = x ] &&
301 { skip_env "compilebench not found" && return; }
303 [ -e $cbench_DIR/compilebench ] || \
304 { skip_env "No compilebench build" && return; }
306 local space=$(df -P $dir | tail -n 1 | awk '{ print $4 }')
307 if [[ $space -le $((1024 * 1024 * cbench_IDIRS)) ]]; then
308 cbench_IDIRS=$((space / 1024 / 1024))
309 [[ $cbench_IDIRS -eq 0 ]] &&
310 skip_env "Need free space at least 1GB, have $space" &&
313 echo "free space=$space, reducing initial dirs to $cbench_IDIRS"
317 # t-f _base needs to be modifyed to set properly tdir
318 # for new "test_foo" functions names
319 # local testdir=$DIR/$tdir
320 local testdir=$dir/d0.compilebench.$$
325 local cmd="./compilebench -D $testdir -i $cbench_IDIRS \
326 -r $cbench_RUNS --makej"
335 [ $rc = 0 ] || error "compilebench failed: $rc"
341 METABENCH=${METABENCH:-$(which metabench 2> /dev/null || true)}
342 mbench_NFILES=${mbench_NFILES:-30400}
344 mbench_THREADS=${mbench_THREADS:-4}
345 mbench_OPTIONS=${mbench_OPTIONS:-}
346 mbench_CLEANUP=${mbench_CLEANUP:-true}
348 [ x$METABENCH = x ] &&
349 { skip_env "metabench not found" && return; }
352 # Need space estimation here.
354 print_opts METABENCH clients mbench_NFILES mbench_THREADS
356 local testdir=$dir/d0.metabench
358 # mpi_run uses mpiuser
361 # -C Run the file creation tests.
362 # -S Run the file stat tests.
363 # -c nfile Number of files to be used in each test.
364 # -k => dont cleanup files when finished.
365 local cmd="$METABENCH -w $testdir -c $mbench_NFILES -C -S -k $mbench_OPTIONS"
368 # find out if we need to use srun by checking $SRUN_PARTITION
369 if [ "$SRUN_PARTITION" ]; then
370 $SRUN $SRUN_OPTIONS -D $testdir -w $clients -N $num_clients \
371 -n $((num_clients * mbench_THREADS)) \
372 -p $SRUN_PARTITION -- $cmd
374 mpi_run ${MACHINEFILE_OPTION} ${MACHINEFILE} \
375 -np $((num_clients * $mbench_THREADS)) $cmd
379 if [ $rc != 0 ] ; then
380 error "metabench failed! $rc"
383 if $mbench_CLEANUP; then
386 mv $dir/d0.metabench $dir/_xxx.$(date +%s).d0.metabench
392 SIMUL=${SIMUL:=$(which simul 2> /dev/null || true)}
394 simul_THREADS=${simul_THREADS:-2}
395 simul_REP=${simul_REP:-20}
397 if [ "$NFSCLIENT" ]; then
398 skip "skipped for NFSCLIENT mode"
403 { skip_env "simul not found" && return; }
406 # Need space estimation here.
408 print_opts SIMUL clients simul_REP simul_THREADS
410 local testdir=$DIR/d0.simul
412 # mpi_run uses mpiuser
415 # -n # : repeat each test # times
416 # -N # : repeat the entire set of tests # times
418 local cmd="$SIMUL -d $testdir -n $simul_REP -N $simul_REP"
421 # find out if we need to use srun by checking $SRUN_PARTITION
422 if [ "$SRUN_PARTITION" ]; then
423 $SRUN $SRUN_OPTIONS -D $testdir -w $clients -N $num_clients \
424 -n $((num_clients * simul_THREADS)) -p $SRUN_PARTITION \
427 mpi_run ${MACHINEFILE_OPTION} ${MACHINEFILE} \
428 -np $((num_clients * simul_THREADS)) $cmd
432 if [ $rc != 0 ] ; then
433 error "simul failed! $rc"
440 MDTEST=${MDTEST:=$(which mdtest 2> /dev/null || true)}
442 mdtest_THREADS=${mdtest_THREADS:-2}
443 mdtest_nFiles=${mdtest_nFiles:-"100000"}
444 # We devide the files by number of core
445 mdtest_nFiles=$((mdtest_nFiles/mdtest_THREADS/num_clients))
446 mdtest_iteration=${mdtest_iteration:-1}
447 local mdtest_custom_params=${mdtest_custom_params:-""}
449 local type=${1:-"ssf"}
451 if [ "$NFSCLIENT" ]; then
452 skip "skipped for NFSCLIENT mode"
457 { skip_env "mdtest not found" && return; }
460 # Need space estimation here.
462 print_opts MDTEST mdtest_iteration mdtest_THREADS mdtest_nFiles
464 local testdir=$DIR/d0.mdtest
466 # mpi_run uses mpiuser
469 # -i # : repeat each test # times
471 # -n # : number of file/dir to create/stat/remove
472 # -u : each process create/stat/remove individually
474 local cmd="$MDTEST -d $testdir -i $mdtest_iteration \
475 -n $mdtest_nFiles $mdtest_custom_params"
477 [ $type = "fpp" ] && cmd="$cmd -u"
480 # find out if we need to use srun by checking $SRUN_PARTITION
481 if [ "$SRUN_PARTITION" ]; then
482 $SRUN $SRUN_OPTIONS -D $testdir -w $clients -N $num_clients \
483 -n $((num_clients * mdtest_THREADS)) \
484 -p $SRUN_PARTITION -- $cmd
486 mpi_run ${MACHINEFILE_OPTION} ${MACHINEFILE} \
487 -np $((num_clients * mdtest_THREADS)) $cmd
491 if [ $rc != 0 ] ; then
492 error "mdtest failed! $rc"
499 cnt_DIR=${cnt_DIR:-""}
500 cnt_NRUN=${cnt_NRUN:-10}
502 print_opts cnt_DIR cnt_NRUN
505 { skip_env "connectathon dir not found" && return; }
507 [ -e $cnt_DIR/runtests ] || \
508 { skip_env "No connectathon runtests found" && return; }
510 local testdir=$dir/d0.connectathon
517 # cthon options (must be in this order)
519 # -N numpasses - will be passed to the runtests script. This argument
520 # is optional. It specifies the number of times to run
523 # One of these test types
528 # -a all of the above
530 # -f a quick functionality test
534 # Include lock tests unless we're running on nfsv4
535 local fstype=$(df -TP $testdir | awk 'NR==2 {print $2}')
536 echo "$testdir: $fstype"
537 if [[ $fstype != "nfs4" ]]; then
541 for test in $tests; do
542 local cmd="./runtests -N $cnt_NRUN $test -f $testdir"
548 [ $rc = 0 ] || error "connectathon failed: $rc"
556 local type=${1:="ssf"}
558 local testdir=$dir/d0.ior.$type
559 local nfs_srvmntpt=$3
561 if [ "$NFSCLIENT" ]; then
562 [[ -n $nfs_srvmntpt ]] ||
563 { error "NFSCLIENT mode, but nfs exported dir"\
564 "is not set!" && return 1; }
567 IOR=${IOR:-$(which IOR 2> /dev/null || true)}
569 { skip_env "IOR not found" && return; }
572 ior_THREADS=${ior_THREADS:-2}
573 ior_iteration=${ior_iteration:-1}
574 ior_blockSize=${ior_blockSize:-6}
575 ior_blockUnit=${ior_blockUnit:-M} # K, M, G
576 ior_xferSize=${ior_xferSize:-1M}
577 ior_type=${ior_type:-POSIX}
578 ior_DURATION=${ior_DURATION:-30} # minutes
580 case ${ior_blockUnit} in
582 multiplier=$((1024 * 1024 * 1024))
585 multiplier=$((1024 * 1024))
590 *) error "Incorrect block unit should be one of [KMG]"
594 # calculate the space in bytes
595 local space=$(df -B 1 -P $dir | tail -n 1 | awk '{ print $4 }')
596 local total_threads=$((num_clients * ior_THREADS))
597 echo "+ $ior_blockSize * $multiplier * $total_threads "
598 if [ $((space / 2)) -le \
599 $((ior_blockSize * multiplier * total_threads)) ]; then
600 ior_blockSize=$((space / 2 / multiplier / total_threads))
601 [ $ior_blockSize -eq 0 ] && \
602 skip_env "Need free space more than $((2 * total_threads)) \
603 ${ior_blockUnit}: have $((space / multiplier))" &&
606 echo "(reduced blockSize to $ior_blockSize \
607 ${ior_blockUnit} bytes)"
610 print_opts IOR ior_THREADS ior_DURATION MACHINEFILE
613 # mpi_run uses mpiuser
615 if [ "$NFSCLIENT" ]; then
616 setstripe_nfsserver $testdir $nfs_srvmntpt -c -1 ||
617 { error "setstripe on nfsserver failed" && return 1; }
619 $LFS setstripe $testdir -c -1 ||
620 { error "setstripe failed" && return 2; }
624 # contiguous bytes to write per task (e.g.: 8, 4K, 2M, 1G)"
626 # -t N transferSize -- size of transfer in bytes (e.g.: 8, 4K, 2M, 1G)"
627 # -w writeFile -- write file"
628 # -r readFile -- read existing file"
629 # -W checkWrite -- check read after write"
630 # -C reorderTasks -- changes task ordering to n+1 ordering for readback
631 # -T maxTimeDuration -- max time in minutes to run tests"
632 # -k keepFile -- keep testFile(s) on program exit
634 local cmd="$IOR -a $ior_type -b ${ior_blockSize}${ior_blockUnit} \
635 -o $testdir/iorData -t $ior_xferSize -v -C -w -r -W \
636 -i $ior_iteration -T $ior_DURATION -k"
638 [ $type = "fpp" ] && cmd="$cmd -F"
641 # find out if we need to use srun by checking $SRUN_PARTITION
642 if [ "$SRUN_PARTITION" ]; then
643 $SRUN $SRUN_OPTIONS -D $testdir -w $clients -N $num_clients \
644 -n $((num_clients * ior_THREADS)) -p $SRUN_PARTITION \
647 mpi_run ${MACHINEFILE_OPTION} ${MACHINEFILE} \
648 -np $((num_clients * $ior_THREADS)) $cmd
652 if [ $rc != 0 ] ; then
653 error "ior failed! $rc"
660 MIB=${MIB:=$(which mib 2> /dev/null || true)}
662 mib_THREADS=${mib_THREADS:-2}
663 mib_xferSize=${mib_xferSize:-1m}
664 mib_xferLimit=${mib_xferLimit:-5000}
665 mib_timeLimit=${mib_timeLimit:-300}
667 if [ "$NFSCLIENT" ]; then
668 skip "skipped for NFSCLIENT mode"
673 { skip_env "MIB not found" && return; }
675 print_opts MIB mib_THREADS mib_xferSize mib_xferLimit mib_timeLimit \
678 local testdir=$DIR/d0.mib
680 # mpi_run uses mpiuser
682 $LFS setstripe $testdir -c -1 ||
683 { error "setstripe failed" && return 2; }
685 # -I Show intermediate values in output
686 # -H Show headers in output
687 # -L Do not issue new system calls after this many seconds
688 # -s Use system calls of this size
690 # -l Issue no more than this many system calls
691 local cmd="$MIB -t $testdir -s $mib_xferSize -l $mib_xferLimit \
692 -L $mib_timeLimit -HI -p mib.$(date +%Y%m%d%H%M%S)"
695 # find out if we need to use srun by checking $SRUN_PARTITION
696 if [ "$SRUN_PARTITION" ]; then
697 $SRUN $SRUN_OPTIONS -D $testdir -w $clients -N $num_clients \
698 -n $((num_clients * mib_THREADS)) -p $SRUN_PARTITION \
701 mpi_run ${MACHINEFILE_OPTION} ${MACHINEFILE} \
702 -np $((num_clients * mib_THREADS)) $cmd
706 if [ $rc != 0 ] ; then
707 error "mib failed! $rc"
714 CASC_RW=${CASC_RW:-$(which cascading_rw 2> /dev/null || true)}
716 casc_THREADS=${casc_THREADS:-2}
717 casc_REP=${casc_REP:-300}
719 if [ "$NFSCLIENT" ]; then
720 skip "skipped for NFSCLIENT mode"
725 { skip_env "cascading_rw not found" && return; }
728 # Need space estimation here.
730 print_opts CASC_RW clients casc_THREADS casc_REP MACHINEFILE
732 local testdir=$DIR/d0.cascading_rw
734 # mpi_run uses mpiuser
738 # -n: repeat test # times
740 local cmd="$CASC_RW -g -d $testdir -n $casc_REP"
743 mpi_run ${MACHINEFILE_OPTION} ${MACHINEFILE} \
744 -np $((num_clients * $casc_THREADS)) $cmd
747 if [ $rc != 0 ] ; then
748 error "cascading_rw failed! $rc"
753 run_write_append_truncate() {
756 write_THREADS=${write_THREADS:-8}
757 write_REP=${write_REP:-10000}
759 if [ "$NFSCLIENT" ]; then
760 skip "skipped for NFSCLIENT mode"
764 # location is lustre/tests dir
765 if ! which write_append_truncate > /dev/null 2>&1 ; then
766 skip_env "write_append_truncate not found"
771 # Need space estimation here.
773 local testdir=$DIR/d0.write_append_truncate
774 local file=$testdir/f0.wat
776 print_opts clients write_REP write_THREADS MACHINEFILE
779 # mpi_run uses mpiuser
782 local cmd="write_append_truncate -n $write_REP $file"
785 mpi_run ${MACHINEFILE_OPTION} ${MACHINEFILE} \
786 -np $((num_clients * $write_THREADS)) $cmd
789 if [ $rc != 0 ] ; then
790 error "write_append_truncate failed! $rc"
796 run_write_disjoint() {
798 WRITE_DISJOINT=${WRITE_DISJOINT:-$(which write_disjoint \
799 2> /dev/null || true)}
801 wdisjoint_THREADS=${wdisjoint_THREADS:-4}
802 wdisjoint_REP=${wdisjoint_REP:-10000}
804 if [ "$NFSCLIENT" ]; then
805 skip "skipped for NFSCLIENT mode"
809 [ x$WRITE_DISJOINT = x ] &&
810 { skip_env "write_disjoint not found" && return; }
813 # Need space estimation here.
815 print_opts WRITE_DISJOINT clients wdisjoint_THREADS wdisjoint_REP \
817 local testdir=$DIR/d0.write_disjoint
819 # mpi_run uses mpiuser
822 local cmd="$WRITE_DISJOINT -f $testdir/file -n $wdisjoint_REP"
825 mpi_run ${MACHINEFILE_OPTION} ${MACHINEFILE} \
826 -np $((num_clients * $wdisjoint_THREADS)) $cmd
829 if [ $rc != 0 ] ; then
830 error "write_disjoint failed! $rc"
835 run_parallel_grouplock() {
837 PARALLEL_GROUPLOCK=${PARALLEL_GROUPLOCK:-$(which parallel_grouplock \
838 2> /dev/null || true)}
839 parallel_grouplock_MINTASKS=${parallel_grouplock_MINTASKS:-5}
841 if [ "$NFSCLIENT" ]; then
842 skip "skipped for NFSCLIENT mode"
846 [ x$PARALLEL_GROUPLOCK = x ] &&
847 { skip "PARALLEL_GROUPLOCK not found" && return; }
849 print_opts clients parallel_grouplock_MINTASKS MACHINEFILE
851 local testdir=$DIR/d0.parallel_grouplock
853 # mpi_run uses mpiuser
859 for i in $(seq 12); do
861 local cmd="$PARALLEL_GROUPLOCK -g -v -d $testdir $subtest"
864 mpi_run ${MACHINEFILE_OPTION} ${MACHINEFILE} \
865 -np $parallel_grouplock_MINTASKS $cmd
867 if [ $rc != 0 ] ; then
868 error_noexit "parallel_grouplock subtests $subtest " \
871 echo "parallel_grouplock subtests $subtest PASS"
873 let status=$((status + rc))
874 # clear debug to collect one log per one test
875 do_nodes $(comma_list $(nodes_list)) lctl clear
877 [ $status -eq 0 ] || error "parallel_grouplock status: $status"
881 cleanup_statahead () {
888 for i in $(seq 0 $num_mntpts);do
889 zconf_umount_clients $clients ${mntpt_root}$i ||
890 error_exit "Failed to umount lustre on ${mntpt_root}$i"
896 statahead_NUMMNTPTS=${statahead_NUMMNTPTS:-5}
897 statahead_NUMFILES=${statahead_NUMFILES:-500000}
899 if [[ -n $NFSCLIENT ]]; then
900 skip "Statahead testing is not supported on NFS clients."
905 { skip_env "mdsrate not found" && return; }
907 print_opts MDSRATE clients statahead_NUMMNTPTS statahead_NUMFILES
911 # do not use default "d[0-9]*" dir name
912 # to avoid of rm $statahead_NUMFILES (500k) files in t-f cleanup
914 local testdir=$DIR/$dir
916 # cleanup only if dir exists
917 # cleanup only $statahead_NUMFILES number of files
918 # ignore the other files created by someone else
920 mdsrate_cleanup $((num_clients * 32)) $MACHINEFILE \
921 $statahead_NUMFILES $testdir 'f%%d' --ignore
924 # mpi_run uses mpiuser
927 local num_files=$statahead_NUMFILES
929 local IFree=$(inodes_available)
930 if [ $IFree -lt $num_files ]; then
936 local cmd1="${MDSRATE} ${MDSRATE_DEBUG} --mknod --dir $testdir"
937 local cmd2="--nfiles $num_files --filefmt 'f%%d'"
938 local cmd="$cmd1 $cmd2"
941 mpi_run ${MACHINEFILE_OPTION} ${MACHINEFILE} \
942 -np $((num_clients * 32)) $cmd
945 if [ $rc != 0 ] ; then
946 error "mdsrate failed to create $rc"
950 local num_mntpts=$statahead_NUMMNTPTS
951 local mntpt_root=$TMP/mntpt/lustre
952 local mntopts=$MNTOPTSTATAHEAD
954 echo "Mounting $num_mntpts lustre clients starts on $clients"
955 trap "cleanup_statahead $clients $mntpt_root $num_mntpts" EXIT ERR
956 for i in $(seq 0 $num_mntpts); do
957 zconf_mount_clients $clients ${mntpt_root}$i "$mntopts" ||
958 error_exit "Failed to mount lustre on ${mntpt_root}$i on $clients"
961 do_rpc_nodes $clients cancel_lru_locks mdc
963 do_rpc_nodes $clients do_ls $mntpt_root $num_mntpts $dir
965 mdsrate_cleanup $((num_clients * 32)) $MACHINEFILE \
966 $num_files $testdir 'f%%d' --ignore
968 # use rm instead of rmdir because of
969 # testdir could contain the files created by someone else,
970 # or by previous run where is num_files prev > num_files current
972 cleanup_statahead $clients $mntpt_root $num_mntpts
976 # fs_test.x is the default name for exe
977 FS_TEST=${FS_TEST:=$(which fs_test.x 2> /dev/null || true)}
979 local clients=${CLIENTS:-$(hostname)}
980 local testdir=$DIR/d0.fs_test
981 local file=${testdir}/fs_test
982 fs_test_threads=${fs_test_threads:-2}
983 fs_test_type=${fs_test_type:-1}
984 fs_test_nobj=${fs_test_nobj:-10}
985 fs_test_check=${fs_test_check:-3}
986 fs_test_strided=${fs_test_strided:-1}
987 fs_test_touch=${fs_test_touch:-3}
988 fs_test_supersize=${fs_test_supersize:-1}
989 fs_test_op=${fs_test_op:-write}
990 fs_test_barriers=${fs_test_barriers:-bopen,bwrite,bclose}
991 fs_test_io=${fs_test_io:-mpi}
992 fs_test_objsize=${fs_test_objsize:-100}
993 fs_test_objunit=${fs_test_objunit:-1048576} # 1 mb
994 fs_test_ndirs=${fs_test_ndirs:-80000}
997 { skip "FS_TEST not found" && return; }
999 # Space estimation in bytes
1000 local space=$(df -B 1 -P $dir | tail -n 1 | awk '{ print $4 }')
1001 local total_threads=$((num_clients * fs_test_threads))
1002 echo "+ $fs_test_objsize * $fs_test_objunit * $total_threads "
1003 if [ $((space / 2)) -le \
1004 $((fs_test_objsize * fs_test_objunit * total_threads)) ]; then
1005 fs_test_objsize=$((space / 2 / fs_test_objunit / \
1007 [ $fs_test_objsize -eq 0 ] && \
1008 skip_env "Need free space more than \
1009 $((2 * total_threads * fs_test_objunit)) \
1010 : have $((space / fs_test_objunit))" &&
1013 echo "(reduced objsize to \
1014 $((fs_test_objsize * fs_test_objunit)) bytes)"
1017 print_opts FS_TEST clients fs_test_threads fs_test_objsize MACHINEFILE
1020 # mpi_run uses mpiuser
1023 # --nodb Turn off the database code at runtime
1024 # -g --target The path to the data file
1025 # -t --type Whether to do N-N (1) or N-1 (2)
1026 # -n --nobj The number of objects written/read by each proc
1027 # -z --size The size of each object
1028 # -d ---num_nn_dirs Number of subdirectories for files
1029 # -C --check Check every byte using argument 3.
1030 # --collective Whether to use collective I/O (for N-1, mpi-io only)
1031 # -s --strided Whether to use a strided pattern (for N-1 only)
1032 # -T --touch Touch every byte using argument 3
1033 # -o --op Whether to read only (read) or write only (write)
1034 # -b --barriers When to barrier.
1035 # -i --io Use POSIX, MPI, or PLFS IO routines (mpi|posix|plfs)
1036 # -S --supersize Specify how many objects per superblock
1038 local cmd="$FS_TEST -nodb -g $file -t $fs_test_type -n $fs_test_nobj \
1039 -z $((fs_test_objsize * fs_test_objunit)) -d $fs_test_ndirs \
1040 -C $fs_test_check -collective -s $fs_test_strided \
1041 -T $fs_test_touch -o $fs_test_op -b $fs_test_barriers \
1042 -i $fs_test_io -S $fs_test_supersize"
1045 mpi_run "-np $((num_clients * fs_test_threads))" $cmd
1048 if [ $rc != 0 ] ; then
1049 error "fs_test failed! $rc"
1055 ior_mdtest_parallel() {
1063 run_mdtest $type || rc2=$?
1064 [[ $rc2 -ne 0 ]] && echo "mdtest failed with error $rc2"
1066 wait $pids || rc1=$?
1067 [[ $rc1 -ne 0 ]] && echo "ior failed with error $rc1"
1069 [[ $rc1 -ne 0 || $rc2 -ne 0 ]] && return 1