}
nids_list () {
- local list
- for i in ${1//,/ }; do
- list="$list $i@$NETTYPE"
- done
- echo $list
+ local list
+ local escape="$2"
+ for i in ${1//,/ }; do
+ if [ "$list" = "" ]; then
+ list="$i@$NETTYPE"
+ else
+ list="$list$escape $i@$NETTYPE"
+ fi
+ done
+ echo $list
}
# FIXME: all setup/cleanup can be done without rpc.sh
export LST_SESSION=`$LST show_session 2>/dev/null | awk -F " " '{print $5}'`
[ "$LST_SESSION" == "" ] && return
+ $LST stop b
if $verbose; then
$LST show_error c s
fi
- $LST stop b
$LST end_session
}
echo $(sed 's/\..*//' <<< $1)
}
+###
+# short_nodename
+#
+# Find remote nodename, stripped of any domain, etc.
+# 'hostname -s' is easy, but not implemented on all systems
+short_nodename() {
+ local rname=$(do_node $1 "uname -n" || echo -1)
+ if [[ "$rname" = "-1" ]]; then
+ rname=$1
+ fi
+ echo $(short_hostname $rname)
+}
+
print_opts () {
local var
}
run_compilebench() {
+ # Space estimation:
+ # compile dir kernel-0 ~1GB
+ # required space ~1GB * cbench_IDIRS
-# Takes:
-# 5 min * cbench_RUNS
-# SLOW=no 10 mins
-# SLOW=yes 50 mins
-# Space estimation:
-# compile dir kernel-1 680MB
-# required space 680MB * cbench_IDIRS = ~7 Gb
+ local dir=${1:-$DIR}
cbench_DIR=${cbench_DIR:-""}
- cbench_IDIRS=${cbench_IDIRS:-4}
- cbench_RUNS=${cbench_RUNS:-4}
+ cbench_IDIRS=${cbench_IDIRS:-2}
+ cbench_RUNS=${cbench_RUNS:-2}
print_opts cbench_DIR cbench_IDIRS cbench_RUNS
[ -e $cbench_DIR/compilebench ] || \
{ skip_env "No compilebench build" && return; }
- local space=$(df -P $DIR | tail -n 1 | awk '{ print $4 }')
- if [ $space -le $((680 * 1024 * cbench_IDIRS)) ]; then
- cbench_IDIRS=$(( space / 680 / 1024))
- [ $cbench_IDIRS = 0 ] && \
- skip_env "Need free space atleast 680 Mb, have $space" && return
+ local space=$(df -P $dir | tail -n 1 | awk '{ print $4 }')
+ if [[ $space -le $((1024 * 1024 * cbench_IDIRS)) ]]; then
+ cbench_IDIRS=$((space / 1024 / 1024))
+ [[ $cbench_IDIRS -eq 0 ]] &&
+ skip_env "Need free space at least 1GB, have $space" &&
+ return
+
+ echo "free space=$space, reducing initial dirs to $cbench_IDIRS"
+ fi
- log free space=$space, reducing initial dirs to $cbench_IDIRS
- fi
# FIXME:
# t-f _base needs to be modifyed to set properly tdir
# for new "test_foo" functions names
# local testdir=$DIR/$tdir
- local testdir=$DIR/d0.compilebench
+ local testdir=$dir/d0.compilebench
mkdir -p $testdir
local savePWD=$PWD
mbench_NFILES=${mbench_NFILES:-30400}
# threads per client
mbench_THREADS=${mbench_THREADS:-4}
+ mbench_OPTIONS=${mbench_OPTIONS:-}
+ mbench_CLEANUP=${mbench_CLEANUP:-true}
[ x$METABENCH = x ] &&
{ skip_env "metabench not found" && return; }
# mpi_run uses mpiuser
chmod 0777 $testdir
- # -C Run the file creation tests.
- # -S Run the file stat tests.
- # -c nfile Number of files to be used in each test.
- # -k Cleanup. Remove the test directories.
- local cmd="$METABENCH -w $testdir -c $mbench_NFILES -C -S -k"
- echo "+ $cmd"
+ # -C Run the file creation tests.
+ # -S Run the file stat tests.
+ # -c nfile Number of files to be used in each test.
+ # -k => dont cleanup files when finished.
+ local cmd="$METABENCH -w $testdir -c $mbench_NFILES -C -S -k $mbench_OPTIONS"
+ echo "+ $cmd"
# find out if we need to use srun by checking $SRUN_PARTITION
if [ "$SRUN_PARTITION" ]; then
-n $((num_clients * mbench_THREADS)) \
-p $SRUN_PARTITION -- $cmd
else
- mpi_run -np $((num_clients * $mbench_THREADS)) \
- ${MACHINEFILE_OPTION} ${MACHINEFILE} $cmd
+ mpi_run ${MACHINEFILE_OPTION} ${MACHINEFILE} \
+ -np $((num_clients * $mbench_THREADS)) $cmd
fi
local rc=$?
if [ $rc != 0 ] ; then
error "metabench failed! $rc"
fi
- rm -rf $testdir
+
+ if $mbench_CLEANUP; then
+ rm -rf $testdir
+ else
+ mv $dir/d0.metabench $dir/_xxx.$(date +%s).d0.metabench
+ fi
}
run_simul() {
-n $((num_clients * simul_THREADS)) -p $SRUN_PARTITION \
-- $cmd
else
- mpi_run -np $((num_clients * simul_THREADS)) \
- ${MACHINEFILE_OPTION} ${MACHINEFILE} $cmd
+ mpi_run ${MACHINEFILE_OPTION} ${MACHINEFILE} \
+ -np $((num_clients * simul_THREADS)) $cmd
fi
local rc=$?
# We devide the files by number of core
mdtest_nFiles=$((mdtest_nFiles/mdtest_THREADS/num_clients))
mdtest_iteration=${mdtest_iteration:-1}
+ local mdtest_custom_params=${mdtest_custom_params:-""}
local type=${1:-"ssf"}
# -n # : number of file/dir to create/stat/remove
# -u : each process create/stat/remove individually
- local cmd="$MDTEST -d $testdir -i $mdtest_iteration -n $mdtest_nFiles"
- [ $type = "fpp" ] && cmd="$cmd -u"
+ local cmd="$MDTEST -d $testdir -i $mdtest_iteration \
+ -n $mdtest_nFiles $mdtest_custom_params"
+
+ [ $type = "fpp" ] && cmd="$cmd -u"
echo "+ $cmd"
# find out if we need to use srun by checking $SRUN_PARTITION
-n $((num_clients * mdtest_THREADS)) \
-p $SRUN_PARTITION -- $cmd
else
- mpi_run -np $((num_clients * mdtest_THREADS)) \
- ${MACHINEFILE_OPTION} ${MACHINEFILE} $cmd
+ mpi_run ${MACHINEFILE_OPTION} ${MACHINEFILE} \
+ -np $((num_clients * mdtest_THREADS)) $cmd
fi
local rc=$?
}
run_ior() {
- local type=${1:="ssf"}
-
- IOR=${IOR:-$(which IOR 2> /dev/null || true)}
- # threads per client
- ior_THREADS=${ior_THREADS:-2}
- ior_iteration=${ior_iteration:-1}
- ior_blockSize=${ior_blockSize:-6} # GB
- ior_xferSize=${ior_xferSize:-2m}
- ior_type=${ior_type:-POSIX}
- ior_DURATION=${ior_DURATION:-30} # minutes
-
- [ x$IOR = x ] &&
+ local type=${1:="ssf"}
+
+ IOR=${IOR:-$(which IOR 2> /dev/null || true)}
+ # threads per client
+ ior_THREADS=${ior_THREADS:-2}
+ ior_iteration=${ior_iteration:-1}
+ ior_blockSize=${ior_blockSize:-6}
+ ior_blockUnit=${ior_blockUnit:-M} # K, M, G
+ ior_xferSize=${ior_xferSize:-1M}
+ ior_type=${ior_type:-POSIX}
+ ior_DURATION=${ior_DURATION:-30} # minutes
+ local multiplier=1
+ case ${ior_blockUnit} in
+ [G])
+ multiplier=$((1024 * 1024 * 1024))
+ ;;
+ [M])
+ multiplier=$((1024 * 1024))
+ ;;
+ [K])
+ multiplier=1024
+ ;;
+ *) error "Incorrect block unit should be one of [KMG]"
+ ;;
+ esac
+
+ [ x$IOR = x ] &&
{ skip_env "IOR not found" && return; }
- local space=$(df -P $DIR | tail -n 1 | awk '{ print $4 }')
- local total_threads=$(( num_clients * ior_THREADS ))
- echo "+ $ior_blockSize * 1024 * 1024 * $total_threads "
- if [ $((space / 2)) -le \
- $(( ior_blockSize * 1024 * 1024 * total_threads)) ]; then
- echo "+ $space * 9/10 / 1024 / 1024 / $num_clients / $ior_THREADS"
- ior_blockSize=$(( space /2 /1024 /1024 / num_clients / ior_THREADS ))
- [ $ior_blockSize = 0 ] && \
- skip_env "Need free space more than $((2 * total_threads))GB: \
- $((total_threads *1024 *1024*2)), have $space" && return
-
- local reduced_size="$num_clients x $ior_THREADS x $ior_blockSize"
- echo "free space=$space, Need: $reduced_size GB"
- echo "(blockSize reduced to $ior_blockSize Gb)"
- fi
+ # calculate the space in bytes
+ local space=$(df -B 1 -P $DIR | tail -n 1 | awk '{ print $4 }')
+ local total_threads=$((num_clients * ior_THREADS))
+ echo "+ $ior_blockSize * $multiplier * $total_threads "
+ if [ $((space / 2)) -le \
+ $((ior_blockSize * multiplier * total_threads)) ]; then
+ ior_blockSize=$((space / 2 / multiplier / total_threads))
+ [ $ior_blockSize -eq 0 ] && \
+ skip_env "Need free space more than $((2 * total_threads)) \
+ ${ior_blockUnit}: have $((space / multiplier))" &&
+ return
+
+ echo "(reduced blockSize to $ior_blockSize \
+ ${ior_blockUnit} bytes)"
+ fi
print_opts IOR ior_THREADS ior_DURATION MACHINEFILE
$LFS setstripe $testdir -c -1 ||
{ error "setstripe failed" && return 2; }
fi
- #
- # -b N blockSize --
- # contiguous bytes to write per task (e.g.: 8, 4k, 2m, 1g)"
- # -o S testFileName
- # -t N transferSize -- size of transfer in bytes (e.g.: 8, 4k, 2m, 1g)"
- # -w writeFile -- write file"
- # -r readFile -- read existing file"
- # -T maxTimeDuration -- max time in minutes to run tests"
- # -k keepFile -- keep testFile(s) on program exit
-
- local cmd="$IOR -a $ior_type -b ${ior_blockSize}g -o $testdir/iorData \
- -t $ior_xferSize -v -w -r -i $ior_iteration -T $ior_DURATION -k"
- [ $type = "fpp" ] && cmd="$cmd -F"
+ #
+ # -b N blockSize --
+ # contiguous bytes to write per task (e.g.: 8, 4K, 2M, 1G)"
+ # -o S testFileName
+ # -t N transferSize -- size of transfer in bytes (e.g.: 8, 4K, 2M, 1G)"
+ # -w writeFile -- write file"
+ # -r readFile -- read existing file"
+ # -W checkWrite -- check read after write"
+ # -C reorderTasks -- changes task ordering to n+1 ordering for readback
+ # -T maxTimeDuration -- max time in minutes to run tests"
+ # -k keepFile -- keep testFile(s) on program exit
+
+ local cmd="$IOR -a $ior_type -b ${ior_blockSize}${ior_blockUnit} \
+ -o $testdir/iorData -t $ior_xferSize -v -C -w -r -W \
+ -i $ior_iteration -T $ior_DURATION -k"
+
+ [ $type = "fpp" ] && cmd="$cmd -F"
echo "+ $cmd"
# find out if we need to use srun by checking $SRUN_PARTITION
-n $((num_clients * ior_THREADS)) -p $SRUN_PARTITION \
-- $cmd
else
- mpi_run -np $((num_clients * $ior_THREADS)) \
- ${MACHINEFILE_OPTION} ${MACHINEFILE} $cmd
+ mpi_run ${MACHINEFILE_OPTION} ${MACHINEFILE} \
+ -np $((num_clients * $ior_THREADS)) $cmd
fi
local rc=$?
-n $((num_clients * mib_THREADS)) -p $SRUN_PARTITION \
-- $cmd
else
- mpi_run -np $((num_clients * mib_THREADS)) \
- ${MACHINEFILE_OPTION} ${MACHINEFILE} $cmd
+ mpi_run ${MACHINEFILE_OPTION} ${MACHINEFILE} \
+ -np $((num_clients * mib_THREADS)) $cmd
fi
local rc=$?
local cmd="$CASC_RW -g -d $testdir -n $casc_REP"
echo "+ $cmd"
- mpi_run -np $((num_clients * $casc_THREADS)) ${MACHINEFILE_OPTION} \
- ${MACHINEFILE} $cmd
+ mpi_run ${MACHINEFILE_OPTION} ${MACHINEFILE} \
+ -np $((num_clients * $casc_THREADS)) $cmd
local rc=$?
if [ $rc != 0 ] ; then
local cmd="write_append_truncate -n $write_REP $file"
echo "+ $cmd"
- mpi_run -np $((num_clients * $write_THREADS)) ${MACHINEFILE_OPTION} \
- ${MACHINEFILE} $cmd
+ mpi_run ${MACHINEFILE_OPTION} ${MACHINEFILE} \
+ -np $((num_clients * $write_THREADS)) $cmd
local rc=$?
if [ $rc != 0 ] ; then
local cmd="$WRITE_DISJOINT -f $testdir/file -n $wdisjoint_REP"
echo "+ $cmd"
- mpi_run -np $((num_clients * $wdisjoint_THREADS)) \
- ${MACHINEFILE_OPTION} ${MACHINEFILE} $cmd
+ mpi_run ${MACHINEFILE_OPTION} ${MACHINEFILE} \
+ -np $((num_clients * $wdisjoint_THREADS)) $cmd
local rc=$?
if [ $rc != 0 ] ; then
# mpi_run uses mpiuser
chmod 0777 $testdir
- do_nodes $clients "lctl set_param llite.*.max_rw_chunk=0" ||
- error "set_param max_rw_chunk=0 failed "
-
local cmd
local status=0
local subtest
local cmd="$PARALLEL_GROUPLOCK -g -v -d $testdir $subtest"
echo "+ $cmd"
- mpi_run -np $parallel_grouplock_MINTASKS ${MACHINEFILE_OPTION} \
- ${MACHINEFILE} $cmd
+ mpi_run ${MACHINEFILE_OPTION} ${MACHINEFILE} \
+ -np $parallel_grouplock_MINTASKS $cmd
local rc=$?
if [ $rc != 0 ] ; then
error_noexit "parallel_grouplock subtests $subtest " \
local cmd="$cmd1 $cmd2"
echo "+ $cmd"
- mpi_run -np $((num_clients * 32)) ${MACHINEFILE_OPTION} ${MACHINEFILE} \
- $cmd
+ mpi_run ${MACHINEFILE_OPTION} ${MACHINEFILE} \
+ -np $((num_clients * 32)) $cmd
local rc=$?
if [ $rc != 0 ] ; then
local num_mntpts=$statahead_NUMMNTPTS
local mntpt_root=$TMP/mntpt/lustre
- local mntopts=${MNTOPTSTATAHEAD:-$MOUNTOPT}
+ local mntopts=$MNTOPTSTATAHEAD
echo "Mounting $num_mntpts lustre clients starts on $clients"
trap "cleanup_statahead $clients $mntpt_root $num_mntpts" EXIT ERR
rm -rf $testdir
cleanup_statahead $clients $mntpt_root $num_mntpts
}
+
+run_fs_test() {
+ # fs_test.x is the default name for exe
+ FS_TEST=${FS_TEST:=$(which fs_test.x 2> /dev/null || true)}
+
+ local clients=${CLIENTS:-$(hostname)}
+ local testdir=$DIR/d0.fs_test
+ local file=${testdir}/fs_test
+ fs_test_threads=${fs_test_threads:-2}
+ fs_test_type=${fs_test_type:-1}
+ fs_test_nobj=${fs_test_nobj:-10}
+ fs_test_check=${fs_test_check:-3}
+ fs_test_strided=${fs_test_strided:-1}
+ fs_test_touch=${fs_test_touch:-3}
+ fs_test_supersize=${fs_test_supersize:-1}
+ fs_test_op=${fs_test_op:-write}
+ fs_test_barriers=${fs_test_barriers:-bopen,bwrite,bclose}
+ fs_test_io=${fs_test_io:-mpi}
+ fs_test_objsize=${fs_test_objsize:-100}
+ fs_test_objunit=${fs_test_objunit:-1048576} # 1 mb
+ fs_test_ndirs=${fs_test_ndirs:-80000}
+
+ [ x$FS_TEST = x ] &&
+ { skip "FS_TEST not found" && return; }
+
+ # Space estimation in bytes
+ local space=$(df -B 1 -P $dir | tail -n 1 | awk '{ print $4 }')
+ local total_threads=$((num_clients * fs_test_threads))
+ echo "+ $fs_test_objsize * $fs_test_objunit * $total_threads "
+ if [ $((space / 2)) -le \
+ $((fs_test_objsize * fs_test_objunit * total_threads)) ]; then
+ fs_test_objsize=$((space / 2 / fs_test_objunit / \
+ total_threads))
+ [ $fs_test_objsize -eq 0 ] && \
+ skip_env "Need free space more than \
+ $((2 * total_threads * fs_test_objunit)) \
+ : have $((space / fs_test_objunit))" &&
+ return
+
+ echo "(reduced objsize to \
+ $((fs_test_objsize * fs_test_objunit)) bytes)"
+ fi
+
+ print_opts FS_TEST clients fs_test_threads fs_test_objsize MACHINEFILE
+
+ mkdir -p $testdir
+ # mpi_run uses mpiuser
+ chmod 0777 $testdir
+
+ # --nodb Turn off the database code at runtime
+ # -g --target The path to the data file
+ # -t --type Whether to do N-N (1) or N-1 (2)
+ # -n --nobj The number of objects written/read by each proc
+ # -z --size The size of each object
+ # -d ---num_nn_dirs Number of subdirectories for files
+ # -C --check Check every byte using argument 3.
+ # --collective Whether to use collective I/O (for N-1, mpi-io only)
+ # -s --strided Whether to use a strided pattern (for N-1 only)
+ # -T --touch Touch every byte using argument 3
+ # -o --op Whether to read only (read) or write only (write)
+ # -b --barriers When to barrier.
+ # -i --io Use POSIX, MPI, or PLFS IO routines (mpi|posix|plfs)
+ # -S --supersize Specify how many objects per superblock
+
+ local cmd="$FS_TEST -nodb -g $file -t $fs_test_type -n $fs_test_nobj \
+ -z $((fs_test_objsize * fs_test_objunit)) -d $fs_test_ndirs \
+ -C $fs_test_check -collective -s $fs_test_strided \
+ -T $fs_test_touch -o $fs_test_op -b $fs_test_barriers \
+ -i $fs_test_io -S $fs_test_supersize"
+
+ echo "+ $cmd"
+ mpi_run "-np $((num_clients * fs_test_threads))" $cmd
+
+ local rc=$?
+ if [ $rc != 0 ] ; then
+ error "fs_test failed! $rc"
+ fi
+
+ rm -rf $testdir
+}
+
+ior_mdtest_parallel() {
+ local rc1=0
+ local rc2=0
+ local type=$1
+
+ run_ior $type &
+ local pids=$!
+
+ run_mdtest $type || rc2=$?
+ [[ $rc2 -ne 0 ]] && echo "mdtest failed with error $rc2"
+
+ wait $pids || rc1=$?
+ [[ $rc1 -ne 0 ]] && echo "ior failed with error $rc1"
+
+ [[ $rc1 -ne 0 || $rc2 -ne 0 ]] && return 1
+ return 0
+}