X-Git-Url: https://git.whamcloud.com/?p=fs%2Flustre-release.git;a=blobdiff_plain;f=lustre%2Ftests%2Ffunctions.sh;h=1807bb8d1771340156db269a6b41dd5ff7022f58;hp=5fc08eed7ad88db0aaf3ef6c0bc61617ab842168;hb=56f69854dae234fa974ab4f1dff909aea601a592;hpb=a033bd837329d1eb98d1dd71f4491f1af56a27f0 diff --git a/lustre/tests/functions.sh b/lustre/tests/functions.sh index 5fc08ee..1807bb8 100644 --- a/lustre/tests/functions.sh +++ b/lustre/tests/functions.sh @@ -51,7 +51,7 @@ lrepl() { EOF # Prompt escapes don't work in read -p, sadly. - prompt=":test_${testnum:-UNKNOWN}:$(uname -n):$(basename $PWD)% " + prompt=":${TESTNAME:-UNKNOWN}:$(uname -n):$(basename $PWD)% " # We use read -r to get close to a shell experience while read -e -r -p "$prompt" rawline; do @@ -126,35 +126,35 @@ $msg" # # In the second usage the new value is appended to the old. setmodopts() { - local _append=false + local _append=false - if [ "$1" = -a ]; then - _append=true - shift - fi + if [ "$1" = -a ]; then + _append=true + shift + fi - local _var=MODOPTS_$1 - local _newvalue=$2 - local _savevar=$3 - local _oldvalue + local _var=MODOPTS_$1 + local _newvalue=$2 + local _savevar=$3 + local _oldvalue - # Dynamic naming of variables is a pain in bash. In ksh93 we could - # write "nameref opts_var=${modname}_MODOPTS" then assign directly - # to opts_var. Associative arrays would also help, alternatively. - # Alas, we're stuck with eval until all distros move to a more recent - # version of bash. Fortunately we don't need to eval unset and export. + # Dynamic naming of variables is a pain in bash. In ksh93 we could + # write "nameref opts_var=${modname}_MODOPTS" then assign directly + # to opts_var. Associative arrays would also help, alternatively. + # Alas, we're stuck with eval until all distros move to a more recent + # version of bash. Fortunately we don't need to eval unset and export. - if [ -z "$_newvalue" ]; then - unset $_var - return 0 - fi + if [ -z "$_newvalue" ]; then + unset $_var + return 0 + fi - _oldvalue=${!var} - $_append && _newvalue="$_oldvalue $_newvalue" - export $_var="$_newvalue" - echo setmodopts: ${_var}=${_newvalue} + _oldvalue=${!_var} + $_append && _newvalue="$_oldvalue $_newvalue" + export $_var="$_newvalue" + echo setmodopts: ${_var}=${_newvalue} - [ -n "$_savevar" ] && eval $_savevar=\""$_oldvalue"\" + [ -n "$_savevar" ] && eval $_savevar=\""$_oldvalue"\" } echoerr () { echo "$@" 1>&2 ; } @@ -169,26 +169,26 @@ signaled() { } mpi_run () { - local mpirun="$MPIRUN $MPIRUN_OPTIONS" - local command="$mpirun $@" - local mpilog=$TMP/mpi.log - local rc - - if [ -n "$MPI_USER" -a "$MPI_USER" != root -a -n "$mpirun" ]; then - echo "+ chmod 0777 $MOUNT" - chmod 0777 $MOUNT - command="su $MPI_USER sh -c \"$command \"" - fi + local mpirun="$MPIRUN $MPIRUN_OPTIONS" + local command="$mpirun $@" + local mpilog=$TMP/mpi.log + local rc + + if [ -n "$MPI_USER" -a "$MPI_USER" != root -a -n "$mpirun" ]; then + echo "+ chmod 0777 $MOUNT" + chmod 0777 $MOUNT + command="su $MPI_USER bash -c \"$command \"" + fi - ls -ald $MOUNT - echo "+ $command" - eval $command 2>&1 | tee $mpilog || true + ls -ald $MOUNT + echo "+ $command" + eval $command 2>&1 | tee $mpilog || true - rc=${PIPESTATUS[0]} - if [ $rc -eq 0 ] && grep -q "p4_error:" $mpilog ; then - rc=1 - fi - return $rc + rc=${PIPESTATUS[0]} + if [ $rc -eq 0 ] && grep -q "p4_error:" $mpilog ; then + rc=1 + fi + return $rc } nids_list () { @@ -282,6 +282,25 @@ print_opts () { [ -e $MACHINEFILE ] && cat $MACHINEFILE } +is_lustre () { + [ "$(stat -f -c %T $1)" = "lustre" ] +} + +setstripe_getstripe () { + local file=$1 + shift + local params=$@ + + is_lustre $file || return 0 + + if [ -n "$params" ]; then + $LFS setstripe $params $file || + error "setstripe $params failed" + fi + $LFS getstripe $file || + error "getstripe $file failed" +} + run_compilebench() { local dir=${1:-$DIR} local cbench_DIR=${cbench_DIR:-""} @@ -291,10 +310,10 @@ run_compilebench() { print_opts cbench_DIR cbench_IDIRS cbench_RUNS [ x$cbench_DIR = x ] && - { skip_env "compilebench not found" && return; } + skip_env "compilebench not found" [ -e $cbench_DIR/compilebench ] || - { skip_env "No compilebench build" && return; } + skip_env "No compilebench build" # Space estimation: # compile dir kernel-0 ~1GB @@ -303,8 +322,7 @@ run_compilebench() { if [[ $space -le $((1024 * 1024 * cbench_IDIRS)) ]]; then cbench_IDIRS=$((space / 1024 / 1024)) [[ $cbench_IDIRS -eq 0 ]] && - skip_env "Need free space at least 1GB, have $space" && - return + skip_env "Need free space at least 1GB, have $space" echo "reducing initial dirs to $cbench_IDIRS" fi @@ -315,7 +333,8 @@ run_compilebench() { # for new "test_foo" functions names # local testdir=$DIR/$tdir local testdir=$dir/d0.compilebench.$$ - mkdir -p $testdir + test_mkdir -p $testdir + setstripe_getstripe $testdir $cbench_STRIPEPARAMS local savePWD=$PWD cd $cbench_DIR @@ -343,13 +362,14 @@ run_metabench() { mbench_OPTIONS=${mbench_OPTIONS:-} mbench_CLEANUP=${mbench_CLEANUP:-true} - [ x$METABENCH = x ] && - { skip_env "metabench not found" && return; } + [ x$METABENCH = x ] && skip_env "metabench not found" print_opts METABENCH clients mbench_NFILES mbench_THREADS local testdir=$dir/d0.metabench - mkdir -p $testdir + test_mkdir -p $testdir + setstripe_getstripe $testdir $mbench_STRIPEPARAMS + # mpi_run uses mpiuser chmod 0777 $testdir @@ -383,34 +403,30 @@ run_metabench() { } run_simul() { + SIMUL=${SIMUL:=$(which simul 2> /dev/null || true)} + [ x$SIMUL = x ] && skip_env "simul not found" + [ "$NFSCLIENT" ] && skip "skipped for NFSCLIENT mode" - SIMUL=${SIMUL:=$(which simul 2> /dev/null || true)} - # threads per client - simul_THREADS=${simul_THREADS:-2} - simul_REP=${simul_REP:-20} - - if [ "$NFSCLIENT" ]; then - skip "skipped for NFSCLIENT mode" - return - fi + # threads per client + simul_THREADS=${simul_THREADS:-2} + simul_REP=${simul_REP:-20} - [ x$SIMUL = x ] && - { skip_env "simul not found" && return; } + # FIXME + # Need space estimation here. - # FIXME - # Need space estimation here. + print_opts SIMUL clients simul_REP simul_THREADS - print_opts SIMUL clients simul_REP simul_THREADS + local testdir=$DIR/d0.simul + test_mkdir $testdir + setstripe_getstripe $testdir $simul_STRIPEPARAMS - local testdir=$DIR/d0.simul - mkdir -p $testdir - # mpi_run uses mpiuser - chmod 0777 $testdir + # mpi_run uses mpiuser + chmod 0777 $testdir - # -n # : repeat each test # times - # -N # : repeat the entire set of tests # times + # -n # : repeat each test # times + # -N # : repeat the entire set of tests # times - local cmd="$SIMUL -d $testdir -n $simul_REP -N $simul_REP" + local cmd="$SIMUL -d $testdir -n $simul_REP -N $simul_REP" echo "+ $cmd" # find out if we need to use srun by checking $SRUN_PARTITION @@ -431,40 +447,51 @@ run_simul() { } run_mdtest() { + MDTEST=${MDTEST:=$(which mdtest 2> /dev/null || true)} + [ x$MDTEST = x ] && skip_env "mdtest not found" + [ "$NFSCLIENT" ] && skip "skipped for NFSCLIENT mode" - MDTEST=${MDTEST:=$(which mdtest 2> /dev/null || true)} - # threads per client - mdtest_THREADS=${mdtest_THREADS:-2} - mdtest_nFiles=${mdtest_nFiles:-"100000"} - # We devide the files by number of core - mdtest_nFiles=$((mdtest_nFiles/mdtest_THREADS/num_clients)) - mdtest_iteration=${mdtest_iteration:-1} + # threads per client + mdtest_THREADS=${mdtest_THREADS:-2} + mdtest_nFiles=${mdtest_nFiles:-"100000"} + # We devide the files by number of core + mdtest_nFiles=$((mdtest_nFiles/mdtest_THREADS/num_clients)) + mdtest_iteration=${mdtest_iteration:-1} local mdtest_custom_params=${mdtest_custom_params:-""} + local type=${1:-"ssf"} - local type=${1:-"ssf"} + local mdtest_Nmntp=${mdtest_Nmntp:-1} - if [ "$NFSCLIENT" ]; then - skip "skipped for NFSCLIENT mode" - return - fi + if [ $type = "ssf" ] && [ $mdtest_Nmntp -ne 1 ]; then + skip "shared directory mode is not compatible" \ + "with multiple directory paths" + fi - [ x$MDTEST = x ] && - { skip_env "mdtest not found" && return; } + # FIXME + # Need space estimation here. - # FIXME - # Need space estimation here. + print_opts MDTEST mdtest_iteration mdtest_THREADS mdtest_nFiles - print_opts MDTEST mdtest_iteration mdtest_THREADS mdtest_nFiles + local testdir=$DIR/d0.mdtest + test_mkdir $testdir + setstripe_getstripe $testdir $mdtest_STRIPEPARAMS + chmod 0777 $testdir - local testdir=$DIR/d0.mdtest - mkdir -p $testdir - # mpi_run uses mpiuser - chmod 0777 $testdir + for ((i=1; i /dev/null || true)} - [ x$IOR = x ] && - { skip_env "IOR not found" && return; } + [ x$IOR = x ] && skip_env "IOR not found" # threads per client ior_THREADS=${ior_THREADS:-2} @@ -582,6 +611,7 @@ run_ior() { ior_xferSize=${ior_xferSize:-1M} ior_type=${ior_type:-POSIX} ior_DURATION=${ior_DURATION:-30} # minutes + ior_CLEANUP=${ior_CLEANUP:-true} local multiplier=1 case ${ior_blockUnit} in [G]) @@ -604,10 +634,9 @@ run_ior() { if [ $((space / 2)) -le \ $((ior_blockSize * multiplier * total_threads)) ]; then ior_blockSize=$((space / 2 / multiplier / total_threads)) - [ $ior_blockSize -eq 0 ] && \ + [ $ior_blockSize -eq 0 ] && skip_env "Need free space more than $((2 * total_threads)) \ - ${ior_blockUnit}: have $((space / multiplier))" && - return + ${ior_blockUnit}: have $((space / multiplier))" echo "(reduced blockSize to $ior_blockSize \ ${ior_blockUnit} bytes)" @@ -615,13 +644,15 @@ run_ior() { print_opts IOR ior_THREADS ior_DURATION MACHINEFILE - mkdir -p $testdir + test_mkdir -p $testdir + # mpi_run uses mpiuser chmod 0777 $testdir - if [ -z "$NFSCLIENT" ]; then - $LFS setstripe $testdir -c -1 || - { error "setstripe failed" && return 2; } - fi + [[ "$ior_stripe_params" && -z "$ior_STRIPEPARAMS" ]] && + ior_STRIPEPARAMS="$ior_stripe_params" && + echo "got deprecated ior_stripe_params,"\ + "use ior_STRIPEPARAMS instead" + setstripe_getstripe $testdir $ior_STRIPEPARAMS # # -b N blockSize -- @@ -635,9 +666,14 @@ run_ior() { # -T maxTimeDuration -- max time in minutes to run tests" # -k keepFile -- keep testFile(s) on program exit - local cmd="$IOR -a $ior_type -b ${ior_blockSize}${ior_blockUnit} \ + local cmd + if [ -n "$ior_custom_params" ]; then + cmd="$IOR -o $testdir/iorData $ior_custom_params" + else + cmd="$IOR -a $ior_type -b ${ior_blockSize}${ior_blockUnit} \ -o $testdir/iorData -t $ior_xferSize -v -C -w -r -W \ -i $ior_iteration -T $ior_DURATION -k" + fi [ $type = "fpp" ] && cmd="$cmd -F" @@ -648,52 +684,49 @@ run_ior() { -n $((num_clients * ior_THREADS)) -p $SRUN_PARTITION \ -- $cmd else + mpi_ior_custom_threads=${mpi_ior_custom_threads:-"$((num_clients * ior_THREADS))"} mpi_run ${MACHINEFILE_OPTION} ${MACHINEFILE} \ - -np $((num_clients * $ior_THREADS)) $cmd + -np $mpi_ior_custom_threads $cmd fi local rc=$? if [ $rc != 0 ] ; then error "ior failed! $rc" fi - rm -rf $testdir + $ior_CLEANUP && rm -rf $testdir || true } run_mib() { + MIB=${MIB:=$(which mib 2> /dev/null || true)} + [ "$NFSCLIENT" ] && skip "skipped for NFSCLIENT mode" + [ x$MIB = x ] && skip_env "MIB not found" - MIB=${MIB:=$(which mib 2> /dev/null || true)} - # threads per client - mib_THREADS=${mib_THREADS:-2} - mib_xferSize=${mib_xferSize:-1m} - mib_xferLimit=${mib_xferLimit:-5000} - mib_timeLimit=${mib_timeLimit:-300} + # threads per client + mib_THREADS=${mib_THREADS:-2} + mib_xferSize=${mib_xferSize:-1m} + mib_xferLimit=${mib_xferLimit:-5000} + mib_timeLimit=${mib_timeLimit:-300} + mib_STRIPEPARAMS=${mib_STRIPEPARAMS:-"-c -1"} - if [ "$NFSCLIENT" ]; then - skip "skipped for NFSCLIENT mode" - return - fi + print_opts MIB mib_THREADS mib_xferSize mib_xferLimit mib_timeLimit \ + MACHINEFILE - [ x$MIB = x ] && - { skip_env "MIB not found" && return; } + local testdir=$DIR/d0.mib + test_mkdir $testdir + setstripe_getstripe $testdir $mib_STRIPEPARAMS - print_opts MIB mib_THREADS mib_xferSize mib_xferLimit mib_timeLimit \ - MACHINEFILE + # mpi_run uses mpiuser + chmod 0777 $testdir - local testdir=$DIR/d0.mib - mkdir -p $testdir - # mpi_run uses mpiuser - chmod 0777 $testdir - $LFS setstripe $testdir -c -1 || - { error "setstripe failed" && return 2; } - # - # -I Show intermediate values in output - # -H Show headers in output - # -L Do not issue new system calls after this many seconds - # -s Use system calls of this size - # -t test dir - # -l Issue no more than this many system calls - local cmd="$MIB -t $testdir -s $mib_xferSize -l $mib_xferLimit \ - -L $mib_timeLimit -HI -p mib.$(date +%Y%m%d%H%M%S)" + # + # -I Show intermediate values in output + # -H Show headers in output + # -L Do not issue new system calls after this many seconds + # -s Use system calls of this size + # -t test dir + # -l Issue no more than this many system calls + local cmd="$MIB -t $testdir -s $mib_xferSize -l $mib_xferLimit \ + -L $mib_timeLimit -HI -p mib.$(date +%Y%m%d%H%M%S)" echo "+ $cmd" # find out if we need to use srun by checking $SRUN_PARTITION @@ -714,34 +747,30 @@ run_mib() { } run_cascading_rw() { + CASC_RW=${CASC_RW:-$(which cascading_rw 2> /dev/null || true)} + [ x$CASC_RW = x ] && skip_env "cascading_rw not found" + [ "$NFSCLIENT" ] && skip "skipped for NFSCLIENT mode" - CASC_RW=${CASC_RW:-$(which cascading_rw 2> /dev/null || true)} - # threads per client - casc_THREADS=${casc_THREADS:-2} - casc_REP=${casc_REP:-300} - - if [ "$NFSCLIENT" ]; then - skip "skipped for NFSCLIENT mode" - return - fi + # threads per client + casc_THREADS=${casc_THREADS:-2} + casc_REP=${casc_REP:-300} - [ x$CASC_RW = x ] && - { skip_env "cascading_rw not found" && return; } + # FIXME + # Need space estimation here. - # FIXME - # Need space estimation here. + print_opts CASC_RW clients casc_THREADS casc_REP MACHINEFILE - print_opts CASC_RW clients casc_THREADS casc_REP MACHINEFILE + local testdir=$DIR/d0.cascading_rw + test_mkdir $testdir + setstripe_getstripe $testdir $casc_STRIPEPARAMS - local testdir=$DIR/d0.cascading_rw - mkdir -p $testdir - # mpi_run uses mpiuser - chmod 0777 $testdir + # mpi_run uses mpiuser + chmod 0777 $testdir - # -g: debug mode - # -n: repeat test # times + # -g: debug mode + # -n: repeat test # times - local cmd="$CASC_RW -g -d $testdir -n $casc_REP" + local cmd="$CASC_RW -g -d $testdir -n $casc_REP" echo "+ $cmd" mpi_run ${MACHINEFILE_OPTION} ${MACHINEFILE} \ @@ -755,35 +784,31 @@ run_cascading_rw() { } run_write_append_truncate() { + [ "$NFSCLIENT" ] && skip "skipped for NFSCLIENT mode" + # location is lustre/tests dir + if ! which write_append_truncate > /dev/null 2>&1 ; then + skip_env "write_append_truncate not found" + fi - # threads per client - write_THREADS=${write_THREADS:-8} - write_REP=${write_REP:-10000} - - if [ "$NFSCLIENT" ]; then - skip "skipped for NFSCLIENT mode" - return - fi + # threads per client + write_THREADS=${write_THREADS:-8} + write_REP=${write_REP:-10000} - # location is lustre/tests dir - if ! which write_append_truncate > /dev/null 2>&1 ; then - skip_env "write_append_truncate not found" - return - fi + # FIXME + # Need space estimation here. - # FIXME - # Need space estimation here. + local testdir=$DIR/d0.write_append_truncate + local file=$testdir/f0.wat - local testdir=$DIR/d0.write_append_truncate - local file=$testdir/f0.wat + print_opts clients write_REP write_THREADS MACHINEFILE - print_opts clients write_REP write_THREADS MACHINEFILE + test_mkdir $testdir + # mpi_run uses mpiuser + setstripe_getstripe $testdir $write_STRIPEPARAMS - mkdir -p $testdir - # mpi_run uses mpiuser - chmod 0777 $testdir + chmod 0777 $testdir - local cmd="write_append_truncate -n $write_REP $file" + local cmd="write_append_truncate -n $write_REP $file" echo "+ $cmd" mpi_run ${MACHINEFILE_OPTION} ${MACHINEFILE} \ @@ -798,31 +823,27 @@ run_write_append_truncate() { } run_write_disjoint() { - if [ "$NFSCLIENT" ]; then - skip "skipped for NFSCLIENT mode" - return - fi - WRITE_DISJOINT=${WRITE_DISJOINT:-$(which write_disjoint 2> /dev/null || true)} - - [ x$WRITE_DISJOINT = x ] && - { skip_env "write_disjoint not found" && return; } + [ x$WRITE_DISJOINT = x ] && skip_env "write_disjoint not found" + [ "$NFSCLIENT" ] && skip "skipped for NFSCLIENT mode" # threads per client wdisjoint_THREADS=${wdisjoint_THREADS:-4} wdisjoint_REP=${wdisjoint_REP:-10000} chunk_size_limit=$1 - # FIXME - # Need space estimation here. + # FIXME + # Need space estimation here. - print_opts WRITE_DISJOINT clients wdisjoint_THREADS wdisjoint_REP \ - MACHINEFILE - local testdir=$DIR/d0.write_disjoint - mkdir -p $testdir - # mpi_run uses mpiuser - chmod 0777 $testdir + print_opts WRITE_DISJOINT clients wdisjoint_THREADS wdisjoint_REP \ + MACHINEFILE + local testdir=$DIR/d0.write_disjoint + test_mkdir $testdir + setstripe_getstripe $testdir $wdisjoint_STRIPEPARAMS + + # mpi_run uses mpiuser + chmod 0777 $testdir local cmd="$WRITE_DISJOINT -f $testdir/file -n $wdisjoint_REP -m \ $chunk_size_limit" @@ -839,29 +860,26 @@ run_write_disjoint() { } run_parallel_grouplock() { + PARALLEL_GROUPLOCK=${PARALLEL_GROUPLOCK:-$(which parallel_grouplock \ + 2> /dev/null || true)} - PARALLEL_GROUPLOCK=${PARALLEL_GROUPLOCK:-$(which parallel_grouplock \ - 2> /dev/null || true)} - parallel_grouplock_MINTASKS=${parallel_grouplock_MINTASKS:-5} + [ x$PARALLEL_GROUPLOCK = x ] && skip "PARALLEL_GROUPLOCK not found" + [ "$NFSCLIENT" ] && skip "skipped for NFSCLIENT mode" - if [ "$NFSCLIENT" ]; then - skip "skipped for NFSCLIENT mode" - return - fi + parallel_grouplock_MINTASKS=${parallel_grouplock_MINTASKS:-5} - [ x$PARALLEL_GROUPLOCK = x ] && - { skip "PARALLEL_GROUPLOCK not found" && return; } + print_opts clients parallel_grouplock_MINTASKS MACHINEFILE - print_opts clients parallel_grouplock_MINTASKS MACHINEFILE + local testdir=$DIR/d0.parallel_grouplock + test_mkdir $testdir + setstripe_getstripe $testdir $parallel_grouplock_STRIPEPARAMS - local testdir=$DIR/d0.parallel_grouplock - mkdir -p $testdir - # mpi_run uses mpiuser - chmod 0777 $testdir + # mpi_run uses mpiuser + chmod 0777 $testdir - local cmd - local status=0 - local subtest + local cmd + local status=0 + local subtest for i in $(seq 12); do subtest="-t $i" local cmd="$PARALLEL_GROUPLOCK -g -v -d $testdir $subtest" @@ -898,35 +916,33 @@ cleanup_statahead () { } run_statahead () { + if [[ -n $NFSCLIENT ]]; then + skip "Statahead testing is not supported on NFS clients." + fi + [ x$MDSRATE = x ] && skip_env "mdsrate not found" - statahead_NUMMNTPTS=${statahead_NUMMNTPTS:-5} - statahead_NUMFILES=${statahead_NUMFILES:-500000} - - if [[ -n $NFSCLIENT ]]; then - skip "Statahead testing is not supported on NFS clients." - return 0 - fi + statahead_NUMMNTPTS=${statahead_NUMMNTPTS:-5} + statahead_NUMFILES=${statahead_NUMFILES:-500000} - [ x$MDSRATE = x ] && - { skip_env "mdsrate not found" && return; } + print_opts MDSRATE clients statahead_NUMMNTPTS statahead_NUMFILES - print_opts MDSRATE clients statahead_NUMMNTPTS statahead_NUMFILES + # create large dir - # create large dir + # do not use default "d[0-9]*" dir name + # to avoid of rm $statahead_NUMFILES (500k) files in t-f cleanup + local dir=dstatahead + local testdir=$DIR/$dir - # do not use default "d[0-9]*" dir name - # to avoid of rm $statahead_NUMFILES (500k) files in t-f cleanup - local dir=dstatahead - local testdir=$DIR/$dir + # cleanup only if dir exists + # cleanup only $statahead_NUMFILES number of files + # ignore the other files created by someone else + [ -d $testdir ] && + mdsrate_cleanup $((num_clients * 32)) $MACHINEFILE \ + $statahead_NUMFILES $testdir 'f%%d' --ignore - # cleanup only if dir exists - # cleanup only $statahead_NUMFILES number of files - # ignore the other files created by someone else - [ -d $testdir ] && - mdsrate_cleanup $((num_clients * 32)) $MACHINEFILE \ - $statahead_NUMFILES $testdir 'f%%d' --ignore + test_mkdir $testdir + setstripe_getstripe $testdir $statahead_STRIPEPARAMS - mkdir -p $testdir # mpi_run uses mpiuser chmod 0777 $testdir @@ -978,6 +994,133 @@ run_statahead () { cleanup_statahead $clients $mntpt_root $num_mntpts } +cleanup_rr_alloc () { + trap 0 + local clients="$1" + local mntpt_root="$2" + local rr_alloc_MNTPTS="$3" + local mntpt_dir=$(dirname ${mntpt_root}) + + for i in $(seq 0 $((rr_alloc_MNTPTS - 1))); do + zconf_umount_clients $clients ${mntpt_root}$i || + error_exit "Failed to umount lustre on ${mntpt_root}$i" + done + do_nodes $clients "rm -rf $mntpt_dir" +} + +run_rr_alloc() { + remote_mds_nodsh && skip "remote MDS with nodsh" + echo "===Test gives more reproduction percentage if number of "\ + "client and ost are more. Test with 44 or more clients "\ + "and 73 or more OSTs gives 100% reproduction rate==" + + RR_ALLOC=${RR_ALLOC:-$(which rr_alloc 2> /dev/null || true)} + [ x$RR_ALLOC = x ] && skip_env "rr_alloc not found" + declare -a diff_max_min_arr + # foeo = file on each ost. calc = calculated. + local ost_idx + local foeo_calc + local qos_prec_objs="${TMP}/qos_and_precreated_objects" + local rr_alloc_NFILES=${rr_alloc_NFILES:-555} + local rr_alloc_MNTPTS=${rr_alloc_MNTPTS:-11} + local total_MNTPTS=$((rr_alloc_MNTPTS * num_clients)) + local mntpt_root="${TMP}/rr_alloc_mntpt/lustre" + test_mkdir $DIR/$tdir + setstripe_getstripe $DIR/$tdir $rr_alloc_STRIPEPARAMS + + chmod 0777 $DIR/$tdir + + trap "cleanup_rr_alloc $clients $mntpt_root $rr_alloc_MNTPTS" EXIT ERR + for i in $(seq 0 $((rr_alloc_MNTPTS - 1))); do + zconf_mount_clients $clients ${mntpt_root}$i $MOUNT_OPTS || + error_exit "Failed to mount lustre on ${mntpt_root}$i $clients" + done + + local cmd="$RR_ALLOC $mntpt_root/$tdir/ash $rr_alloc_NFILES \ + $num_clients" + + # Save mdt values, set threshold to 100% i.e always Round Robin, + # restore the saved values again after creating files... + save_lustre_params mds1 \ + "lod.$FSNAME-MDT0000*.qos_threshold_rr" > $qos_prec_objs + save_lustre_params mds1 \ + "osp.$FSNAME-OST*-osc-MDT0000.create_count" >> $qos_prec_objs + + local old_create_count=$(grep -e "create_count" $qos_prec_objs | + cut -d'=' -f 2 | sort -nr | head -n1) + + # Make sure that every osp has enough precreated objects for the file + # creation app + + # create_count is always set to the power of 2 only, so if the files + # per OST are not multiple of that then it will be set to nearest + # lower power of 2. So set 'create_count' to the upper power of 2. + + foeo_calc=$((rr_alloc_NFILES * total_MNTPTS / OSTCOUNT)) + local create_count=$((2 * foeo_calc)) + + # create_count accepted values: + # [OST_MIN_PRECREATE=32, OST_MAX_PRECREATE=20000] + # values exceeding OST_MAX_PRECREATE are lowered to the maximum. + [[ $create_count -lt 32 ]] && create_count=32 + local i + for i in $(seq $MDSCOUNT); do + do_facet mds$i "$LCTL set_param -n \ + lod.$FSNAME-MDT*.qos_threshold_rr=100 \ + osp.$FSNAME-OST*-osc-MDT*.create_count=$create_count" + done + + # Create few temporary files in order to increase the precreated objects + # to a desired value, before starting 'rr_alloc' app. Due to default + # value 32 of precreation count (OST_MIN_PRECREATE=32), precreated + # objects available are 32 initially, these gets exhausted very soon, + # which causes skip of some osps when very large number of files + # is created per OSTs. + createmany -o $DIR/$tdir/foo- $(((old_create_count + 1) * OSTCOUNT)) \ + > /dev/null + rm -f /$DIR/$tdir/foo* + + # Check for enough precreated objects... We should not + # fail here because code(osp_precreate.c) also takes care of it. + # So we have good chances of passing test even if this check fails. + local mdt_idx=0 + for ost_idx in $(seq 0 $((OSTCOUNT - 1))); do + [[ $(precreated_ost_obj_count $mdt_idx $ost_idx) -ge \ + $foeo_calc ]] || echo "Warning: test may fail because" \ + "of lack of precreated objects on OST${ost_idx}" + done + + if [[ $total_MNTPTS -ne 0 ]]; then + # Now start the actual file creation app. + mpi_run "-np $total_MNTPTS" $cmd || return + else + error "No mount point" + fi + + restore_lustre_params < $qos_prec_objs + rm -f $qos_prec_objs + + diff_max_min_arr=($($LFS getstripe -r $DIR/$tdir/ | + grep "lmm_stripe_offset:" | awk '{print $2}' | sort -n | + uniq -c | awk 'NR==1 {min=max=$1} \ + { $1max ? max=$1 : max} \ + END {print max-min, max, min}')) + + rm -rf $DIR/$tdir + + # In-case of fairly large number of file creation using RR (round-robin) + # there can be two cases in which deviation will occur than the regular + # RR algo behaviour- + # 1- When rr_alloc does not start right with 'lqr_start_count' reseeded, + # 2- When rr_alloc does not finish with 'lqr_start_count == 0'. + # So the difference of files b/w any 2 OST should not be more than 2. + [[ ${diff_max_min_arr[0]} -le 2 ]] || + error "Uneven distribution detected: difference between" \ + "maximum files per OST (${diff_max_min_arr[1]}) and" \ + "minimum files per OST (${diff_max_min_arr[2]}) must not be" \ + "greater than 2" +} + run_fs_test() { # fs_test.x is the default name for exe FS_TEST=${FS_TEST:=$(which fs_test.x 2> /dev/null || true)} @@ -999,8 +1142,7 @@ run_fs_test() { fs_test_objunit=${fs_test_objunit:-1048576} # 1 mb fs_test_ndirs=${fs_test_ndirs:-80000} - [ x$FS_TEST = x ] && - { skip "FS_TEST not found" && return; } + [ x$FS_TEST = x ] && skip "FS_TEST not found" # Space estimation in bytes local space=$(df -B 1 -P $dir | tail -n 1 | awk '{ print $4 }') @@ -1010,11 +1152,10 @@ run_fs_test() { $((fs_test_objsize * fs_test_objunit * total_threads)) ]; then fs_test_objsize=$((space / 2 / fs_test_objunit / \ total_threads)) - [ $fs_test_objsize -eq 0 ] && \ + [ $fs_test_objsize -eq 0 ] && skip_env "Need free space more than \ $((2 * total_threads * fs_test_objunit)) \ - : have $((space / fs_test_objunit))" && - return + : have $((space / fs_test_objunit))" echo "(reduced objsize to \ $((fs_test_objsize * fs_test_objunit)) bytes)" @@ -1022,7 +1163,9 @@ run_fs_test() { print_opts FS_TEST clients fs_test_threads fs_test_objsize MACHINEFILE - mkdir -p $testdir + test_mkdir $testdir + setstripe_getstripe $testdir $fs_test_STRIPEPARAMS + # mpi_run uses mpiuser chmod 0777 $testdir @@ -1090,10 +1233,10 @@ run_fio() { [ "$SLOW" = "no" ] || runtime=600 - [ x$FIO = x ] && - { skip_env "FIO not found" && return; } + [ x$FIO = x ] && skip_env "FIO not found" - mkdir -p $testdir + test_mkdir $testdir + setstripe_getstripe $testdir $fio_STRIPEPARAMS # use fio job file if exists, # create a simple one if missing @@ -1157,13 +1300,13 @@ run_xdd() { local xdd_custom_params=${xdd_custom_params:-"-dio -stoponerror \ -maxpri -minall -noproclock -nomemlock"} - [ x$XDD = x ] && - { skip "XDD not found" && return; } + [ x$XDD = x ] && skip "XDD not found" print_opts XDD clients xdd_queuedepth xdd_blocksize xdd_reqsize \ xdd_mbytes xdd_passes xdd_rwratio - mkdir -p $testdir + test_mkdir $testdir + setstripe_getstripe $testdir $xdd_STRIPEPARAMS local files="" # Target files creates based on the given number of targets