[fs/lustre-release.git] / lustre / tests / functions.sh

#!/bin/bash

# Simple function used by run_*.sh scripts

assert_env() {
    local failed=""
    for name in $@; do
        if [ -z "${!name}" ]; then
            echo "$0: $name must be set"
            failed=1
        fi
    done
    [ $failed ] && exit 1 || true
}

# lrepl - Lustre test Read-Eval-Print Loop.
#
# This function implements a REPL for the Lustre test framework.  It
# doesn't exec an actual shell because the user may want to inspect
# variables and use functions from the test framework.
lrepl() {
    local line
    local rawline
    local prompt

    cat <<EOF
        This is an interactive read-eval-print loop interactive shell
        simulation that you can use to debug failing tests.  You can
        enter most bash command lines (see notes below).

        Use this REPL to inspect variables, set them, call test
        framework shell functions, etcetera.

        'exit' or EOF to exit this shell.

        set \$retcode to 0 to cause the assertion failure that
        triggered this REPL to be ignored.

        Examples:
            do_facet ost1 lctl get_param ost.*.ost.threads_*
            do_rpc_nodes \$OSTNODES unload_modules

        NOTES:
            All but the last line of multi-line statements or blocks
            must end in a backslash.

            "Here documents" are not supported.

            History is not supported, but command-line editing is.

EOF

    # Prompt escapes don't work in read -p, sadly.
    prompt=":test_${testnum:-UNKNOWN}:$(uname -n):$(basename $PWD)% "

    # We use read -r to get close to a shell experience
    while read -e -r -p "$prompt" rawline; do
        line=
        case "$rawline" in
        # Don't want to exit-exit, just exit the REPL
        exit) break;;
        # We need to handle continuations, and read -r doesn't do
        # that for us.  Yet we need read -r.
        #
        # We also use case/esac to compare lines read to "*\\"
        # because [ "$line" = *\\ ] and variants of that don't work.
        *\\) line="$rawline"
            while read -e -r -p '> ' rawline
            do
                line="$line"$'\n'"$rawline"
                case "$rawline" in
                # We could check for here documents by matching
                # against *<<*, but who cares.
                *\\) continue;;
                *) break;;
                esac
            done
            ;;
        *) line=$rawline
        esac

        case "$line" in
        *\\) break;;
        esac

        # Finally!  Time to eval.
        eval "$line"
    done

    echo $'\n\tExiting interactive shell...\n'
    return 0
}

# lassert - Lustre test framework assert
#
# Arguments: failure code, failure message, expression/statement
#
# lassert evaluates the expression given, and, if false, calls
# error() to trigger test failure.  If REPL_ON_LASSERT is true then
# lassert will call lrepl() to give the user an interactive shell.
# If the REPL sets retcode=0 then the assertion failure will be
# ignored.
lassert() {
    local retcode=$1
    local msg=$2
    shift 2

    echo "checking $* ($(eval echo \""$*"\"))..."
    eval "$@" && return 0;

    if ${REPL_ON_LASSERT:-false}; then
        echo "Assertion $retcode failed: $* (expanded: $(eval echo \""$*"\"))
$msg"
        lrepl
    fi

    error "Assertion $retcode failed: $* (expanded: $(eval echo \""$*"\"))
$msg"
    return $retcode
}

# setmodopts- set module options for subsequent calls to load_modules
#
# Usage: setmodopts module_name new_value [var_in_which_to_save_old_value]
#        setmodopts -a module_name new_value [var_in_which_to_save_old_value]
#
# In the second usage the new value is appended to the old.
setmodopts() {
        local _append=false

        if [ "$1" = -a ]; then
            _append=true
            shift
        fi

        local _var=MODOPTS_$1
        local _newvalue=$2
        local _savevar=$3
        local _oldvalue

        # Dynamic naming of variables is a pain in bash.  In ksh93 we could
        # write "nameref opts_var=${modname}_MODOPTS" then assign directly
        # to opts_var.  Associative arrays would also help, alternatively.
        # Alas, we're stuck with eval until all distros move to a more recent
        # version of bash.  Fortunately we don't need to eval unset and export.

        if [ -z "$_newvalue" ]; then
            unset $_var
            return 0
        fi

        _oldvalue=${!var}
        $_append && _newvalue="$_oldvalue $_newvalue"
        export $_var="$_newvalue"
        echo setmodopts: ${_var}=${_newvalue}

        [ -n "$_savevar" ] && eval $_savevar=\""$_oldvalue"\"
}

echoerr () { echo "$@" 1>&2 ; }

signaled() {
    echoerr "$(date +'%F %H:%M:%S'): client load was signaled to terminate"

    local PGID=$(ps -eo "%c %p %r" | awk "/ $PPID / {print \$3}")
    kill -TERM -$PGID
    sleep 5
    kill -KILL -$PGID
}

mpi_run () {
    local mpirun="$MPIRUN $MPIRUN_OPTIONS"
    local command="$mpirun $@"
    local mpilog=$TMP/mpi.log
    local rc

    if [ -n "$MPI_USER" -a "$MPI_USER" != root -a -n "$mpirun" ]; then
        echo "+ chmod 0777 $MOUNT"
        chmod 0777 $MOUNT
        command="su $MPI_USER sh -c \"$command \""
    fi

    ls -ald $MOUNT
    echo "+ $command"
    eval $command 2>&1 | tee $mpilog || true

    rc=${PIPESTATUS[0]}
    if [ $rc -eq 0 ] && grep -q "p4_error:" $mpilog ; then
       rc=1
    fi
    return $rc
}

nids_list () {
        local list
        local escape="$2"
        for i in ${1//,/ }; do
                if [ "$list" = "" ]; then
                        list="$i@$NETTYPE"
                else
                        list="$list$escape $i@$NETTYPE"
                fi
        done
        echo $list
}

# FIXME: all setup/cleanup can be done without rpc.sh
lst_end_session () {
    local verbose=false
    [ x$1 = x--verbose ] && verbose=true

    export LST_SESSION=`$LST show_session 2>/dev/null | awk -F " " '{print $5}'`
    [ "$LST_SESSION" == "" ] && return

        $LST stop b
    if $verbose; then
        $LST show_error c s
    fi
    $LST end_session
}

lst_session_cleanup_all () {
    local list=$(comma_list $(nodes_list))
    do_rpc_nodes $list lst_end_session
}

lst_cleanup () {
    lsmod | grep -q lnet_selftest && \
        rmmod lnet_selftest > /dev/null 2>&1 || true
}

lst_cleanup_all () {
   local list=$(comma_list $(nodes_list))

   # lst end_session needs to be executed only locally
   # i.e. on node where lst new_session was called
   lst_end_session --verbose
   do_rpc_nodes $list lst_cleanup
}

lst_setup () {
    load_module lnet_selftest
}

lst_setup_all () {
    local list=$(comma_list $(nodes_list))
    do_rpc_nodes $list lst_setup
}

###
# short_hostname
#
# Passed a single argument, strips everything off following
# and includes the first period.
# client-20.lab.whamcloud.com becomes client-20
short_hostname() {
  echo $(sed 's/\..*//' <<< $1)
}

###
# short_nodename
#
# Find remote nodename, stripped of any domain, etc.
# 'hostname -s' is easy, but not implemented on all systems
short_nodename() {
        local rname=$(do_node $1 "uname -n" || echo -1)
        if [[ "$rname" = "-1" ]]; then
                rname=$1
        fi
        echo $(short_hostname $rname)
}

print_opts () {
    local var

    echo OPTIONS:

    for i in $@; do
        var=$i
        echo "${var}=${!var}"
    done
    [ -e $MACHINEFILE ] && cat $MACHINEFILE
}

run_compilebench() {
        local dir=${1:-$DIR}
        local cbench_DIR=${cbench_DIR:-""}
        local cbench_IDIRS=${cbench_IDIRS:-2}
        local cbench_RUNS=${cbench_RUNS:-2}

        print_opts cbench_DIR cbench_IDIRS cbench_RUNS

        [ x$cbench_DIR = x ] &&
                { skip_env "compilebench not found" && return; }

        [ -e $cbench_DIR/compilebench ] ||
                { skip_env "No compilebench build" && return; }

        # Space estimation:
        # compile dir kernel-0  ~1GB
        # required space        ~1GB * cbench_IDIRS
        local space=$(df -P $dir | tail -n 1 | awk '{ print $4 }')
        if [[ $space -le $((1024 * 1024 * cbench_IDIRS)) ]]; then
                cbench_IDIRS=$((space / 1024 / 1024))
                [[ $cbench_IDIRS -eq 0 ]] &&
                        skip_env "Need free space at least 1GB, have $space" &&
                        return

                echo "reducing initial dirs to $cbench_IDIRS"
        fi
        echo "free space = $space KB"

        # FIXME:
        # t-f _base needs to be modifyed to set properly tdir
        # for new "test_foo" functions names
        # local testdir=$DIR/$tdir
        local testdir=$dir/d0.compilebench.$$
        mkdir -p $testdir

    local savePWD=$PWD
    cd $cbench_DIR
    local cmd="./compilebench -D $testdir -i $cbench_IDIRS \
        -r $cbench_RUNS --makej"

    log "$cmd"

    local rc=0
    eval $cmd
    rc=$?

    cd $savePWD
    [ $rc = 0 ] || error "compilebench failed: $rc"
    rm -rf $testdir
}

run_metabench() {
        local dir=${1:-$DIR}
        local mntpt=${2:-$MOUNT}
        METABENCH=${METABENCH:-$(which metabench 2> /dev/null || true)}
        mbench_NFILES=${mbench_NFILES:-30400}
        # threads per client
        mbench_THREADS=${mbench_THREADS:-4}
        mbench_OPTIONS=${mbench_OPTIONS:-}
        mbench_CLEANUP=${mbench_CLEANUP:-true}

        [ x$METABENCH = x ] &&
                { skip_env "metabench not found" && return; }

        print_opts METABENCH clients mbench_NFILES mbench_THREADS

        local testdir=$dir/d0.metabench
        mkdir -p $testdir
        # mpi_run uses mpiuser
        chmod 0777 $testdir

        # -C             Run the file creation tests. Creates zero byte files.
        # -S             Run the file stat tests.
        # -c nfile       Number of files to be used in each test.
        # -k             Cleanup files when finished.
        local cmd="$METABENCH -w $testdir -c $mbench_NFILES -C -S $mbench_OPTIONS"
        echo "+ $cmd"

        # find out if we need to use srun by checking $SRUN_PARTITION
        if [ "$SRUN_PARTITION" ]; then
                $SRUN $SRUN_OPTIONS -D $testdir -w $clients -N $num_clients \
                        -n $((num_clients * mbench_THREADS)) \
                        -p $SRUN_PARTITION -- $cmd
        else
                mpi_run ${MACHINEFILE_OPTION} ${MACHINEFILE} \
                        -np $((num_clients * $mbench_THREADS)) $cmd
        fi

        local rc=$?
        if [ $rc != 0 ] ; then
                error "metabench failed! $rc"
        fi

        if $mbench_CLEANUP; then
                rm -rf $testdir
        else
                mv $dir/d0.metabench $mntpt/_xxx.$(date +%s).d0.metabench
        fi
}

run_simul() {

    SIMUL=${SIMUL:=$(which simul 2> /dev/null || true)}
    # threads per client
    simul_THREADS=${simul_THREADS:-2}
    simul_REP=${simul_REP:-20}

    if [ "$NFSCLIENT" ]; then
        skip "skipped for NFSCLIENT mode"
        return
    fi

    [ x$SIMUL = x ] &&
        { skip_env "simul not found" && return; }

    # FIXME
    # Need space estimation here.

    print_opts SIMUL clients simul_REP simul_THREADS

    local testdir=$DIR/d0.simul
    mkdir -p $testdir
    # mpi_run uses mpiuser
    chmod 0777 $testdir

    # -n # : repeat each test # times
    # -N # : repeat the entire set of tests # times

    local cmd="$SIMUL -d $testdir -n $simul_REP -N $simul_REP"

        echo "+ $cmd"
        # find out if we need to use srun by checking $SRUN_PARTITION
        if [ "$SRUN_PARTITION" ]; then
                $SRUN $SRUN_OPTIONS -D $testdir -w $clients -N $num_clients \
                        -n $((num_clients * simul_THREADS)) -p $SRUN_PARTITION \
                        -- $cmd
        else
                mpi_run ${MACHINEFILE_OPTION} ${MACHINEFILE} \
                        -np $((num_clients * simul_THREADS)) $cmd
        fi

    local rc=$?
    if [ $rc != 0 ] ; then
        error "simul failed! $rc"
    fi
    rm -rf $testdir
}

run_mdtest() {

    MDTEST=${MDTEST:=$(which mdtest 2> /dev/null || true)}
    # threads per client
    mdtest_THREADS=${mdtest_THREADS:-2}
    mdtest_nFiles=${mdtest_nFiles:-"100000"}
    # We devide the files by number of core
    mdtest_nFiles=$((mdtest_nFiles/mdtest_THREADS/num_clients))
    mdtest_iteration=${mdtest_iteration:-1}
        local mdtest_custom_params=${mdtest_custom_params:-""}

    local type=${1:-"ssf"}

    if [ "$NFSCLIENT" ]; then
        skip "skipped for NFSCLIENT mode"
        return
    fi

    [ x$MDTEST = x ] &&
        { skip_env "mdtest not found" && return; }

    # FIXME
    # Need space estimation here.

    print_opts MDTEST mdtest_iteration mdtest_THREADS mdtest_nFiles

    local testdir=$DIR/d0.mdtest
    mkdir -p $testdir
    # mpi_run uses mpiuser
    chmod 0777 $testdir

    # -i # : repeat each test # times
    # -d   : test dir
    # -n # : number of file/dir to create/stat/remove
    # -u   : each process create/stat/remove individually

        local cmd="$MDTEST -d $testdir -i $mdtest_iteration \
                -n $mdtest_nFiles $mdtest_custom_params"

        [ $type = "fpp" ] && cmd="$cmd -u"

        echo "+ $cmd"
        # find out if we need to use srun by checking $SRUN_PARTITION
        if [ "$SRUN_PARTITION" ]; then
                $SRUN $SRUN_OPTIONS -D $testdir -w $clients -N $num_clients \
                        -n $((num_clients * mdtest_THREADS)) \
                        -p $SRUN_PARTITION -- $cmd
        else
                mpi_run ${MACHINEFILE_OPTION} ${MACHINEFILE} \
                        -np $((num_clients * mdtest_THREADS)) $cmd
        fi

    local rc=$?
    if [ $rc != 0 ] ; then
        error "mdtest failed! $rc"
    fi
    rm -rf $testdir
}

run_connectathon() {
        local dir=${1:-$DIR}
        cnt_DIR=${cnt_DIR:-""}
        cnt_NRUN=${cnt_NRUN:-10}

        print_opts cnt_DIR cnt_NRUN

        [ x$cnt_DIR = x ] &&
                { skip_env "connectathon dir not found" && return; }

        [ -e $cnt_DIR/runtests ] ||
                { skip_env "No connectathon runtests found" && return; }

        # Space estimation:
        # "special" tests create a 30 MB file + misc. small files
        # required space ~40 MB
        local space=$(df -P $dir | tail -n 1 | awk '{ print $4 }')
        if [[ $space -le $((1024 * 40)) ]]; then
                skip_env "Need free space at least 40MB, have $space KB" &&
                return
        fi
        echo "free space = $space KB"

        local testdir=$dir/d0.connectathon
        mkdir -p $testdir

        local savePWD=$PWD
        cd $cnt_DIR

        #
        # To run connectathon:
        # runtests [-a|-b|-g|-s|-l] [-f|-n|-t] [-N numpasses] [test-directory]
        #
        # One of the following test types
        #    -b  basic
        #    -g  general
        #    -s  special
        #    -l  lock
        #    -a  all of the above
        #
        # -f  a quick functional test
        # -n  suppress directory operations (mkdir and rmdir)
        # -t  run with time statistics (default for basic tests)
        #
        # -N numpasses - specifies the number of times to run
        #                the tests. Optional.

        tests="-b -g -s"
        # Include lock tests unless we're running on nfsv4
        local fstype=$(df -TP $testdir | awk 'NR==2  {print $2}')
        echo "$testdir: $fstype"
        if [[ $fstype != "nfs4" ]]; then
                tests="$tests -l"
        fi
        echo "tests: $tests"
        for test in $tests; do
                local cmd="./runtests -N $cnt_NRUN $test -f $testdir"
                local rc=0

                log "$cmd"
                eval $cmd
                rc=$?
                [ $rc = 0 ] || error "connectathon failed: $rc"
        done

        cd $savePWD
        rm -rf $testdir
}

run_ior() {
        local type=${1:="ssf"}
        local dir=${2:-$DIR}
        local testdir=$dir/d0.ior.$type
        local nfs_srvmntpt=$3

        if [ "$NFSCLIENT" ]; then
                [[ -n $nfs_srvmntpt ]] ||
                        { error "NFSCLIENT mode, but nfs exported dir"\
                                "is not set!" && return 1; }
        fi

        IOR=${IOR:-$(which IOR 2> /dev/null || true)}
        [ x$IOR = x ] &&
                { skip_env "IOR not found" && return; }

        # threads per client
        ior_THREADS=${ior_THREADS:-2}
        ior_iteration=${ior_iteration:-1}
        ior_blockSize=${ior_blockSize:-6}
        ior_blockUnit=${ior_blockUnit:-M}   # K, M, G
        ior_xferSize=${ior_xferSize:-1M}
        ior_type=${ior_type:-POSIX}
        ior_DURATION=${ior_DURATION:-30}        # minutes
        local multiplier=1
        case ${ior_blockUnit} in
                [G])
                        multiplier=$((1024 * 1024 * 1024))
                        ;;
                [M])
                        multiplier=$((1024 * 1024))
                        ;;
                [K])
                        multiplier=1024
                        ;;
                *)      error "Incorrect block unit should be one of [KMG]"
                        ;;
        esac

        # calculate the space in bytes
        local space=$(df -B 1 -P $dir | tail -n 1 | awk '{ print $4 }')
        local total_threads=$((num_clients * ior_THREADS))
        echo "+ $ior_blockSize * $multiplier * $total_threads "
        if [ $((space / 2)) -le \
             $((ior_blockSize * multiplier * total_threads)) ]; then
                ior_blockSize=$((space / 2 / multiplier / total_threads))
                [ $ior_blockSize -eq 0 ] && \
                skip_env "Need free space more than $((2 * total_threads)) \
                         ${ior_blockUnit}: have $((space / multiplier))" &&
                         return

                echo "(reduced blockSize to $ior_blockSize \
                     ${ior_blockUnit} bytes)"
        fi

        print_opts IOR ior_THREADS ior_DURATION MACHINEFILE

        mkdir -p $testdir
        # mpi_run uses mpiuser
        chmod 0777 $testdir
        if [ -z "$NFSCLIENT" ]; then
                $LFS setstripe $testdir -c -1 ||
                        { error "setstripe failed" && return 2; }
        fi

        #
        # -b N  blockSize --
        #       contiguous bytes to write per task (e.g.: 8, 4K, 2M, 1G)"
        # -o S  testFileName
        # -t N  transferSize -- size of transfer in bytes (e.g.: 8, 4K, 2M, 1G)"
        # -w    writeFile -- write file"
        # -r    readFile -- read existing file"
        # -W    checkWrite -- check read after write"
        # -C    reorderTasks -- changes task ordering to n+1 ordering for readback
        # -T    maxTimeDuration -- max time in minutes to run tests"
        # -k    keepFile -- keep testFile(s) on program exit

        local cmd="$IOR -a $ior_type -b ${ior_blockSize}${ior_blockUnit} \
                -o $testdir/iorData -t $ior_xferSize -v -C -w -r -W \
                -i $ior_iteration -T $ior_DURATION -k"

        [ $type = "fpp" ] && cmd="$cmd -F"

        echo "+ $cmd"
        # find out if we need to use srun by checking $SRUN_PARTITION
        if [ "$SRUN_PARTITION" ]; then
                $SRUN $SRUN_OPTIONS -D $testdir -w $clients -N $num_clients \
                        -n $((num_clients * ior_THREADS)) -p $SRUN_PARTITION \
                        -- $cmd
        else
                mpi_run ${MACHINEFILE_OPTION} ${MACHINEFILE} \
                        -np $((num_clients * $ior_THREADS)) $cmd
        fi

    local rc=$?
    if [ $rc != 0 ] ; then
        error "ior failed! $rc"
    fi
    rm -rf $testdir
}

run_mib() {

    MIB=${MIB:=$(which mib 2> /dev/null || true)}
    # threads per client
    mib_THREADS=${mib_THREADS:-2}
    mib_xferSize=${mib_xferSize:-1m}
    mib_xferLimit=${mib_xferLimit:-5000}
    mib_timeLimit=${mib_timeLimit:-300}

    if [ "$NFSCLIENT" ]; then
        skip "skipped for NFSCLIENT mode"
        return
    fi

    [ x$MIB = x ] &&
        { skip_env "MIB not found" && return; }

    print_opts MIB mib_THREADS mib_xferSize mib_xferLimit mib_timeLimit \
        MACHINEFILE

    local testdir=$DIR/d0.mib
    mkdir -p $testdir
    # mpi_run uses mpiuser
    chmod 0777 $testdir
    $LFS setstripe $testdir -c -1 ||
        { error "setstripe failed" && return 2; }
    #
    # -I    Show intermediate values in output
    # -H    Show headers in output
    # -L    Do not issue new system calls after this many seconds
    # -s    Use system calls of this size
    # -t    test dir
    # -l    Issue no more than this many system calls
    local cmd="$MIB -t $testdir -s $mib_xferSize -l $mib_xferLimit \
        -L $mib_timeLimit -HI -p mib.$(date +%Y%m%d%H%M%S)"

        echo "+ $cmd"
        # find out if we need to use srun by checking $SRUN_PARTITION
        if [ "$SRUN_PARTITION" ]; then
                $SRUN $SRUN_OPTIONS -D $testdir -w $clients -N $num_clients \
                        -n $((num_clients * mib_THREADS)) -p $SRUN_PARTITION \
                        -- $cmd
        else
                mpi_run ${MACHINEFILE_OPTION} ${MACHINEFILE} \
                        -np $((num_clients * mib_THREADS)) $cmd
        fi

    local rc=$?
    if [ $rc != 0 ] ; then
        error "mib failed! $rc"
    fi
    rm -rf $testdir
}

run_cascading_rw() {

    CASC_RW=${CASC_RW:-$(which cascading_rw 2> /dev/null || true)}
    # threads per client
    casc_THREADS=${casc_THREADS:-2}
    casc_REP=${casc_REP:-300}

    if [ "$NFSCLIENT" ]; then
        skip "skipped for NFSCLIENT mode"
        return
    fi

    [ x$CASC_RW = x ] &&
        { skip_env "cascading_rw not found" && return; }

    # FIXME
    # Need space estimation here.

    print_opts CASC_RW clients casc_THREADS casc_REP MACHINEFILE

    local testdir=$DIR/d0.cascading_rw
    mkdir -p $testdir
    # mpi_run uses mpiuser
    chmod 0777 $testdir

    # -g: debug mode
    # -n: repeat test # times

    local cmd="$CASC_RW -g -d $testdir -n $casc_REP"

        echo "+ $cmd"
        mpi_run ${MACHINEFILE_OPTION} ${MACHINEFILE} \
                -np $((num_clients * $casc_THREADS)) $cmd

    local rc=$?
    if [ $rc != 0 ] ; then
        error "cascading_rw failed! $rc"
    fi
    rm -rf $testdir
}

run_write_append_truncate() {

    # threads per client
    write_THREADS=${write_THREADS:-8}
    write_REP=${write_REP:-10000}

    if [ "$NFSCLIENT" ]; then
        skip "skipped for NFSCLIENT mode"
        return
    fi

    # location is lustre/tests dir
    if ! which write_append_truncate > /dev/null 2>&1 ; then
        skip_env "write_append_truncate not found"
        return
    fi

    # FIXME
    # Need space estimation here.

    local testdir=$DIR/d0.write_append_truncate
    local file=$testdir/f0.wat

    print_opts clients write_REP write_THREADS MACHINEFILE

    mkdir -p $testdir
    # mpi_run uses mpiuser
    chmod 0777 $testdir

    local cmd="write_append_truncate -n $write_REP $file"

        echo "+ $cmd"
        mpi_run ${MACHINEFILE_OPTION} ${MACHINEFILE} \
                -np $((num_clients * $write_THREADS)) $cmd

    local rc=$?
    if [ $rc != 0 ] ; then
        error "write_append_truncate failed! $rc"
        return $rc
    fi
    rm -rf $testdir
}

run_write_disjoint() {
        if [ "$NFSCLIENT" ]; then
                skip "skipped for NFSCLIENT mode"
                return
        fi

        WRITE_DISJOINT=${WRITE_DISJOINT:-$(which write_disjoint 2> /dev/null ||
                                           true)}

        [ x$WRITE_DISJOINT = x ] &&
                { skip_env "write_disjoint not found" && return; }

        # threads per client
        wdisjoint_THREADS=${wdisjoint_THREADS:-4}
        wdisjoint_REP=${wdisjoint_REP:-10000}
        chunk_size_limit=$1

    # FIXME
    # Need space estimation here.

    print_opts WRITE_DISJOINT clients wdisjoint_THREADS wdisjoint_REP \
        MACHINEFILE
    local testdir=$DIR/d0.write_disjoint
    mkdir -p $testdir
    # mpi_run uses mpiuser
    chmod 0777 $testdir

        local cmd="$WRITE_DISJOINT -f $testdir/file -n $wdisjoint_REP -m \
                        $chunk_size_limit"

        echo "+ $cmd"
        mpi_run ${MACHINEFILE_OPTION} ${MACHINEFILE} \
                -np $((num_clients * $wdisjoint_THREADS)) $cmd

    local rc=$?
    if [ $rc != 0 ] ; then
        error "write_disjoint failed! $rc"
    fi
    rm -rf $testdir
}

run_parallel_grouplock() {

    PARALLEL_GROUPLOCK=${PARALLEL_GROUPLOCK:-$(which parallel_grouplock \
        2> /dev/null || true)}
    parallel_grouplock_MINTASKS=${parallel_grouplock_MINTASKS:-5}

    if [ "$NFSCLIENT" ]; then
        skip "skipped for NFSCLIENT mode"
        return
    fi

    [ x$PARALLEL_GROUPLOCK = x ] &&
        { skip "PARALLEL_GROUPLOCK not found" && return; }

    print_opts clients parallel_grouplock_MINTASKS MACHINEFILE

    local testdir=$DIR/d0.parallel_grouplock
    mkdir -p $testdir
    # mpi_run uses mpiuser
    chmod 0777 $testdir

    local cmd
    local status=0
    local subtest
        for i in $(seq 12); do
                subtest="-t $i"
                local cmd="$PARALLEL_GROUPLOCK -g -v -d $testdir $subtest"
                echo "+ $cmd"

                mpi_run ${MACHINEFILE_OPTION} ${MACHINEFILE} \
                        -np $parallel_grouplock_MINTASKS $cmd
                local rc=$?
                if [ $rc != 0 ] ; then
                        error_noexit "parallel_grouplock subtests $subtest " \
                                     "failed! $rc"
                else
                        echo "parallel_grouplock subtests $subtest PASS"
                fi
                let status=$((status + rc))
                # clear debug to collect one log per one test
                do_nodes $(comma_list $(nodes_list)) lctl clear
        done
        [ $status -eq 0 ] || error "parallel_grouplock status: $status"
        rm -rf $testdir
}

cleanup_statahead () {
    trap 0

    local clients=$1
    local mntpt_root=$2
    local num_mntpts=$3

    for i in $(seq 0 $num_mntpts);do
        zconf_umount_clients $clients ${mntpt_root}$i ||
            error_exit "Failed to umount lustre on ${mntpt_root}$i"
    done
}

run_statahead () {

    statahead_NUMMNTPTS=${statahead_NUMMNTPTS:-5}
    statahead_NUMFILES=${statahead_NUMFILES:-500000}

    if [[ -n $NFSCLIENT ]]; then
        skip "Statahead testing is not supported on NFS clients."
        return 0
    fi

    [ x$MDSRATE = x ] &&
        { skip_env "mdsrate not found" && return; }

    print_opts MDSRATE clients statahead_NUMMNTPTS statahead_NUMFILES

    # create large dir

    # do not use default "d[0-9]*" dir name
    # to avoid of rm $statahead_NUMFILES (500k) files in t-f cleanup
    local dir=dstatahead
    local testdir=$DIR/$dir

    # cleanup only if dir exists
    # cleanup only $statahead_NUMFILES number of files
    # ignore the other files created by someone else
    [ -d $testdir ] &&
        mdsrate_cleanup $((num_clients * 32)) $MACHINEFILE \
            $statahead_NUMFILES $testdir 'f%%d' --ignore

    mkdir -p $testdir
    # mpi_run uses mpiuser
    chmod 0777 $testdir

    local num_files=$statahead_NUMFILES

    local IFree=$(inodes_available)
    if [ $IFree -lt $num_files ]; then
      num_files=$IFree
    fi

    cancel_lru_locks mdc

    local cmd1="${MDSRATE} ${MDSRATE_DEBUG} --mknod --dir $testdir"
    local cmd2="--nfiles $num_files --filefmt 'f%%d'"
    local cmd="$cmd1 $cmd2"
    echo "+ $cmd"

        mpi_run ${MACHINEFILE_OPTION} ${MACHINEFILE} \
                -np $((num_clients * 32)) $cmd

    local rc=$?
    if [ $rc != 0 ] ; then
        error "mdsrate failed to create $rc"
        return $rc
    fi

    local num_mntpts=$statahead_NUMMNTPTS
    local mntpt_root=$TMP/mntpt/lustre
    local mntopts=$MNTOPTSTATAHEAD

    echo "Mounting $num_mntpts lustre clients starts on $clients"
    trap "cleanup_statahead $clients $mntpt_root $num_mntpts" EXIT ERR
    for i in $(seq 0 $num_mntpts); do
        zconf_mount_clients $clients ${mntpt_root}$i "$mntopts" ||
            error_exit "Failed to mount lustre on ${mntpt_root}$i on $clients"
    done

    do_rpc_nodes $clients cancel_lru_locks mdc

    do_rpc_nodes $clients do_ls $mntpt_root $num_mntpts $dir

    mdsrate_cleanup $((num_clients * 32)) $MACHINEFILE \
        $num_files $testdir 'f%%d' --ignore

    # use rm instead of rmdir because of
    # testdir could contain the files created by someone else,
    # or by previous run where is num_files prev > num_files current
    rm -rf $testdir
    cleanup_statahead $clients $mntpt_root $num_mntpts
}

cleanup_rr_alloc () {
        trap 0
        local clients="$1"
        local mntpt_root="$2"
        local rr_alloc_MNTPTS="$3"
        local mntpt_dir=$(dirname ${mntpt_root})

        for i in $(seq 0 $((rr_alloc_MNTPTS - 1))); do
                zconf_umount_clients $clients ${mntpt_root}$i ||
                error_exit "Failed to umount lustre on ${mntpt_root}$i"
        done
        do_nodes $clients "rm -rf $mntpt_dir"
}

run_rr_alloc() {
        remote_mds_nodsh && skip "remote MDS with nodsh" && return
        echo "===Test gives more reproduction percentage if number of "\
                "client and ost are more. Test with 44 or more clients "\
                "and 73 or more OSTs gives 100% reproduction rate=="

        RR_ALLOC=${RR_ALLOC:-$(which rr_alloc 2> /dev/null || true)}
        [ x$RR_ALLOC = x ] && { skip_env "rr_alloc not found" && return; }
        declare -a diff_max_min_arr
        # foeo = file on each ost. calc = calculated.
        local ost_idx
        local foeo_calc
        local qos_prec_objs="${TMP}/qos_and_precreated_objects"
        local rr_alloc_NFILES=${rr_alloc_NFILES:-555}
        local rr_alloc_MNTPTS=${rr_alloc_MNTPTS:-11}
        local total_MNTPTS=$((rr_alloc_MNTPTS * num_clients))
        local mntpt_root="${TMP}/rr_alloc_mntpt/lustre"
        if [ $MDSCOUNT -lt 2 ]; then
                [ -e $DIR/$tdir ] || mkdir -p $DIR/$tdir
        else
                [ -e $DIR/$tdir ] || $LFS mkdir -i 0 $DIR/$tdir
        fi
        chmod 0777 $DIR/$tdir
        $SETSTRIPE -c 1 /$DIR/$tdir

        trap "cleanup_rr_alloc $clients $mntpt_root $rr_alloc_MNTPTS" EXIT ERR
        for i in $(seq 0 $((rr_alloc_MNTPTS - 1))); do
                zconf_mount_clients $clients ${mntpt_root}$i $MOUNT_OPTS ||
                error_exit "Failed to mount lustre on ${mntpt_root}$i $clients"
        done

        local cmd="$RR_ALLOC $mntpt_root/$tdir/ash $rr_alloc_NFILES \
                $num_clients"

        # Save mdt values, set threshold to 100% i.e always Round Robin,
        # restore the saved values again after creating files...
        save_lustre_params mds1 \
                "lov.lustre-MDT0000*.qos_threshold_rr" > $qos_prec_objs
        save_lustre_params mds1 \
                "osp.lustre-OST*-osc-MDT0000.create_count" >> $qos_prec_objs

        local old_create_count=$(grep -e "create_count" $qos_prec_objs |
                cut -d'=' -f 2 | sort -nr | head -n1)

        # Make sure that every osp has enough precreated objects for the file
        # creation app

        # create_count is always set to the power of 2 only, so if the files
        # per OST are not multiple of that then it will be set to nearest
        # lower power of 2. So set 'create_count' to the upper power of 2.

        foeo_calc=$((rr_alloc_NFILES * total_MNTPTS / OSTCOUNT))
        local create_count=$((2 * foeo_calc))
        do_facet mds1 "$LCTL set_param -n \
                lov.lustre-MDT0000*.qos_threshold_rr 100 \
                osp.lustre-OST*-osc-MDT0000.create_count $create_count" ||
                error "failed while setting qos_threshold_rr & creat_count"

        # Create few temporary files in order to increase the precreated objects
        # to a desired value, before starting 'rr_alloc' app. Due to default
        # value 32 of precreation count (OST_MIN_PRECREATE=32), precreated
        # objects available are 32 initially, these gets exhausted very soon,
        # which causes skip of some osps when very large number of files
        # is created per OSTs.
        createmany -o $DIR/$tdir/foo- $(((old_create_count + 1) * OSTCOUNT)) \
                > /dev/null
        rm -f /$DIR/$tdir/foo*

        # Check for enough precreated objects... We should not
        # fail here because code(osp_precreate.c) also takes care of it.
        # So we have good chances of passing test even if this check fails.
        local mdt_idx=0
        for ost_idx in $(seq 0 $((OSTCOUNT - 1))); do
                [[ $(precreated_ost_obj_count $mdt_idx $ost_idx) -ge \
                        $foeo_calc ]] || echo "Warning: test may fail because" \
                        "of lack of precreated objects on OST${ost_idx}"
        done

        if [[ $total_MNTPTS -ne 0 ]]; then
                # Now start the actual file creation app.
                mpi_run "-np $total_MNTPTS" $cmd || return
        else
                error "No mount point"
        fi

        restore_lustre_params < $qos_prec_objs
        rm -f $qos_prec_objs

        diff_max_min_arr=($($GETSTRIPE -r $DIR/$tdir/ |
                grep "lmm_stripe_offset:" | awk '{print $2}' | sort -n |
                uniq -c | awk 'NR==1 {min=max=$1} \
                { $1<min ? min=$1 : min; $1>max ? max=$1 : max} \
                END {print max-min, max, min}'))

        rm -rf $DIR/$tdir

        # In-case of fairly large number of file creation using RR (round-robin)
        # there can be two cases in which deviation will occur than the regular
        # RR algo behaviour-
        # 1- When rr_alloc does not start right with 'lqr_start_count' reseeded,
        # 2- When rr_alloc does not finish with 'lqr_start_count == 0'.
        # So the difference of files b/w any 2 OST should not be more than 2.
        [[ ${diff_max_min_arr[0]} -le 2 ]] ||
                error "Uneven distribution detected: difference between" \
                "maximum files per OST (${diff_max_min_arr[1]}) and" \
                "minimum files per OST (${diff_max_min_arr[2]}) must not be" \
                "greater than 2"
}

run_fs_test() {
        # fs_test.x is the default name for exe
        FS_TEST=${FS_TEST:=$(which fs_test.x 2> /dev/null || true)}

        local clients=${CLIENTS:-$(hostname)}
        local testdir=$DIR/d0.fs_test
        local file=${testdir}/fs_test
        fs_test_threads=${fs_test_threads:-2}
        fs_test_type=${fs_test_type:-1}
        fs_test_nobj=${fs_test_nobj:-10}
        fs_test_check=${fs_test_check:-3}
        fs_test_strided=${fs_test_strided:-1}
        fs_test_touch=${fs_test_touch:-3}
        fs_test_supersize=${fs_test_supersize:-1}
        fs_test_op=${fs_test_op:-write}
        fs_test_barriers=${fs_test_barriers:-bopen,bwrite,bclose}
        fs_test_io=${fs_test_io:-mpi}
        fs_test_objsize=${fs_test_objsize:-100}
        fs_test_objunit=${fs_test_objunit:-1048576} # 1 mb
        fs_test_ndirs=${fs_test_ndirs:-80000}

        [ x$FS_TEST = x ] &&
                { skip "FS_TEST not found" && return; }

        # Space estimation  in bytes
        local space=$(df -B 1 -P $dir | tail -n 1 | awk '{ print $4 }')
        local total_threads=$((num_clients * fs_test_threads))
        echo "+ $fs_test_objsize * $fs_test_objunit * $total_threads "
        if [ $((space / 2)) -le \
                $((fs_test_objsize * fs_test_objunit * total_threads)) ]; then
                        fs_test_objsize=$((space / 2 / fs_test_objunit / \
                                total_threads))
                        [ $fs_test_objsize -eq 0 ] && \
                        skip_env "Need free space more than \
                                $((2 * total_threads * fs_test_objunit)) \
                                : have $((space / fs_test_objunit))" &&
                                return

                        echo "(reduced objsize to \
                                $((fs_test_objsize * fs_test_objunit)) bytes)"
        fi

        print_opts FS_TEST clients fs_test_threads fs_test_objsize MACHINEFILE

        mkdir -p $testdir
        # mpi_run uses mpiuser
        chmod 0777 $testdir

        # --nodb          Turn off the database code at runtime
        # -g --target     The path to the data file
        # -t --type       Whether to do N-N (1) or N-1 (2)
        # -n --nobj       The number of objects written/read by each proc
        # -z --size       The size of each object
        # -d ---num_nn_dirs Number of subdirectories for files
        # -C --check      Check every byte using argument 3.
        # --collective    Whether to use collective I/O (for N-1, mpi-io only)
        # -s --strided    Whether to use a strided pattern (for N-1 only)
        # -T --touch      Touch every byte using argument 3
        # -o --op         Whether to read only (read) or write only (write)
        # -b --barriers   When to barrier.
        # -i --io         Use POSIX, MPI, or PLFS IO routines (mpi|posix|plfs)
        # -S --supersize  Specify how many objects per superblock

        local cmd="$FS_TEST -nodb -g $file -t $fs_test_type -n $fs_test_nobj \
                -z $((fs_test_objsize * fs_test_objunit)) -d $fs_test_ndirs \
                -C $fs_test_check -collective -s $fs_test_strided \
                -T $fs_test_touch -o $fs_test_op -b $fs_test_barriers \
                -i $fs_test_io -S $fs_test_supersize"

        echo "+ $cmd"
        mpi_run "-np $((num_clients * fs_test_threads))" $cmd

        local rc=$?
        if [ $rc != 0 ] ; then
                error "fs_test failed! $rc"
        fi

        rm -rf $testdir
}

ior_mdtest_parallel() {
        local rc1=0
        local rc2=0
        local type=$1

        run_ior $type &
        local pids=$!

        run_mdtest $type || rc2=$?
        [[ $rc2 -ne 0 ]] && echo "mdtest failed with error $rc2"

        wait $pids || rc1=$?
        [[ $rc1 -ne 0 ]] && echo "ior failed with error $rc1"

        [[ $rc1 -ne 0 || $rc2 -ne 0 ]] && return 1
        return 0
}

run_fio() {
        FIO=${FIO:=$(which fio 2> /dev/null || true)}

        local clients=${CLIENTS:-$(hostname)}
        local fio_jobNum=${fio_jobNum:-4}
        local fio_jobFile=${fio_jobFile:-$TMP/fiojobfile.$(date +%s)}
        local fio_bs=${fio_bs:-1}
        local testdir=$DIR/d0.fio
        local file=${testdir}/fio
        local runtime=60
        local propagate=false

        [ "$SLOW" = "no" ] || runtime=600

        [ x$FIO = x ] &&
                { skip_env "FIO not found" && return; }

        mkdir -p $testdir

        # use fio job file if exists,
        # create a simple one if missing
        if ! [ -f $fio_jobFile ]; then
                cat >> $fio_jobFile <<EOF
[global]
rw=randwrite
size=128m
time_based=1
runtime=$runtime
filename=${file}_\$(hostname)
EOF
                # bs size increased by $i for each job
                for ((i=1; i<=fio_jobNum; i++)); do
                        cat >> $fio_jobFile <<EOF

[job$i]
bs=$(( fio_bs * i ))m
EOF
                done
                # job file is created, should be propagated to all clients
                propagate=true
        fi


        # propagate the job file if not all clients have it yet or
        # if the job file was created during the test run
        if ! do_nodesv $clients " [ -f $fio_jobFile ] " ||
           $propagate; then
                local cfg=$(cat $fio_jobFile)
                do_nodes $clients "echo \\\"$cfg\\\" > ${fio_jobFile}" ||
                        error "job file $fio_jobFile is not propagated"
                do_nodesv $clients "cat ${fio_jobFile}"
        fi

        cmd="$FIO $fio_jobFile"
        echo "+ $cmd"

        log "clients: $clients $cmd"

        local rc=0
        do_nodesv $clients "$cmd "
        rc=$?

        [ $rc = 0 ] || error "fio failed: $rc"
        rm -rf $testdir
}

run_xdd() {
        XDD=${XDD:=$(which xdd 2> /dev/null || true)}

        local clients=${CLIENTS:-$(hostname)}
        local testdir=$DIR/d0.xdd
        xdd_queuedepth=${xdd_queuedepth:-4}
        xdd_blocksize=${xdd_blocksize:-512}
        xdd_reqsize=${xdd_reqsize:-128}
        xdd_mbytes=${xdd_mbytes:-100}
        xdd_passes=${xdd_passes:-40}
        xdd_rwratio=${xdd_rwratio:-0}
        xdd_ntargets=${xdd_ntargets:-6}
        local xdd_custom_params=${xdd_custom_params:-"-dio -stoponerror \
                -maxpri -minall -noproclock -nomemlock"}

        [ x$XDD = x ] &&
                { skip "XDD not found" && return; }

        print_opts XDD clients xdd_queuedepth xdd_blocksize xdd_reqsize \
                xdd_mbytes xdd_passes xdd_rwratio

        mkdir -p $testdir

        local files=""
        # Target files creates based on the given number of targets
        for (( i=0; i < $xdd_ntargets; i++ ))
        do
                files+="${testdir}/xdd"$i" "
        done

        # -targets      specifies the devices or files to perform operation
        # -reqsize      number of 'blocks' per operation
        # -mbytes       number of 1024*1024-byte blocks to transfer
        # -blocksize    size of a single 'block'
        # -passes       number of times to read mbytes
        # -queuedepth   number of commands to queue on the target
        # -rwratio      percentage of read to write operations
        # -verbose      will print out statistics on each pass

        local cmd="$XDD -targets $xdd_ntargets $files -reqsize $xdd_reqsize \
                -mbytes $xdd_mbytes -blocksize $xdd_blocksize \
                -passes $xdd_passes -queuedepth $xdd_queuedepth \
                -rwratio $xdd_rwratio -verbose $xdd_custom_params"
        echo "+ $cmd"

        local rc=0
        do_nodesv $clients "$cmd "
        rc=$?

        [ $rc = 0 ] || error "xdd failed: $rc"

        rm -rf $testdir
}