X-Git-Url: https://git.whamcloud.com/?p=fs%2Flustre-release.git;a=blobdiff_plain;f=lustre%2Ftests%2Ffunctions.sh;h=5fc08eed7ad88db0aaf3ef6c0bc61617ab842168;hp=84916b0e0b903aa43d3462068db79c7e6e4e2c7c;hb=a033bd837329d1eb98d1dd71f4491f1af56a27f0;hpb=1bf48948a99f51ede84a467a23c59c55eb055d7b diff --git a/lustre/tests/functions.sh b/lustre/tests/functions.sh index 84916b0..5fc08ee 100644 --- a/lustre/tests/functions.sh +++ b/lustre/tests/functions.sh @@ -1,6 +1,4 @@ #!/bin/bash -# -*- mode: Bash; tab-width: 4; indent-tabs-mode: t; -*- -# vim:shiftwidth=4:softtabstop=4:tabstop=4: # Simple function used by run_*.sh scripts @@ -214,10 +212,10 @@ lst_end_session () { export LST_SESSION=`$LST show_session 2>/dev/null | awk -F " " '{print $5}'` [ "$LST_SESSION" == "" ] && return + $LST stop b if $verbose; then $LST show_error c s fi - $LST stop b $LST end_session } @@ -285,24 +283,22 @@ print_opts () { } run_compilebench() { - # Space estimation: - # compile dir kernel-0 ~1GB - # required space ~1GB * cbench_IDIRS - local dir=${1:-$DIR} + local cbench_DIR=${cbench_DIR:-""} + local cbench_IDIRS=${cbench_IDIRS:-2} + local cbench_RUNS=${cbench_RUNS:-2} - cbench_DIR=${cbench_DIR:-""} - cbench_IDIRS=${cbench_IDIRS:-2} - cbench_RUNS=${cbench_RUNS:-2} + print_opts cbench_DIR cbench_IDIRS cbench_RUNS - print_opts cbench_DIR cbench_IDIRS cbench_RUNS + [ x$cbench_DIR = x ] && + { skip_env "compilebench not found" && return; } - [ x$cbench_DIR = x ] && - { skip_env "compilebench not found" && return; } - - [ -e $cbench_DIR/compilebench ] || \ - { skip_env "No compilebench build" && return; } + [ -e $cbench_DIR/compilebench ] || + { skip_env "No compilebench build" && return; } + # Space estimation: + # compile dir kernel-0 ~1GB + # required space ~1GB * cbench_IDIRS local space=$(df -P $dir | tail -n 1 | awk '{ print $4 }') if [[ $space -le $((1024 * 1024 * cbench_IDIRS)) ]]; then cbench_IDIRS=$((space / 1024 / 1024)) @@ -310,15 +306,16 @@ run_compilebench() { skip_env "Need free space at least 1GB, have $space" && return - echo "free space=$space, reducing initial dirs to $cbench_IDIRS" + echo "reducing initial dirs to $cbench_IDIRS" fi + echo "free space = $space KB" - # FIXME: - # t-f _base needs to be modifyed to set properly tdir - # for new "test_foo" functions names - # local testdir=$DIR/$tdir - local testdir=$dir/d0.compilebench - mkdir -p $testdir + # FIXME: + # t-f _base needs to be modifyed to set properly tdir + # for new "test_foo" functions names + # local testdir=$DIR/$tdir + local testdir=$dir/d0.compilebench.$$ + mkdir -p $testdir local savePWD=$PWD cd $cbench_DIR @@ -337,32 +334,31 @@ run_compilebench() { } run_metabench() { - - METABENCH=${METABENCH:-$(which metabench 2> /dev/null || true)} - mbench_NFILES=${mbench_NFILES:-30400} - # threads per client - mbench_THREADS=${mbench_THREADS:-4} + local dir=${1:-$DIR} + local mntpt=${2:-$MOUNT} + METABENCH=${METABENCH:-$(which metabench 2> /dev/null || true)} + mbench_NFILES=${mbench_NFILES:-30400} + # threads per client + mbench_THREADS=${mbench_THREADS:-4} mbench_OPTIONS=${mbench_OPTIONS:-} + mbench_CLEANUP=${mbench_CLEANUP:-true} - [ x$METABENCH = x ] && - { skip_env "metabench not found" && return; } - - # FIXME - # Need space estimation here. + [ x$METABENCH = x ] && + { skip_env "metabench not found" && return; } - print_opts METABENCH clients mbench_NFILES mbench_THREADS + print_opts METABENCH clients mbench_NFILES mbench_THREADS - local testdir=$DIR/d0.metabench - mkdir -p $testdir - # mpi_run uses mpiuser - chmod 0777 $testdir + local testdir=$dir/d0.metabench + mkdir -p $testdir + # mpi_run uses mpiuser + chmod 0777 $testdir - # -C Run the file creation tests. - # -S Run the file stat tests. - # -c nfile Number of files to be used in each test. - # -k Cleanup. Remove the test directories. - local cmd="$METABENCH -w $testdir -c $mbench_NFILES -C -S -k $mbench_OPTIONS" - echo "+ $cmd" + # -C Run the file creation tests. Creates zero byte files. + # -S Run the file stat tests. + # -c nfile Number of files to be used in each test. + # -k Cleanup files when finished. + local cmd="$METABENCH -w $testdir -c $mbench_NFILES -C -S $mbench_OPTIONS" + echo "+ $cmd" # find out if we need to use srun by checking $SRUN_PARTITION if [ "$SRUN_PARTITION" ]; then @@ -374,11 +370,16 @@ run_metabench() { -np $((num_clients * $mbench_THREADS)) $cmd fi - local rc=$? - if [ $rc != 0 ] ; then - error "metabench failed! $rc" - fi - rm -rf $testdir + local rc=$? + if [ $rc != 0 ] ; then + error "metabench failed! $rc" + fi + + if $mbench_CLEANUP; then + rm -rf $testdir + else + mv $dir/d0.metabench $mntpt/_xxx.$(date +%s).d0.metabench + fi } run_simul() { @@ -438,6 +439,7 @@ run_mdtest() { # We devide the files by number of core mdtest_nFiles=$((mdtest_nFiles/mdtest_THREADS/num_clients)) mdtest_iteration=${mdtest_iteration:-1} + local mdtest_custom_params=${mdtest_custom_params:-""} local type=${1:-"ssf"} @@ -464,8 +466,10 @@ run_mdtest() { # -n # : number of file/dir to create/stat/remove # -u : each process create/stat/remove individually - local cmd="$MDTEST -d $testdir -i $mdtest_iteration -n $mdtest_nFiles" - [ $type = "fpp" ] && cmd="$cmd -u" + local cmd="$MDTEST -d $testdir -i $mdtest_iteration \ + -n $mdtest_nFiles $mdtest_custom_params" + + [ $type = "fpp" ] && cmd="$cmd -u" echo "+ $cmd" # find out if we need to use srun by checking $SRUN_PARTITION @@ -486,122 +490,156 @@ run_mdtest() { } run_connectathon() { + local dir=${1:-$DIR} + cnt_DIR=${cnt_DIR:-""} + cnt_NRUN=${cnt_NRUN:-10} - cnt_DIR=${cnt_DIR:-""} - cnt_NRUN=${cnt_NRUN:-10} - - print_opts cnt_DIR cnt_NRUN + print_opts cnt_DIR cnt_NRUN - [ x$cnt_DIR = x ] && - { skip_env "connectathon dir not found" && return; } + [ x$cnt_DIR = x ] && + { skip_env "connectathon dir not found" && return; } - [ -e $cnt_DIR/runtests ] || \ - { skip_env "No connectathon runtests found" && return; } + [ -e $cnt_DIR/runtests ] || + { skip_env "No connectathon runtests found" && return; } - local testdir=$DIR/d0.connectathon - mkdir -p $testdir + # Space estimation: + # "special" tests create a 30 MB file + misc. small files + # required space ~40 MB + local space=$(df -P $dir | tail -n 1 | awk '{ print $4 }') + if [[ $space -le $((1024 * 40)) ]]; then + skip_env "Need free space at least 40MB, have $space KB" && + return + fi + echo "free space = $space KB" + + local testdir=$dir/d0.connectathon + mkdir -p $testdir + + local savePWD=$PWD + cd $cnt_DIR + + # + # To run connectathon: + # runtests [-a|-b|-g|-s|-l] [-f|-n|-t] [-N numpasses] [test-directory] + # + # One of the following test types + # -b basic + # -g general + # -s special + # -l lock + # -a all of the above + # + # -f a quick functional test + # -n suppress directory operations (mkdir and rmdir) + # -t run with time statistics (default for basic tests) + # + # -N numpasses - specifies the number of times to run + # the tests. Optional. + + tests="-b -g -s" + # Include lock tests unless we're running on nfsv4 + local fstype=$(df -TP $testdir | awk 'NR==2 {print $2}') + echo "$testdir: $fstype" + if [[ $fstype != "nfs4" ]]; then + tests="$tests -l" + fi + echo "tests: $tests" + for test in $tests; do + local cmd="./runtests -N $cnt_NRUN $test -f $testdir" + local rc=0 + + log "$cmd" + eval $cmd + rc=$? + [ $rc = 0 ] || error "connectathon failed: $rc" + done - local savePWD=$PWD - cd $cnt_DIR + cd $savePWD + rm -rf $testdir +} - # - # cthon options (must be in this order) - # - # -N numpasses - will be passed to the runtests script. This argument - # is optional. It specifies the number of times to run - # through the tests. - # - # One of these test types - # -b basic - # -g general - # -s special - # -l lock - # -a all of the above - # - # -f a quick functionality test - # +run_ior() { + local type=${1:="ssf"} + local dir=${2:-$DIR} + local testdir=$dir/d0.ior.$type + local nfs_srvmntpt=$3 + + if [ "$NFSCLIENT" ]; then + [[ -n $nfs_srvmntpt ]] || + { error "NFSCLIENT mode, but nfs exported dir"\ + "is not set!" && return 1; } + fi - tests="-b -g -s" - # Include lock tests unless we're running on nfsv4 - local fstype=$(df -TP $testdir | awk 'NR==2 {print $2}') - echo "$testdir: $fstype" - if [[ $fstype != "nfs4" ]]; then - tests="$tests -l" - fi - echo "tests: $tests" - for test in $tests; do - local cmd="./runtests -N $cnt_NRUN $test -f $testdir" - local rc=0 - - log "$cmd" - eval $cmd - rc=$? - [ $rc = 0 ] || error "connectathon failed: $rc" - done + IOR=${IOR:-$(which IOR 2> /dev/null || true)} + [ x$IOR = x ] && + { skip_env "IOR not found" && return; } + + # threads per client + ior_THREADS=${ior_THREADS:-2} + ior_iteration=${ior_iteration:-1} + ior_blockSize=${ior_blockSize:-6} + ior_blockUnit=${ior_blockUnit:-M} # K, M, G + ior_xferSize=${ior_xferSize:-1M} + ior_type=${ior_type:-POSIX} + ior_DURATION=${ior_DURATION:-30} # minutes + local multiplier=1 + case ${ior_blockUnit} in + [G]) + multiplier=$((1024 * 1024 * 1024)) + ;; + [M]) + multiplier=$((1024 * 1024)) + ;; + [K]) + multiplier=1024 + ;; + *) error "Incorrect block unit should be one of [KMG]" + ;; + esac + + # calculate the space in bytes + local space=$(df -B 1 -P $dir | tail -n 1 | awk '{ print $4 }') + local total_threads=$((num_clients * ior_THREADS)) + echo "+ $ior_blockSize * $multiplier * $total_threads " + if [ $((space / 2)) -le \ + $((ior_blockSize * multiplier * total_threads)) ]; then + ior_blockSize=$((space / 2 / multiplier / total_threads)) + [ $ior_blockSize -eq 0 ] && \ + skip_env "Need free space more than $((2 * total_threads)) \ + ${ior_blockUnit}: have $((space / multiplier))" && + return + + echo "(reduced blockSize to $ior_blockSize \ + ${ior_blockUnit} bytes)" + fi - cd $savePWD - rm -rf $testdir -} + print_opts IOR ior_THREADS ior_DURATION MACHINEFILE -run_ior() { - local type=${1:="ssf"} + mkdir -p $testdir + # mpi_run uses mpiuser + chmod 0777 $testdir + if [ -z "$NFSCLIENT" ]; then + $LFS setstripe $testdir -c -1 || + { error "setstripe failed" && return 2; } + fi - IOR=${IOR:-$(which IOR 2> /dev/null || true)} - # threads per client - ior_THREADS=${ior_THREADS:-2} - ior_iteration=${ior_iteration:-1} - ior_blockSize=${ior_blockSize:-6} # GB - ior_xferSize=${ior_xferSize:-2m} - ior_type=${ior_type:-POSIX} - ior_DURATION=${ior_DURATION:-30} # minutes - - [ x$IOR = x ] && - { skip_env "IOR not found" && return; } - - local space=$(df -P $DIR | tail -n 1 | awk '{ print $4 }') - local total_threads=$(( num_clients * ior_THREADS )) - echo "+ $ior_blockSize * 1024 * 1024 * $total_threads " - if [ $((space / 2)) -le \ - $(( ior_blockSize * 1024 * 1024 * total_threads)) ]; then - echo "+ $space * 9/10 / 1024 / 1024 / $num_clients / $ior_THREADS" - ior_blockSize=$(( space /2 /1024 /1024 / num_clients / ior_THREADS )) - [ $ior_blockSize = 0 ] && \ - skip_env "Need free space more than $((2 * total_threads))GB: \ - $((total_threads *1024 *1024*2)), have $space" && return - - local reduced_size="$num_clients x $ior_THREADS x $ior_blockSize" - echo "free space=$space, Need: $reduced_size GB" - echo "(blockSize reduced to $ior_blockSize Gb)" - fi + # + # -b N blockSize -- + # contiguous bytes to write per task (e.g.: 8, 4K, 2M, 1G)" + # -o S testFileName + # -t N transferSize -- size of transfer in bytes (e.g.: 8, 4K, 2M, 1G)" + # -w writeFile -- write file" + # -r readFile -- read existing file" + # -W checkWrite -- check read after write" + # -C reorderTasks -- changes task ordering to n+1 ordering for readback + # -T maxTimeDuration -- max time in minutes to run tests" + # -k keepFile -- keep testFile(s) on program exit - print_opts IOR ior_THREADS ior_DURATION MACHINEFILE + local cmd="$IOR -a $ior_type -b ${ior_blockSize}${ior_blockUnit} \ + -o $testdir/iorData -t $ior_xferSize -v -C -w -r -W \ + -i $ior_iteration -T $ior_DURATION -k" - local testdir=$DIR/d0.ior.$type - mkdir -p $testdir - # mpi_run uses mpiuser - chmod 0777 $testdir - if [ "$NFSCLIENT" ]; then - setstripe_nfsserver $testdir -c -1 || - { error "setstripe on nfsserver failed" && return 1; } - else - $LFS setstripe $testdir -c -1 || - { error "setstripe failed" && return 2; } - fi - # - # -b N blockSize -- - # contiguous bytes to write per task (e.g.: 8, 4k, 2m, 1g)" - # -o S testFileName - # -t N transferSize -- size of transfer in bytes (e.g.: 8, 4k, 2m, 1g)" - # -w writeFile -- write file" - # -r readFile -- read existing file" - # -W checkWrite -- check read after write" - # -C reorderTasks -- changes task ordering to n+1 ordering for readback - # -T maxTimeDuration -- max time in minutes to run tests" - # -k keepFile -- keep testFile(s) on program exit - - local cmd="$IOR -a $ior_type -b ${ior_blockSize}g -o $testdir/iorData \ - -t $ior_xferSize -v -C -w -r -W -i $ior_iteration -T $ior_DURATION -k" - [ $type = "fpp" ] && cmd="$cmd -F" + [ $type = "fpp" ] && cmd="$cmd -F" echo "+ $cmd" # find out if we need to use srun by checking $SRUN_PARTITION @@ -760,20 +798,21 @@ run_write_append_truncate() { } run_write_disjoint() { + if [ "$NFSCLIENT" ]; then + skip "skipped for NFSCLIENT mode" + return + fi - WRITE_DISJOINT=${WRITE_DISJOINT:-$(which write_disjoint \ - 2> /dev/null || true)} - # threads per client - wdisjoint_THREADS=${wdisjoint_THREADS:-4} - wdisjoint_REP=${wdisjoint_REP:-10000} + WRITE_DISJOINT=${WRITE_DISJOINT:-$(which write_disjoint 2> /dev/null || + true)} - if [ "$NFSCLIENT" ]; then - skip "skipped for NFSCLIENT mode" - return - fi + [ x$WRITE_DISJOINT = x ] && + { skip_env "write_disjoint not found" && return; } - [ x$WRITE_DISJOINT = x ] && - { skip_env "write_disjoint not found" && return; } + # threads per client + wdisjoint_THREADS=${wdisjoint_THREADS:-4} + wdisjoint_REP=${wdisjoint_REP:-10000} + chunk_size_limit=$1 # FIXME # Need space estimation here. @@ -785,7 +824,8 @@ run_write_disjoint() { # mpi_run uses mpiuser chmod 0777 $testdir - local cmd="$WRITE_DISJOINT -f $testdir/file -n $wdisjoint_REP" + local cmd="$WRITE_DISJOINT -f $testdir/file -n $wdisjoint_REP -m \ + $chunk_size_limit" echo "+ $cmd" mpi_run ${MACHINEFILE_OPTION} ${MACHINEFILE} \ @@ -937,3 +977,221 @@ run_statahead () { rm -rf $testdir cleanup_statahead $clients $mntpt_root $num_mntpts } + +run_fs_test() { + # fs_test.x is the default name for exe + FS_TEST=${FS_TEST:=$(which fs_test.x 2> /dev/null || true)} + + local clients=${CLIENTS:-$(hostname)} + local testdir=$DIR/d0.fs_test + local file=${testdir}/fs_test + fs_test_threads=${fs_test_threads:-2} + fs_test_type=${fs_test_type:-1} + fs_test_nobj=${fs_test_nobj:-10} + fs_test_check=${fs_test_check:-3} + fs_test_strided=${fs_test_strided:-1} + fs_test_touch=${fs_test_touch:-3} + fs_test_supersize=${fs_test_supersize:-1} + fs_test_op=${fs_test_op:-write} + fs_test_barriers=${fs_test_barriers:-bopen,bwrite,bclose} + fs_test_io=${fs_test_io:-mpi} + fs_test_objsize=${fs_test_objsize:-100} + fs_test_objunit=${fs_test_objunit:-1048576} # 1 mb + fs_test_ndirs=${fs_test_ndirs:-80000} + + [ x$FS_TEST = x ] && + { skip "FS_TEST not found" && return; } + + # Space estimation in bytes + local space=$(df -B 1 -P $dir | tail -n 1 | awk '{ print $4 }') + local total_threads=$((num_clients * fs_test_threads)) + echo "+ $fs_test_objsize * $fs_test_objunit * $total_threads " + if [ $((space / 2)) -le \ + $((fs_test_objsize * fs_test_objunit * total_threads)) ]; then + fs_test_objsize=$((space / 2 / fs_test_objunit / \ + total_threads)) + [ $fs_test_objsize -eq 0 ] && \ + skip_env "Need free space more than \ + $((2 * total_threads * fs_test_objunit)) \ + : have $((space / fs_test_objunit))" && + return + + echo "(reduced objsize to \ + $((fs_test_objsize * fs_test_objunit)) bytes)" + fi + + print_opts FS_TEST clients fs_test_threads fs_test_objsize MACHINEFILE + + mkdir -p $testdir + # mpi_run uses mpiuser + chmod 0777 $testdir + + # --nodb Turn off the database code at runtime + # -g --target The path to the data file + # -t --type Whether to do N-N (1) or N-1 (2) + # -n --nobj The number of objects written/read by each proc + # -z --size The size of each object + # -d ---num_nn_dirs Number of subdirectories for files + # -C --check Check every byte using argument 3. + # --collective Whether to use collective I/O (for N-1, mpi-io only) + # -s --strided Whether to use a strided pattern (for N-1 only) + # -T --touch Touch every byte using argument 3 + # -o --op Whether to read only (read) or write only (write) + # -b --barriers When to barrier. + # -i --io Use POSIX, MPI, or PLFS IO routines (mpi|posix|plfs) + # -S --supersize Specify how many objects per superblock + + local cmd="$FS_TEST -nodb -g $file -t $fs_test_type -n $fs_test_nobj \ + -z $((fs_test_objsize * fs_test_objunit)) -d $fs_test_ndirs \ + -C $fs_test_check -collective -s $fs_test_strided \ + -T $fs_test_touch -o $fs_test_op -b $fs_test_barriers \ + -i $fs_test_io -S $fs_test_supersize" + + echo "+ $cmd" + mpi_run "-np $((num_clients * fs_test_threads))" $cmd + + local rc=$? + if [ $rc != 0 ] ; then + error "fs_test failed! $rc" + fi + + rm -rf $testdir +} + +ior_mdtest_parallel() { + local rc1=0 + local rc2=0 + local type=$1 + + run_ior $type & + local pids=$! + + run_mdtest $type || rc2=$? + [[ $rc2 -ne 0 ]] && echo "mdtest failed with error $rc2" + + wait $pids || rc1=$? + [[ $rc1 -ne 0 ]] && echo "ior failed with error $rc1" + + [[ $rc1 -ne 0 || $rc2 -ne 0 ]] && return 1 + return 0 +} + +run_fio() { + FIO=${FIO:=$(which fio 2> /dev/null || true)} + + local clients=${CLIENTS:-$(hostname)} + local fio_jobNum=${fio_jobNum:-4} + local fio_jobFile=${fio_jobFile:-$TMP/fiojobfile.$(date +%s)} + local fio_bs=${fio_bs:-1} + local testdir=$DIR/d0.fio + local file=${testdir}/fio + local runtime=60 + local propagate=false + + [ "$SLOW" = "no" ] || runtime=600 + + [ x$FIO = x ] && + { skip_env "FIO not found" && return; } + + mkdir -p $testdir + + # use fio job file if exists, + # create a simple one if missing + if ! [ -f $fio_jobFile ]; then + cat >> $fio_jobFile <> $fio_jobFile < ${fio_jobFile}" || + error "job file $fio_jobFile is not propagated" + do_nodesv $clients "cat ${fio_jobFile}" + fi + + cmd="$FIO $fio_jobFile" + echo "+ $cmd" + + log "clients: $clients $cmd" + + local rc=0 + do_nodesv $clients "$cmd " + rc=$? + + [ $rc = 0 ] || error "fio failed: $rc" + rm -rf $testdir +} + +run_xdd() { + XDD=${XDD:=$(which xdd 2> /dev/null || true)} + + local clients=${CLIENTS:-$(hostname)} + local testdir=$DIR/d0.xdd + xdd_queuedepth=${xdd_queuedepth:-4} + xdd_blocksize=${xdd_blocksize:-512} + xdd_reqsize=${xdd_reqsize:-128} + xdd_mbytes=${xdd_mbytes:-100} + xdd_passes=${xdd_passes:-40} + xdd_rwratio=${xdd_rwratio:-0} + xdd_ntargets=${xdd_ntargets:-6} + local xdd_custom_params=${xdd_custom_params:-"-dio -stoponerror \ + -maxpri -minall -noproclock -nomemlock"} + + [ x$XDD = x ] && + { skip "XDD not found" && return; } + + print_opts XDD clients xdd_queuedepth xdd_blocksize xdd_reqsize \ + xdd_mbytes xdd_passes xdd_rwratio + + mkdir -p $testdir + + local files="" + # Target files creates based on the given number of targets + for (( i=0; i < $xdd_ntargets; i++ )) + do + files+="${testdir}/xdd"$i" " + done + + # -targets specifies the devices or files to perform operation + # -reqsize number of 'blocks' per operation + # -mbytes number of 1024*1024-byte blocks to transfer + # -blocksize size of a single 'block' + # -passes number of times to read mbytes + # -queuedepth number of commands to queue on the target + # -rwratio percentage of read to write operations + # -verbose will print out statistics on each pass + + local cmd="$XDD -targets $xdd_ntargets $files -reqsize $xdd_reqsize \ + -mbytes $xdd_mbytes -blocksize $xdd_blocksize \ + -passes $xdd_passes -queuedepth $xdd_queuedepth \ + -rwratio $xdd_rwratio -verbose $xdd_custom_params" + echo "+ $cmd" + + local rc=0 + do_nodesv $clients "$cmd " + rc=$? + + [ $rc = 0 ] || error "xdd failed: $rc" + + rm -rf $testdir +}