From 0b8e9558e88930814857c97dfe2394f8c8e24a9a Mon Sep 17 00:00:00 2001 From: James Nunez Date: Sun, 19 Nov 2017 21:22:38 -0700 Subject: [PATCH] LU-9872 tests: modify check space requirements for NFS test In parallel-scale-nfs testing, the Lustre clients NFS mounts the Lustre file system. In some cases, the file system looks like it has enough space (du) to run NFS testing, but one or more OSTs is full or is almost full. Since the default striping for Lustre is a single stripe, some NFS tests will fail because the file it is writing gets written to that almost full OST. Increase the default stripe to all OSTs for all parallel-scale-nfs tests. Test-Parameters: trivial testlist=parallel-scale,parallel-scale-nfsv3,parallel-scale-nfsv4 Signed-off-by: James Nunez Change-Id: Iff187dc6c158d9063cd36ced0801b88cdc8814ae Reviewed-on: https://review.whamcloud.com/29786 Tested-by: Jenkins Tested-by: Maloo Reviewed-by: James Casper Reviewed-by: Saurabh Tandan Reviewed-by: Oleg Drokin --- lustre/tests/functions.sh | 164 +++++++++++++++++++------------------ lustre/tests/parallel-scale-nfs.sh | 33 ++++---- 2 files changed, 103 insertions(+), 94 deletions(-) diff --git a/lustre/tests/functions.sh b/lustre/tests/functions.sh index db134ce..00889e4 100644 --- a/lustre/tests/functions.sh +++ b/lustre/tests/functions.sh @@ -1,6 +1,4 @@ #!/bin/bash -# -*- mode: Bash; tab-width: 4; indent-tabs-mode: t; -*- -# vim:shiftwidth=4:softtabstop=4:tabstop=4: # Simple function used by run_*.sh scripts @@ -285,24 +283,22 @@ print_opts () { } run_compilebench() { - # Space estimation: - # compile dir kernel-0 ~1GB - # required space ~1GB * cbench_IDIRS - local dir=${1:-$DIR} + local cbench_DIR=${cbench_DIR:-""} + local cbench_IDIRS=${cbench_IDIRS:-2} + local cbench_RUNS=${cbench_RUNS:-2} - cbench_DIR=${cbench_DIR:-""} - cbench_IDIRS=${cbench_IDIRS:-2} - cbench_RUNS=${cbench_RUNS:-2} - - print_opts cbench_DIR cbench_IDIRS cbench_RUNS + print_opts cbench_DIR cbench_IDIRS cbench_RUNS - [ x$cbench_DIR = x ] && - { skip_env "compilebench not found" && return; } + [ x$cbench_DIR = x ] && + { skip_env "compilebench not found" && return; } - [ -e $cbench_DIR/compilebench ] || \ - { skip_env "No compilebench build" && return; } + [ -e $cbench_DIR/compilebench ] || + { skip_env "No compilebench build" && return; } + # Space estimation: + # compile dir kernel-0 ~1GB + # required space ~1GB * cbench_IDIRS local space=$(df -P $dir | tail -n 1 | awk '{ print $4 }') if [[ $space -le $((1024 * 1024 * cbench_IDIRS)) ]]; then cbench_IDIRS=$((space / 1024 / 1024)) @@ -310,15 +306,16 @@ run_compilebench() { skip_env "Need free space at least 1GB, have $space" && return - echo "free space=$space, reducing initial dirs to $cbench_IDIRS" + echo "reducing initial dirs to $cbench_IDIRS" fi + echo "free space = $space KB" - # FIXME: - # t-f _base needs to be modifyed to set properly tdir - # for new "test_foo" functions names - # local testdir=$DIR/$tdir - local testdir=$dir/d0.compilebench.$$ - mkdir -p $testdir + # FIXME: + # t-f _base needs to be modifyed to set properly tdir + # for new "test_foo" functions names + # local testdir=$DIR/$tdir + local testdir=$dir/d0.compilebench.$$ + mkdir -p $testdir local savePWD=$PWD cd $cbench_DIR @@ -338,6 +335,7 @@ run_compilebench() { run_metabench() { local dir=${1:-$DIR} + local mntpt=${2:-$MOUNT} METABENCH=${METABENCH:-$(which metabench 2> /dev/null || true)} mbench_NFILES=${mbench_NFILES:-30400} # threads per client @@ -348,9 +346,6 @@ run_metabench() { [ x$METABENCH = x ] && { skip_env "metabench not found" && return; } - # FIXME - # Need space estimation here. - print_opts METABENCH clients mbench_NFILES mbench_THREADS local testdir=$dir/d0.metabench @@ -358,11 +353,11 @@ run_metabench() { # mpi_run uses mpiuser chmod 0777 $testdir - # -C Run the file creation tests. + # -C Run the file creation tests. Creates zero byte files. # -S Run the file stat tests. # -c nfile Number of files to be used in each test. - # -k => dont cleanup files when finished. - local cmd="$METABENCH -w $testdir -c $mbench_NFILES -C -S -k $mbench_OPTIONS" + # -k Cleanup files when finished. + local cmd="$METABENCH -w $testdir -c $mbench_NFILES -C -S $mbench_OPTIONS" echo "+ $cmd" # find out if we need to use srun by checking $SRUN_PARTITION @@ -375,15 +370,15 @@ run_metabench() { -np $((num_clients * $mbench_THREADS)) $cmd fi - local rc=$? - if [ $rc != 0 ] ; then - error "metabench failed! $rc" - fi + local rc=$? + if [ $rc != 0 ] ; then + error "metabench failed! $rc" + fi if $mbench_CLEANUP; then rm -rf $testdir else - mv $dir/d0.metabench $dir/_xxx.$(date +%s).d0.metabench + mv $dir/d0.metabench $mntpt/_xxx.$(date +%s).d0.metabench fi } @@ -504,52 +499,63 @@ run_connectathon() { [ x$cnt_DIR = x ] && { skip_env "connectathon dir not found" && return; } - [ -e $cnt_DIR/runtests ] || \ + [ -e $cnt_DIR/runtests ] || { skip_env "No connectathon runtests found" && return; } + # Space estimation: + # "special" tests create a 30 MB file + misc. small files + # required space ~40 MB + local space=$(df -P $dir | tail -n 1 | awk '{ print $4 }') + if [[ $space -le $((1024 * 40)) ]]; then + skip_env "Need free space at least 40MB, have $space KB" && + return + fi + echo "free space = $space KB" + local testdir=$dir/d0.connectathon mkdir -p $testdir - local savePWD=$PWD - cd $cnt_DIR - - # - # cthon options (must be in this order) - # - # -N numpasses - will be passed to the runtests script. This argument - # is optional. It specifies the number of times to run - # through the tests. - # - # One of these test types - # -b basic - # -g general - # -s special - # -l lock - # -a all of the above - # - # -f a quick functionality test - # + local savePWD=$PWD + cd $cnt_DIR - tests="-b -g -s" - # Include lock tests unless we're running on nfsv4 - local fstype=$(df -TP $testdir | awk 'NR==2 {print $2}') - echo "$testdir: $fstype" - if [[ $fstype != "nfs4" ]]; then - tests="$tests -l" - fi - echo "tests: $tests" - for test in $tests; do - local cmd="./runtests -N $cnt_NRUN $test -f $testdir" - local rc=0 - - log "$cmd" - eval $cmd - rc=$? - [ $rc = 0 ] || error "connectathon failed: $rc" - done + # + # To run connectathon: + # runtests [-a|-b|-g|-s|-l] [-f|-n|-t] [-N numpasses] [test-directory] + # + # One of the following test types + # -b basic + # -g general + # -s special + # -l lock + # -a all of the above + # + # -f a quick functional test + # -n suppress directory operations (mkdir and rmdir) + # -t run with time statistics (default for basic tests) + # + # -N numpasses - specifies the number of times to run + # the tests. Optional. + + tests="-b -g -s" + # Include lock tests unless we're running on nfsv4 + local fstype=$(df -TP $testdir | awk 'NR==2 {print $2}') + echo "$testdir: $fstype" + if [[ $fstype != "nfs4" ]]; then + tests="$tests -l" + fi + echo "tests: $tests" + for test in $tests; do + local cmd="./runtests -N $cnt_NRUN $test -f $testdir" + local rc=0 + + log "$cmd" + eval $cmd + rc=$? + [ $rc = 0 ] || error "connectathon failed: $rc" + done - cd $savePWD - rm -rf $testdir + cd $savePWD + rm -rf $testdir } run_ior() { @@ -607,18 +613,16 @@ run_ior() { ${ior_blockUnit} bytes)" fi - print_opts IOR ior_THREADS ior_DURATION MACHINEFILE + print_opts IOR ior_THREADS ior_DURATION MACHINEFILE - mkdir -p $testdir - # mpi_run uses mpiuser - chmod 0777 $testdir - if [ "$NFSCLIENT" ]; then - setstripe_nfsserver $testdir $nfs_srvmntpt -c -1 || - { error "setstripe on nfsserver failed" && return 1; } - else + mkdir -p $testdir + # mpi_run uses mpiuser + chmod 0777 $testdir + if [ -z "$NFSCLIENT" ]; then $LFS setstripe $testdir -c -1 || { error "setstripe failed" && return 2; } fi + # # -b N blockSize -- # contiguous bytes to write per task (e.g.: 8, 4K, 2M, 1G)" diff --git a/lustre/tests/parallel-scale-nfs.sh b/lustre/tests/parallel-scale-nfs.sh index 4188c3b..74ae30b 100755 --- a/lustre/tests/parallel-scale-nfs.sh +++ b/lustre/tests/parallel-scale-nfs.sh @@ -7,7 +7,7 @@ LUSTRE=${LUSTRE:-$(cd $(dirname $0)/..; echo $PWD)} . $LUSTRE/tests/test-framework.sh # only call init_test_env if this script is called directly if [[ -z "$TESTSUITE" || "$TESTSUITE" = "$(basename $0 .sh)" ]]; then - init_test_env $@ + init_test_env $@ fi . ${CONFIG:=$LUSTRE/tests/cfg/$NAME.sh} init_logging @@ -15,10 +15,6 @@ init_logging racer=$LUSTRE/tests/racer/racer.sh . $LUSTRE/tests/setup-nfs.sh -check_and_setup_lustre - -# first unmount all the lustre clients -cleanup_mount $MOUNT # lustre client used as nfs server (default is mds node) LUSTRE_CLIENT_NFSSRV=${LUSTRE_CLIENT_NFSSRV:-$(facet_active_host $SINGLEMDS)} NFS_SRVMNTPT=${NFS_SRVMNTPT:-$MOUNT} @@ -30,7 +26,16 @@ NFS_CLIMNTPT=${NFS_CLIMNTPT:-$MOUNT} skip_env "need at least two nodes: nfs server and nfs client" && exit 0 [ "$NFSVERSION" = "4" ] && cl_mnt_opt="${MOUNT_OPTS:+$MOUNT_OPTS,}32bitapi" || - cl_mnt_opt="" + cl_mnt_opt="" + +check_and_setup_lustre +$LFS df +TESTDIR=$NFS_CLIMNTPT/d0.$(basename $0 .sh) +mkdir -p $TESTDIR +$LFS setstripe -c -1 $TESTDIR + +# first unmount all the lustre clients +cleanup_mount $MOUNT cleanup_exit () { trap 0 @@ -53,7 +58,7 @@ cleanup () { trap cleanup_exit EXIT SIGHUP SIGINT zconf_mount $LUSTRE_CLIENT_NFSSRV $NFS_SRVMNTPT "$cl_mnt_opt" || - error "mount lustre on $LUSTRE_CLIENT_NFSSRV failed" + error "mount lustre on $LUSTRE_CLIENT_NFSSRV failed" # setup the nfs setup_nfs "$NFSVERSION" "$NFS_SRVMNTPT" "$LUSTRE_CLIENT_NFSSRV" \ @@ -66,8 +71,8 @@ FAIL_ON_ERROR=false # common setup MACHINEFILE=${MACHINEFILE:-$TMP/$(basename $0 .sh).machines} clients=${NFS_CLIENTS:-$HOSTNAME} -generate_machine_file $clients $MACHINEFILE || \ - error "Failed to generate machine file" +generate_machine_file $clients $MACHINEFILE || + error "Failed to generate machine file" num_clients=$(get_node_count ${clients//,/ }) # compilbench @@ -95,27 +100,27 @@ MPI_RUNAS=${MPI_RUNAS:-"runas -u $MPI_USER_UID -g $MPI_USER_GID"} $GSS_KRB5 && refresh_krb5_tgt $MPI_USER_UID $MPI_USER_GID $MPI_RUNAS test_compilebench() { - run_compilebench $NFS_CLIMNTPT + run_compilebench $TESTDIR } run_test compilebench "compilebench" test_metabench() { - run_metabench $NFS_CLIMNTPT + run_metabench $TESTDIR $NFS_CLIMNTPT } run_test metabench "metabench" test_connectathon() { - run_connectathon $NFS_CLIMNTPT + run_connectathon $TESTDIR } run_test connectathon "connectathon" test_iorssf() { - run_ior "ssf" $NFS_CLIMNTPT $NFS_SRVMNTPT + run_ior "ssf" $TESTDIR $NFS_SRVMNTPT } run_test iorssf "iorssf" test_iorfpp() { - run_ior "fpp" $NFS_CLIMNTPT $NFS_SRVMNTPT + run_ior "fpp" $TESTDIR $NFS_SRVMNTPT } run_test iorfpp "iorfpp" -- 1.8.3.1