From d238e5d049e2b73f7997fa0add1a094fec432cb2 Mon Sep 17 00:00:00 2001 From: Elena Gryaznova Date: Fri, 26 May 2017 18:40:13 +0300 Subject: [PATCH] LU-6900 tests: parallel-scale-nfs improvement This patch addresses a set of parallel-scale-nfs.sh suite defects and fixes: 1. Suite deals only with nfs server exported from SINGLEMDS (lustre_client) and nfs clients set to default CLIENTS list. -- Patch adds the possiblity to specify NFS_SERVER, NFS_CLIENTS, nfs server's lustre mount point and mount points for nfs clients; functions which run applications operate with nfs client mntpt instead of global $DIR. 2. Suite does not cleanup the nfs setup in case of failure, this breaks next tests. -- Patch adds trap to perform cleanup in case of any error. 3. Suite can not run on solo setup because it requires nfs clients and nfs servers are to be the different nodes. -- Patch skips the suite for singe node setup. Test-Parameters: mdtcount=1 ostsizegb=2 testlist=parallel-scale-nfsv3,parallel-scale-nfsv4 Signed-off-by: Elena Gryaznova Seagate-bug-id: MRP-1742, MRP-1622, MRP-2056, MRP-2682 Reviewed-by: Andriy Skulysh Reviewed-by: Vladimir Saveliev Reviewed-by: Alexander Lezhoev Reviewed-by: Alexey Lyashkov Change-Id: I3e1c877764db561f7e8ca7703b90cc5cf6f91ca0 Reviewed-on: https://review.whamcloud.com/15712 Tested-by: Jenkins Tested-by: Maloo Reviewed-by: Jian Yu Reviewed-by: Wei Liu Reviewed-by: Oleg Drokin --- lustre/tests/functions.sh | 80 +++++++++++++++++++++----------------- lustre/tests/parallel-scale-nfs.sh | 76 ++++++++++++++++++++---------------- lustre/tests/setup-nfs.sh | 41 ++++++++++--------- lustre/tests/test-framework.sh | 14 +++++-- 4 files changed, 119 insertions(+), 92 deletions(-) diff --git a/lustre/tests/functions.sh b/lustre/tests/functions.sh index 7e8af72..5723c2a 100644 --- a/lustre/tests/functions.sh +++ b/lustre/tests/functions.sh @@ -337,26 +337,26 @@ run_compilebench() { } run_metabench() { - - METABENCH=${METABENCH:-$(which metabench 2> /dev/null || true)} - mbench_NFILES=${mbench_NFILES:-30400} - # threads per client - mbench_THREADS=${mbench_THREADS:-4} + local dir=${1:-$DIR} + METABENCH=${METABENCH:-$(which metabench 2> /dev/null || true)} + mbench_NFILES=${mbench_NFILES:-30400} + # threads per client + mbench_THREADS=${mbench_THREADS:-4} mbench_OPTIONS=${mbench_OPTIONS:-} mbench_CLEANUP=${mbench_CLEANUP:-true} - [ x$METABENCH = x ] && - { skip_env "metabench not found" && return; } + [ x$METABENCH = x ] && + { skip_env "metabench not found" && return; } - # FIXME - # Need space estimation here. + # FIXME + # Need space estimation here. - print_opts METABENCH clients mbench_NFILES mbench_THREADS + print_opts METABENCH clients mbench_NFILES mbench_THREADS - local testdir=$DIR/d0.metabench - mkdir -p $testdir - # mpi_run uses mpiuser - chmod 0777 $testdir + local testdir=$dir/d0.metabench + mkdir -p $testdir + # mpi_run uses mpiuser + chmod 0777 $testdir # -C Run the file creation tests. # -S Run the file stat tests. @@ -495,20 +495,20 @@ run_mdtest() { } run_connectathon() { + local dir=${1:-$DIR} + cnt_DIR=${cnt_DIR:-""} + cnt_NRUN=${cnt_NRUN:-10} - cnt_DIR=${cnt_DIR:-""} - cnt_NRUN=${cnt_NRUN:-10} - - print_opts cnt_DIR cnt_NRUN + print_opts cnt_DIR cnt_NRUN - [ x$cnt_DIR = x ] && - { skip_env "connectathon dir not found" && return; } + [ x$cnt_DIR = x ] && + { skip_env "connectathon dir not found" && return; } - [ -e $cnt_DIR/runtests ] || \ - { skip_env "No connectathon runtests found" && return; } + [ -e $cnt_DIR/runtests ] || \ + { skip_env "No connectathon runtests found" && return; } - local testdir=$DIR/d0.connectathon - mkdir -p $testdir + local testdir=$dir/d0.connectathon + mkdir -p $testdir local savePWD=$PWD cd $cnt_DIR @@ -554,8 +554,20 @@ run_connectathon() { run_ior() { local type=${1:="ssf"} + local dir=${2:-$DIR} + local testdir=$dir/d0.ior.$type + local nfs_srvmntpt=$3 + + if [ "$NFSCLIENT" ]; then + [[ -n $nfs_srvmntpt ]] || + { error "NFSCLIENT mode, but nfs exported dir"\ + "is not set!" && return 1; } + fi IOR=${IOR:-$(which IOR 2> /dev/null || true)} + [ x$IOR = x ] && + { skip_env "IOR not found" && return; } + # threads per client ior_THREADS=${ior_THREADS:-2} ior_iteration=${ior_iteration:-1} @@ -579,11 +591,8 @@ run_ior() { ;; esac - [ x$IOR = x ] && - { skip_env "IOR not found" && return; } - # calculate the space in bytes - local space=$(df -B 1 -P $DIR | tail -n 1 | awk '{ print $4 }') + local space=$(df -B 1 -P $dir | tail -n 1 | awk '{ print $4 }') local total_threads=$((num_clients * ior_THREADS)) echo "+ $ior_blockSize * $multiplier * $total_threads " if [ $((space / 2)) -le \ @@ -600,17 +609,16 @@ run_ior() { print_opts IOR ior_THREADS ior_DURATION MACHINEFILE - local testdir=$DIR/d0.ior.$type mkdir -p $testdir # mpi_run uses mpiuser chmod 0777 $testdir - if [ "$NFSCLIENT" ]; then - setstripe_nfsserver $testdir -c -1 || - { error "setstripe on nfsserver failed" && return 1; } - else - $LFS setstripe $testdir -c -1 || - { error "setstripe failed" && return 2; } - fi + if [ "$NFSCLIENT" ]; then + setstripe_nfsserver $testdir $nfs_srvmntpt -c -1 || + { error "setstripe on nfsserver failed" && return 1; } + else + $LFS setstripe $testdir -c -1 || + { error "setstripe failed" && return 2; } + fi # # -b N blockSize -- # contiguous bytes to write per task (e.g.: 8, 4K, 2M, 1G)" diff --git a/lustre/tests/parallel-scale-nfs.sh b/lustre/tests/parallel-scale-nfs.sh index 4f698c3..4188c3b 100755 --- a/lustre/tests/parallel-scale-nfs.sh +++ b/lustre/tests/parallel-scale-nfs.sh @@ -17,35 +17,55 @@ racer=$LUSTRE/tests/racer/racer.sh check_and_setup_lustre -# first unmount all the lustre client +# first unmount all the lustre clients cleanup_mount $MOUNT -# mount lustre on mds -lustre_client=$(facet_active_host $SINGLEMDS) +# lustre client used as nfs server (default is mds node) +LUSTRE_CLIENT_NFSSRV=${LUSTRE_CLIENT_NFSSRV:-$(facet_active_host $SINGLEMDS)} +NFS_SRVMNTPT=${NFS_SRVMNTPT:-$MOUNT} +NFS_CLIENTS=${NFS_CLIENTS:-$CLIENTS} +NFS_CLIENTS=$(exclude_items_from_list $NFS_CLIENTS $LUSTRE_CLIENT_NFSSRV) +NFS_CLIMNTPT=${NFS_CLIMNTPT:-$MOUNT} + +[ -z "$NFS_CLIENTS" ] && + skip_env "need at least two nodes: nfs server and nfs client" && exit 0 + [ "$NFSVERSION" = "4" ] && cl_mnt_opt="${MOUNT_OPTS:+$MOUNT_OPTS,}32bitapi" || cl_mnt_opt="" -zconf_mount_clients $lustre_client $MOUNT "$cl_mnt_opt" || \ - error "mount lustre on $lustre_client failed" + +cleanup_exit () { + trap 0 + cleanup + check_and_cleanup_lustre + exit +} + +cleanup () { + cleanup_nfs "$NFS_CLIMNTPT" "$LUSTRE_CLIENT_NFSSRV" "$NFS_CLIENTS" || + error_noexit false "failed to cleanup nfs" + zconf_umount $LUSTRE_CLIENT_NFSSRV $NFS_SRVMNTPT force || + error_noexit false "failed to umount lustre on"\ + "$LUSTRE_CLIENT_NFSSRV" + # restore lustre mount + restore_mount $MOUNT || + error_noexit false "failed to mount lustre" +} + +trap cleanup_exit EXIT SIGHUP SIGINT + +zconf_mount $LUSTRE_CLIENT_NFSSRV $NFS_SRVMNTPT "$cl_mnt_opt" || + error "mount lustre on $LUSTRE_CLIENT_NFSSRV failed" # setup the nfs -if ! setup_nfs "$NFSVERSION" "$MOUNT" "$lustre_client" "$CLIENTS"; then - error_noexit false "setup nfs failed!" - cleanup_nfs "$MOUNT" "$lustre_client" "$CLIENTS" || \ - error_noexit false "failed to cleanup nfs" - if ! zconf_umount_clients $lustre_client $MOUNT force; then - error_noexit false "failed to umount lustre on $lustre_client" - elif ! zconf_mount_clients $CLIENTS $MOUNT; then - error_noexit false "failed to mount lustre" - fi - check_and_cleanup_lustre - exit -fi +setup_nfs "$NFSVERSION" "$NFS_SRVMNTPT" "$LUSTRE_CLIENT_NFSSRV" \ + "$NFS_CLIENTS" "$NFS_CLIMNTPT" || + error false "setup nfs failed!" NFSCLIENT=true FAIL_ON_ERROR=false # common setup MACHINEFILE=${MACHINEFILE:-$TMP/$(basename $0 .sh).machines} -clients=${CLIENTS:-$HOSTNAME} +clients=${NFS_CLIENTS:-$HOSTNAME} generate_machine_file $clients $MACHINEFILE || \ error "Failed to generate machine file" num_clients=$(get_node_count ${clients//,/ }) @@ -75,27 +95,27 @@ MPI_RUNAS=${MPI_RUNAS:-"runas -u $MPI_USER_UID -g $MPI_USER_GID"} $GSS_KRB5 && refresh_krb5_tgt $MPI_USER_UID $MPI_USER_GID $MPI_RUNAS test_compilebench() { - run_compilebench + run_compilebench $NFS_CLIMNTPT } run_test compilebench "compilebench" test_metabench() { - run_metabench + run_metabench $NFS_CLIMNTPT } run_test metabench "metabench" test_connectathon() { - run_connectathon + run_connectathon $NFS_CLIMNTPT } run_test connectathon "connectathon" test_iorssf() { - run_ior "ssf" + run_ior "ssf" $NFS_CLIMNTPT $NFS_SRVMNTPT } run_test iorssf "iorssf" test_iorfpp() { - run_ior "fpp" + run_ior "fpp" $NFS_CLIMNTPT $NFS_SRVMNTPT } run_test iorfpp "iorfpp" @@ -104,15 +124,5 @@ test_racer_on_nfs() { } run_test racer_on_nfs "racer on NFS client" -# cleanup nfs -cleanup_nfs "$MOUNT" "$lustre_client" "$CLIENTS" || \ - error_noexit false "cleanup_nfs failed" -if ! zconf_umount_clients $lustre_client $MOUNT force; then - error_noexit false "failed to umount lustre on $lustre_client" -elif ! zconf_mount_clients $CLIENTS $MOUNT; then - error_noexit false "failed to mount lustre after nfs test" -fi - complete $SECONDS -check_and_cleanup_lustre exit_status diff --git a/lustre/tests/setup-nfs.sh b/lustre/tests/setup-nfs.sh index ea2eb19..284381b 100755 --- a/lustre/tests/setup-nfs.sh +++ b/lustre/tests/setup-nfs.sh @@ -3,10 +3,12 @@ EXPORT_OPTS=${EXPORT_OPTS:-"rw,async,no_root_squash"} setup_nfs() { - local NFS_VER=${1} - local MNTPNT=${2} - local LUSTRE_CLIENT=${3} - local NFS_CLIENTS=${4} + local NFS_VER=${1} + local MNTPNT=${2} + local LUSTRE_CLIENT=${3} + local NFS_CLIENTS=${4} + local nfs_climntpt=${5:-$MNTPNT} + local export_opts_v=$EXPORT_OPTS echo "Exporting Lustre filesystem..." @@ -34,25 +36,26 @@ setup_nfs() { echo -e "\nMounting NFS clients (version $NFS_VER)..." - do_nodes $NFS_CLIENTS "mkdir -p $MNTPNT" || return 1 - if [ "$NFS_VER" = "4" ]; then - do_nodes $NFS_CLIENTS \ - "mount -t nfs$NFS_VER -o async $LUSTRE_CLIENT:/ $MNTPNT" || return 1 - else - do_nodes $NFS_CLIENTS \ - "mount -t nfs -o nfsvers=$NFS_VER,async \ - $LUSTRE_CLIENT:$MNTPNT $MNTPNT" || return 1 - fi - return 0 + do_nodes $NFS_CLIENTS "mkdir -p $nfs_climntpt" || return 1 + if [ "$NFS_VER" = "4" ]; then + do_nodes $NFS_CLIENTS \ + "mount -t nfs$NFS_VER -o async \ + $LUSTRE_CLIENT:/ $nfs_climntpt" || return 1 + else + do_nodes $NFS_CLIENTS \ + "mount -t nfs -o nfsvers=$NFS_VER,async \ + $LUSTRE_CLIENT:$MNTPNT $nfs_climntpt" || return 1 + fi + return 0 } cleanup_nfs() { - local MNTPNT=${1} - local LUSTRE_CLIENT=${2} - local NFS_CLIENTS=${3} + local MNTPNT=${1} + local LUSTRE_CLIENT=${2} + local NFS_CLIENTS=${3} - echo -e "\nUnmounting NFS clients..." - do_nodes $NFS_CLIENTS "umount -f $MNTPNT" || return 1 + echo -e "\nUnmounting NFS clients..." + do_nodes $NFS_CLIENTS "umount -f $MNTPNT" || true echo -e "\nUnexporting Lustre filesystem..." do_nodes $NFS_CLIENTS "chkconfig --list rpcidmapd 2>/dev/null | diff --git a/lustre/tests/test-framework.sh b/lustre/tests/test-framework.sh index 1c0533e..e4a7ec8 100755 --- a/lustre/tests/test-framework.sh +++ b/lustre/tests/test-framework.sh @@ -5667,13 +5667,19 @@ get_stripe () { setstripe_nfsserver () { local dir=$1 + local nfsexportdir=$2 + shift + shift - local nfsserver=$(awk '"'$dir'" ~ $2 && $3 ~ "nfs" && $2 != "/" \ - { print $1 }' /proc/mounts | cut -f 1 -d : | head -n1) + local -a nfsexport=($(awk '"'$dir'" ~ $2 && $3 ~ "nfs" && $2 != "/" \ + { print $1 }' /proc/mounts | cut -f 1 -d :)) - [ -z $nfsserver ] && echo "$dir is not nfs mounted" && return 1 + # check that only one nfs mounted + [[ -z $nfsexport ]] && echo "$dir is not nfs mounted" && return 1 + (( ${#nfsexport[@]} == 1 )) || + error "several nfs mounts found for $dir: ${nfsexport[@]} !" - do_nodev $nfsserver lfs setstripe "$@" + do_nodev ${nfsexport[0]} lfs setstripe $nfsexportdir "$@" } # Check and add a test group. -- 1.8.3.1