From: Andreas Dilger Date: Tue, 13 Aug 2019 21:50:03 +0000 (-0600) Subject: LU-1538 tests: standardize test script init - dne-part-2 X-Git-Tag: 2.13.51~202 X-Git-Url: https://git.whamcloud.com/?p=fs%2Flustre-release.git;a=commitdiff_plain;h=c8790ae52393d96fc71aa4edea0e5051c54431e9;ds=sidebyside LU-1538 tests: standardize test script init - dne-part-2 Standardize the initial Lustre test script initialization for clarity and consistency for test suites in review-dne-part-2. The LUSTRE path is already normalized in init_test_env(), so this doesn't need to be done in the caller. Use $(...) subshells instead of `...` in the affected lines. Remove NAME, CHECKSTAT, TMP, SAVE_PWD,SRCDIR, PATH, MULTIOP, SETUP, CLEANUP variable initialization, since it is already done in init_test_env() or not needed in the test script. Remove all calls to get_lustre_env() in the test scripts since this is called in init_test_env(). Move all definitions of ALWAYS_EXCEPT and SLOW to after init_test_env() and init_logging() and call build_test_filter() immediately after the ALWAYS_EXCEPT and SLOW definitions. Test-Parameters: trivial Test-Parameters: testgroup=review-dne-part-2 Signed-off-by: James Nunez Change-Id: I6a04c9ffc0ce965c7f170119814d6ee8a30631df Reviewed-on: https://review.whamcloud.com/35787 Tested-by: jenkins Tested-by: Maloo Reviewed-by: Emoly Liu Reviewed-by: Wei Liu Reviewed-by: Oleg Drokin --- diff --git a/lustre/tests/large-scale.sh b/lustre/tests/large-scale.sh index 9fd35a7..7973edb 100644 --- a/lustre/tests/large-scale.sh +++ b/lustre/tests/large-scale.sh @@ -1,34 +1,27 @@ #!/bin/bash -# -*- mode: Bash; tab-width: 4; indent-tabs-mode: t; -*- -# vim:shiftwidth=4:softtabstop=4:tabstop=4: set -e -# bug number: -ALWAYS_EXCEPT="$LARGE_SCALE_EXCEPT" - -SAVE_PWD=$PWD PTLDEBUG=${PTLDEBUG:--1} -LUSTRE=${LUSTRE:-`dirname $0`/..} SETUP=${SETUP:-""} CLEANUP=${CLEANUP:-""} -. $LUSTRE/tests/test-framework.sh +LUSTRE=${LUSTRE:-$(dirname $0)/..} +. $LUSTRE/tests/test-framework.sh init_test_env $@ - -. ${CONFIG:=$LUSTRE/tests/cfg/$NAME.sh} init_logging +ALWAYS_EXCEPT="$LARGE_SCALE_EXCEPT " + +build_test_filter + remote_mds_nodsh && skip "remote MDS with nodsh" [ -z "$CLIENTS" ] && skip_env "$TESTSUITE: Need two or more clients" [ $CLIENTCOUNT -lt 2 ] && skip_env "$TESTSUITE: Need 2+ clients, have only $CLIENTCOUNT" -[ "$SLOW" = "no" ] && EXCEPT_SLOW="" - MOUNT_2="" -build_test_filter check_and_setup_lustre rm -rf $DIR/[df][0-9]* @@ -40,49 +33,42 @@ $GSS_KRB5 && refresh_krb5_tgt $MPI_USER_UID $MPI_USER_GID $MPI_RUNAS [ "$DAEMONFILE" ] && $LCTL debug_daemon start $DAEMONFILE $DAEMONSIZE test_3a() { - assert_env CLIENTS MDSRATE MPIRUN - - local -a nodes=(${CLIENTS//,/ }) - - # INCREMENT is a number of clients - # a half of clients by default - increment=${INCREMENT:-$(( CLIENTCOUNT / 2 ))} - - machinefile=${MACHINEFILE:-$TMP/$TESTSUITE.machines} - local LOG=$TMP/${TESTSUITE}_$tfile - - local var=${SINGLEMDS}_svc - local procfile="*.${!var}.recovery_status" - local iters=${ITERS:-3} - local nfiles=${NFILES:-50000} - local nthreads=${THREADS_PER_CLIENT:-3} - - local IFree=$(inodes_available) - [ $IFree -gt $nfiles ] || nfiles=$IFree - - local dir=$DIR/d0.$TESTNAME - mkdir -p $dir - chmod 0777 $dir - - local pid - local list - local -a res - - local num=$increment + assert_env CLIENTS MDSRATE MPIRUN + + local -a nodes=(${CLIENTS//,/ }) + # INCREMENT is a number of clients a half of clients by default + local increment=${INCREMENT:-$(( CLIENTCOUNT / 2 ))} + local num=$increment + local LOG=$TMP/${TESTSUITE}_$tfile + local var=${SINGLEMDS}_svc + local procfile="*.${!var}.recovery_status" + local iters=${ITERS:-3} + local nfiles=${NFILES:-50000} + local nthreads=${THREADS_PER_CLIENT:-3} + local IFree=$(inodes_available) + local pid + local list + local -a res + local dir=$DIR/d0.$TESTNAME + + [ $IFree -gt $nfiles ] || nfiles=$IFree + + mkdir -p $dir + chmod 0777 $dir while [ $num -le $CLIENTCOUNT ]; do list=$(comma_list ${nodes[@]:0:$num}) - generate_machine_file $list $machinefile || - { error "can not generate machinefile"; exit 1; } + generate_machine_file $list $MACHINEFILE || + error "can not generate machinefile" for i in $(seq $iters); do - mdsrate_cleanup $num $machinefile $nfiles $dir 'f%%d' \ + mdsrate_cleanup $num $MACHINEFILE $nfiles $dir 'f%%d' \ --ignore COMMAND="${MDSRATE} --create --nfiles $nfiles --dir $dir --filefmt 'f%%d'" - mpi_run ${MACHINEFILE_OPTION} $machinefile \ + mpi_run ${MACHINEFILE_OPTION} $MACHINEFILE \ -np $((num * nthreads)) ${COMMAND} | tee ${LOG}& pid=$! @@ -111,13 +97,13 @@ test_3a() { num=$((num + increment)) done - mdsrate_cleanup $num $machinefile $nfiles $dir 'f%%d' --ignore + mdsrate_cleanup $num $MACHINEFILE $nfiles $dir 'f%%d' --ignore - i=0 - while [ $i -lt ${#res[@]} ]; do - echo "RECOVERY TIME: NFILES=$nfiles number of clients: ${res[i]} ${res[i+1]}" - i=$((i+2)) - done + i=0 + while [ $i -lt ${#res[@]} ]; do + echo "RECOVERY TIME: NFILES=$nfiles number of clients: ${res[i]} ${res[i+1]}" + i=$((i+2)) + done } run_test 3a "recovery time, $CLIENTCOUNT clients" diff --git a/lustre/tests/mds-survey.sh b/lustre/tests/mds-survey.sh index e95452d..3d1ca65 100644 --- a/lustre/tests/mds-survey.sh +++ b/lustre/tests/mds-survey.sh @@ -1,22 +1,14 @@ #!/bin/bash -#set -x set -e -LUSTRE=${LUSTRE:-`dirname $0`/..} +LUSTRE=${LUSTRE:-$(dirname $0)/..} . $LUSTRE/tests/test-framework.sh init_test_env $@ -. ${CONFIG:=$LUSTRE/tests/cfg/$NAME.sh} init_logging -file_count=${file_count:-150000} -dir_count=${dir_count:-4} -thrhi=${thrhi:-8} -thrlo=${thrlo:-1} +ALWAYS_EXCEPT="$MDS_SURVEY_EXCEPT " -[ "$SLOW" = no ] && { file_count=50000; dir_count=2; thrhi=4; } - -# Skip these tests -ALWAYS_EXCEPT="$MDS_SURVEY_EXCEPT" +build_test_filter MDSSURVEY=${MDSSURVEY:-$(which mds-survey 2>/dev/null || true)} [ -z ${MDSSURVEY} ] && skip_env "mds-survey not found" @@ -25,7 +17,13 @@ if [ $(lustre_version_code $SINGLEMDS) -lt $(version_code 2.3.51) ]; then skip_env "Need MDS version at least 2.3.51" fi -build_test_filter +file_count=${file_count:-150000} +dir_count=${dir_count:-4} +thrhi=${thrhi:-8} +thrlo=${thrlo:-1} + +[ "$SLOW" = no ] && { file_count=50000; dir_count=2; thrhi=4; } + check_and_setup_lustre adjust_inode() { diff --git a/lustre/tests/ost-pools.sh b/lustre/tests/ost-pools.sh index e91fb95..2f1700f 100755 --- a/lustre/tests/ost-pools.sh +++ b/lustre/tests/ost-pools.sh @@ -4,26 +4,20 @@ # Skip specific tests by setting EXCEPT. # -SRCDIR=$(dirname $0) -export PATH=$PWD/$SRCDIR:$SRCDIR:$PWD/$SRCDIR/../utils:$PATH:/sbin - ONLY=${ONLY:-"$*"} -ALWAYS_EXCEPT="$OST_POOLS_EXCEPT" -# bug number for skipped test: - -# UPDATE THE COMMENT ABOVE WITH BUG NUMBERS WHEN CHANGING ALWAYS_EXCEPT! - -[ "$ALWAYS_EXCEPT$EXCEPT" ] && - echo "Skipping tests: $(echo $ALWAYS_EXCEPT $EXCEPT)" - -TMP=${TMP:-/tmp} ORIG_PWD=${PWD} -LUSTRE=${LUSTRE:-$(cd $(dirname $0)/..; echo $PWD)} +LUSTRE=${LUSTRE:-$(dirname $0)/..} . $LUSTRE/tests/test-framework.sh init_test_env $@ -. ${CONFIG:=$LUSTRE/tests/cfg/$NAME.sh} init_logging +ALWAYS_EXCEPT="$OST_POOLS_EXCEPT" +# bug number for skipped test: - +# UPDATE THE COMMENT ABOVE WITH BUG NUMBERS WHEN CHANGING ALWAYS_EXCEPT! + +build_test_filter + check_and_setup_lustre if ! combined_mgs_mds; then @@ -35,8 +29,6 @@ fi DIR=${DIR:-$MOUNT} assert_DIR -build_test_filter - MAXFREE=${MAXFREE:-$((2000000 * OSTCOUNT))} # OST pools tests diff --git a/lustre/tests/runtests b/lustre/tests/runtests index dd64560..6e80caf 100755 --- a/lustre/tests/runtests +++ b/lustre/tests/runtests @@ -1,18 +1,10 @@ #!/bin/bash # # Script which does some basic tests to ensure we haven't regressed. -# Probably a good idea to run this before doing any checkins. -# In the future this can become more fancy, but it's OK for now. LUSTRE=${LUSTRE:-$(dirname $0)/..} -SRCDIR="$(dirname $0)" -export PATH=/sbin:/usr/sbin:$SRCDIR:$SRCDIR/../utils:$PATH - -export NAME=${NAME:-local} - . $LUSTRE/tests/test-framework.sh init_test_env $@ -. ${CONFIG:=$LUSTRE/tests/cfg/$NAME.sh} init_logging RUNTESTS_SRC=${RUNTESTS_SRC:-"/etc /bin"} @@ -23,6 +15,7 @@ RUNTESTS_SRC=${RUNTESTS_SRC:-"/etc /bin"} [ "$RMDIRMANY" ] || RMDIRMANY="unlinkmany -d" check_and_setup_lustre + test_1() { sleep 5 # let MDS refresh aggregated statfs # Include some extra space for the status file diff --git a/lustre/tests/sanity-hsm.sh b/lustre/tests/sanity-hsm.sh index 7a4d3c2..ae49009 100755 --- a/lustre/tests/sanity-hsm.sh +++ b/lustre/tests/sanity-hsm.sh @@ -7,29 +7,24 @@ set -e set +o monitor -SRCDIR=$(dirname $0) -export PATH=$PWD/$SRCDIR:$SRCDIR:$PWD/$SRCDIR/utils:$PATH:/sbin:/usr/sbin - ONLY=${ONLY:-"$*"} -# bug number for skipped test: -ALWAYS_EXCEPT="$SANITY_HSM_EXCEPT" -# UPDATE THE COMMENT ABOVE WITH BUG NUMBERS WHEN CHANGING ALWAYS_EXCEPT! -if $SHARED_KEY; then -# bug number for skipped tests: LU-9795 LU-9795 - ALWAYS_EXCEPT=" 13 402b $ALWAYS_EXCEPT" -fi - -LUSTRE=${LUSTRE:-$(cd $(dirname $0)/..; echo $PWD)} +LUSTRE=${LUSTRE:-$(dirname $0)/..} . $LUSTRE/tests/test-framework.sh init_test_env $@ -. ${CONFIG:=$LUSTRE/tests/cfg/$NAME.sh} -get_lustre_env init_logging +ALWAYS_EXCEPT="$SANITY_HSM_EXCEPT " +if $SHARED_KEY; then +# bug number for skipped tests: LU-9795 LU-9795 + ALWAYS_EXCEPT+=" 13 402b" +# UPDATE THE COMMENT ABOVE WITH BUG NUMBERS WHEN CHANGING ALWAYS_EXCEPT! +fi + +build_test_filter + [ -n "$FILESET" ] && skip "Not functional for FILESET set" -MULTIOP=${MULTIOP:-multiop} OPENFILE=${OPENFILE:-openfile} MMAP_CAT=${MMAP_CAT:-mmap_cat} MOUNT_2=${MOUNT_2:-"yes"} @@ -37,7 +32,7 @@ FAIL_ON_ERROR=false # script only handles up to 10 MDTs (because of MDT_PREFIX) [ $MDSCOUNT -gt 9 ] && - error "script cannot handle more than 9 MDTs, please fix" && exit + error "script cannot handle more than 9 MDTs, please fix" check_and_setup_lustre @@ -47,7 +42,7 @@ fi # $RUNAS_ID may get set incorrectly somewhere else if [[ $UID -eq 0 && $RUNAS_ID -eq 0 ]]; then - skip_env "\$RUNAS_ID set to 0, but \$UID is also 0!" && exit + skip_env "\$RUNAS_ID set to 0, but \$UID is also 0!" fi check_runas_id $RUNAS_ID $RUNAS_GID $RUNAS if getent group nobody; then @@ -58,8 +53,6 @@ else error "No generic nobody group" fi -build_test_filter - # if there is no CLIENT1 defined, some tests can be ran on localhost CLIENT1=${CLIENT1:-$HOSTNAME} # if CLIENT2 doesn't exist then use CLIENT1 instead diff --git a/lustre/tests/sanity-lfsck.sh b/lustre/tests/sanity-lfsck.sh index 2107436..3e9e1b4 100644 --- a/lustre/tests/sanity-lfsck.sh +++ b/lustre/tests/sanity-lfsck.sh @@ -8,18 +8,31 @@ set -e ONLY=${ONLY:-"$*"} -#Bug number for excepting test -ALWAYS_EXCEPT="$SANITY_LFSCK_EXCEPT" - -[ "$SLOW" = "no" ] && EXCEPT_SLOW="" -# UPDATE THE COMMENT ABOVE WITH BUG NUMBERS WHEN CHANGING ALWAYS_EXCEPT! - -LUSTRE=${LUSTRE:-$(cd $(dirname $0)/..; echo $PWD)} +LUSTRE=${LUSTRE:-$(dirname $0)/..} . $LUSTRE/tests/test-framework.sh init_test_env $@ -. ${CONFIG:=$LUSTRE/tests/cfg/$NAME.sh} init_logging +ALWAYS_EXCEPT="$SANITY_LFSCK_EXCEPT " + +# DNE does not support striped directory on zfs-based backend yet. +[ $(facet_fstype $SINGLEMDS) != ldiskfs ] && +#Bug number for excepting test + ALWAYS_EXCEPT+=" 31" +# UPDATE THE COMMENT ABOVE WITH BUG NUMBERS WHEN CHANGING ALWAYS_EXCEPT! + +[[ $(lustre_version_code $SINGLEMDS) -le $(version_code 2.4.90) ]] && + ALWAYS_EXCEPT="$ALWAYS_EXCEPT 2c" + +[[ $(lustre_version_code ost1) -lt $(version_code 2.5.55) ]] && + ALWAYS_EXCEPT="$ALWAYS_EXCEPT 11 12 13 14 15 16 17 18 19 20 21" + +[[ $(lustre_version_code $SINGLEMDS) -lt $(version_code 2.6.50) ]] && + ALWAYS_EXCEPT="$ALWAYS_EXCEPT 2d 2e 3 22 23 24 25 26 27 28 29 30 31" + +[ "$SLOW" = "no" ] && EXCEPT_SLOW="" +build_test_filter + require_dsh_mds || exit 0 load_modules @@ -51,21 +64,6 @@ cleanupall # build up a clean test environment. REFORMAT="yes" check_and_setup_lustre -[[ $(lustre_version_code $SINGLEMDS) -le $(version_code 2.4.90) ]] && - ALWAYS_EXCEPT="$ALWAYS_EXCEPT 2c" - -[[ $(lustre_version_code ost1) -lt $(version_code 2.5.55) ]] && - ALWAYS_EXCEPT="$ALWAYS_EXCEPT 11 12 13 14 15 16 17 18 19 20 21" - -[[ $(lustre_version_code $SINGLEMDS) -lt $(version_code 2.6.50) ]] && - ALWAYS_EXCEPT="$ALWAYS_EXCEPT 2d 2e 3 22 23 24 25 26 27 28 29 30 31" - -# DNE does not support striped directory on zfs-based backend yet. -[ $(facet_fstype $SINGLEMDS) != ldiskfs ] && - ALWAYS_EXCEPT="$ALWAYS_EXCEPT 31" - -build_test_filter - MDT_DEV="${FSNAME}-MDT0000" OST_DEV="${FSNAME}-OST0000" MDT_DEVNAME=$(mdsdevname ${SINGLEMDS//mds/}) diff --git a/lustre/tests/sanity-scrub.sh b/lustre/tests/sanity-scrub.sh index c84ce25..dae61a3 100644 --- a/lustre/tests/sanity-scrub.sh +++ b/lustre/tests/sanity-scrub.sh @@ -7,16 +7,18 @@ set -e ONLY=${ONLY:-"$*"} + +LUSTRE=${LUSTRE:-$(dirname $0)/..} +. $LUSTRE/tests/test-framework.sh +init_test_env $@ +init_logging + ALWAYS_EXCEPT="$SANITY_SCRUB_EXCEPT" [ "$SLOW" = "no" ] && EXCEPT_SLOW="" # UPDATE THE COMMENT ABOVE WITH BUG NUMBERS WHEN CHANGING ALWAYS_EXCEPT! -LUSTRE=${LUSTRE:-$(cd $(dirname $0)/..; echo $PWD)} -. $LUSTRE/tests/test-framework.sh -init_test_env $@ -. ${CONFIG:=$LUSTRE/tests/cfg/$NAME.sh} -init_logging +build_test_filter require_dsh_mds || exit 0 @@ -45,8 +47,6 @@ OSTSIZE=200000 # build up a clean test environment. REFORMAT="yes" check_and_setup_lustre -build_test_filter - MDT_DEV="${FSNAME}-MDT0000" OST_DEV="${FSNAME}-OST0000" diff --git a/lustre/tests/sanity-sec.sh b/lustre/tests/sanity-sec.sh index 79901a5..949e146 100755 --- a/lustre/tests/sanity-sec.sh +++ b/lustre/tests/sanity-sec.sh @@ -7,21 +7,20 @@ set -e ONLY=${ONLY:-"$*"} -# bug number for skipped test: -ALWAYS_EXCEPT=" $SANITY_SEC_EXCEPT" -# UPDATE THE COMMENT ABOVE WITH BUG NUMBERS WHEN CHANGING ALWAYS_EXCEPT! - -SRCDIR=$(dirname $0) -export PATH=$PWD/$SRCDIR:$SRCDIR:$PWD/$SRCDIR/../utils:$PATH:/sbin -export NAME=${NAME:-local} LUSTRE=${LUSTRE:-$(dirname $0)/..} . $LUSTRE/tests/test-framework.sh init_test_env $@ -. ${CONFIG:=$LUSTRE/tests/cfg/$NAME.sh} -get_lustre_env + init_logging +ALWAYS_EXCEPT="$SANITY_SEC_EXCEPT " +# bug number for skipped test: +ALWAYS_EXCEPT+=" " +# UPDATE THE COMMENT ABOVE WITH BUG NUMBERS WHEN CHANGING ALWAYS_EXCEPT! + +[ "$SLOW" = "no" ] && EXCEPT_SLOW="26" + NODEMAP_TESTS=$(seq 7 26) if ! check_versions; then @@ -29,10 +28,7 @@ if ! check_versions; then EXCEPT="$EXCEPT $NODEMAP_TESTS" fi -[ "$SLOW" = "no" ] && EXCEPT_SLOW="26" - -[ "$ALWAYS_EXCEPT$EXCEPT$EXCEPT_SLOW" ] && - echo "Skipping tests: $ALWAYS_EXCEPT $EXCEPT $EXCEPT_SLOW" +build_test_filter RUNAS_CMD=${RUNAS_CMD:-runas} @@ -99,8 +95,6 @@ IDENTITY_UPCALL=mdt.$MDT.identity_upcall SAVE_PWD=$PWD -build_test_filter - sec_login() { local user=$1 local group=$2