From: Yu Jian Date: Tue, 10 Apr 2012 10:23:02 +0000 (+0800) Subject: LU-427 test: Test failure on test suite lfsck X-Git-Tag: v1_8_7_81_WC1~6 X-Git-Url: https://git.whamcloud.com/?a=commitdiff_plain;h=33ebd2bb569180b6a47fa368741f5728c8cbf822;p=fs%2Flustre-release.git LU-427 test: Test failure on test suite lfsck - Reset $MDSDB & $OSTDB in generate_db(). Else they will stale if user redefine $SHARED_DIRECTORY. - Add a function check_shared_dir() to ensure $SHARED_DIRECTORY is shared among tests nodes. - Fix check_logdir() and check_write_access() to avoid using node.$(hostname).yml files which should not be deleted. Signed-off-by: Yang Sheng Signed-off-by: Yu Jian Change-Id: Ie7d1d08c0d2c701fa9fb74ef8b252fa8b31bf111 Reviewed-on: http://review.whamcloud.com/2498 Tested-by: Hudson Reviewed-by: Wei Liu Tested-by: Maloo Reviewed-by: Johann Lombardi --- diff --git a/lustre/tests/cfg/local.sh b/lustre/tests/cfg/local.sh index e6dcc95..fef5646 100644 --- a/lustre/tests/cfg/local.sh +++ b/lustre/tests/cfg/local.sh @@ -112,3 +112,7 @@ MPIRUN=$(which mpirun 2>/dev/null) || true MPI_USER=${MPI_USER:-mpiuser} SHARED_DIR_LOGS=${SHARED_DIR_LOGS:-""} +# This is used by a small number of tests to share state between the client +# running the tests, or in some cases between the servers (e.g. lfsck.sh). +# It needs to be a non-lustre filesystem that is available on all the nodes. +SHARED_DIRECTORY=${SHARED_DIRECTORY:-""} # bug 17839 comment 65 diff --git a/lustre/tests/cfg/ncli.sh b/lustre/tests/cfg/ncli.sh index ca0b7bd..0837eb5 100644 --- a/lustre/tests/cfg/ncli.sh +++ b/lustre/tests/cfg/ncli.sh @@ -14,7 +14,6 @@ init_clients_lists # for recovery scale tests # default boulder cluster iozone location export PATH=/opt/iozone/bin:$PATH -SHARED_DIRECTORY=${SHARED_DIRECTORY:-""} # bug 17839 comment 65 LOADS=${LOADS:-"dd tar dbench iozone"} for i in $LOADS; do [ -f $LUSTRE/tests/run_${i}.sh ] || \ diff --git a/lustre/tests/recovery-double-scale.sh b/lustre/tests/recovery-double-scale.sh index 4dff18f..c0bbad4 100644 --- a/lustre/tests/recovery-double-scale.sh +++ b/lustre/tests/recovery-double-scale.sh @@ -32,6 +32,9 @@ set -x [ "$SHARED_DIRECTORY" ] || \ { FAIL_ON_ERROR=true skip_env "$0 Empty SHARED_DIRECTORY" && exit 0; } +check_shared_dir $SHARED_DIRECTORY || + error "$SHARED_DIRECTORY isn't a shared directory" + [ -n "$CLIENTS" ] || \ { FAIL_ON_ERROR=true skip_env "$0 Need two or more remote clients" && exit 0; } diff --git a/lustre/tests/recovery-mds-scale.sh b/lustre/tests/recovery-mds-scale.sh index 496c71c..6bbc86c 100644 --- a/lustre/tests/recovery-mds-scale.sh +++ b/lustre/tests/recovery-mds-scale.sh @@ -30,6 +30,9 @@ set -x [ "$SHARED_DIRECTORY" ] || \ { FAIL_ON_ERROR=true skip_env "$0 Empty SHARED_DIRECTORY" && exit 0; } +check_shared_dir $SHARED_DIRECTORY || + error "$SHARED_DIRECTORY isn't a shared directory" + [ -n "$CLIENTS" ] || \ { FAIL_ON_ERROR=true skip_env "$0 Need two or more remote clients" && exit 0; } diff --git a/lustre/tests/recovery-random-scale.sh b/lustre/tests/recovery-random-scale.sh index fb281e1..5b79583 100644 --- a/lustre/tests/recovery-random-scale.sh +++ b/lustre/tests/recovery-random-scale.sh @@ -34,6 +34,9 @@ set -x [ "$SHARED_DIRECTORY" ] || \ { FAIL_ON_ERROR=true skip_env "$0 Empty SHARED_DIRECTORY" && exit 0; } +check_shared_dir $SHARED_DIRECTORY || + error "$SHARED_DIRECTORY isn't a shared directory" + [ -n "$CLIENTS" ] || \ { FAIL_ON_ERROR=true skip_env "$0 Need two or more remote clients" && exit 0; } diff --git a/lustre/tests/test-framework.sh b/lustre/tests/test-framework.sh index d152a89..6de0445 100644 --- a/lustre/tests/test-framework.sh +++ b/lustre/tests/test-framework.sh @@ -122,9 +122,6 @@ init_test_env() { export LFSCK_ALWAYS=${LFSCK_ALWAYS:-"no"} # check fs after each test suite export FSCK_MAX_ERR=4 # File system errors left uncorrected - export SHARED_DIRECTORY=${SHARED_DIRECTORY:-"/tmp"} - export MDSDB=${MDSDB:-$SHARED_DIRECTORY/mdsdb} - export OSTDB=${OSTDB:-$SHARED_DIRECTORY/ostdb} declare -a OSTDEVS #[ -d /r ] && export ROOT=${ROOT:-/r} @@ -2231,23 +2228,27 @@ run_e2fsck() { return 0 } +# verify a directory is shared among nodes. +check_shared_dir() { + local dir=$1 + + [ -z "$dir" ] && return 1 + do_rpc_nodes $(comma_list $(nodes_list)) check_logdir $dir + check_write_access $dir || return 1 + return 0 +} + # Run e2fsck on MDT and OST(s) to generate databases used for lfsck. generate_db() { local i local ostidx local dev - local tmp_file - - tmp_file=$(mktemp -p $SHARED_DIRECTORY || - error_exit "fail to create file in $SHARED_DIRECTORY") - # make sure everything gets to the backing store - local list=$(comma_list $CLIENTS $(facet_host mds) $(osts_nodes)) - do_nodes $list "sync; sleep 2; sync" + check_shared_dir $SHARED_DIRECTORY || + error "$SHARED_DIRECTORY isn't a shared directory" - do_nodes $list ls $tmp_file || \ - error_exit "$SHARED_DIRECTORY is not a shared directory" - rm $tmp_file + export MDSDB=$SHARED_DIRECTORY/mdsdb + export OSTDB=$SHARED_DIRECTORY/ostdb run_e2fsck $(facet_host mds) $MDSDEV "--mdsdb $MDSDB" @@ -3887,18 +3888,23 @@ check_logdir() { # Not found. Create local logdir mkdir -p $dir else - touch $dir/node.$(hostname).yml + touch $dir/check_file.$(hostname) fi return 0 } check_write_access() { local dir=$1 + local node + local file + for node in $(nodes_list); do - if [ ! -f "$dir/node.${node}.yml" ]; then + file=$dir/check_file.$node + if [[ ! -f $file ]]; then # Logdir not accessible/writable from this node. return 1 fi + rm -f $file || return 1 done return 0 } @@ -3911,18 +3917,18 @@ init_logging() { mkdir -p $LOGDIR init_clients_lists - if [ ! -f $YAML_LOG ]; then # If the yaml log already exists then we will just append to it - do_rpc_nodes $(comma_list $(nodes_list)) check_logdir $LOGDIR - if check_write_access $LOGDIR; then - touch $LOGDIR/shared - echo "Logging to shared log directory: $LOGDIR" - else - echo "Logging to local directory: $LOGDIR" - fi + # If the yaml log already exists then we will just append to it. + [[ -f $YAML_LOG ]] && return 0 - yml_nodes_file $LOGDIR >> $YAML_LOG - yml_results_file >> $YAML_LOG + if check_shared_dir $LOGDIR; then + touch $LOGDIR/shared + echo "Logging to shared log directory: $LOGDIR" + else + echo "Logging to local directory: $LOGDIR" fi + + yml_nodes_file $LOGDIR >> $YAML_LOG + yml_results_file >> $YAML_LOG } log_test() {