LU-427 test: Test failure on test suite lfsck

- Reset $MDSDB & $OSTDB in generate_db(). Else they will
  stale if user redefine $SHARED_DIRECTORY.
- Add a function check_shared_dir() to ensure
  $SHARED_DIRECTORY is shared among tests nodes.
- Fix check_logdir() and check_write_access() to avoid using
  node.$(hostname).yml files which should not be deleted.

Signed-off-by: Yang Sheng <ys@whamcloud.com>
Signed-off-by: Yu Jian <yujian@whamcloud.com>
Change-Id: Ie7d1d08c0d2c701fa9fb74ef8b252fa8b31bf111
Reviewed-on: http://review.whamcloud.com/2498
Tested-by: Hudson
Reviewed-by: Wei Liu <sarah@whamcloud.com>
Tested-by: Maloo <whamcloud.maloo@gmail.com>
Reviewed-by: Johann Lombardi <johann@whamcloud.com>

diff --git a/lustre/tests/cfg/local.sh b/lustre/tests/cfg/local.sh
index e6dcc95..fef5646 100644 (file)
--- a/lustre/tests/cfg/local.sh
+++ b/lustre/tests/cfg/local.sh
@@ -112,3 +112,7 @@ MPIRUN=$(which mpirun 2>/dev/null) || true
 MPI_USER=${MPI_USER:-mpiuser}
 SHARED_DIR_LOGS=${SHARED_DIR_LOGS:-""}
 
+# This is used by a small number of tests to share state between the client
+# running the tests, or in some cases between the servers (e.g. lfsck.sh).
+# It needs to be a non-lustre filesystem that is available on all the nodes.
+SHARED_DIRECTORY=${SHARED_DIRECTORY:-""}    # bug 17839 comment 65

diff --git a/lustre/tests/cfg/ncli.sh b/lustre/tests/cfg/ncli.sh
index ca0b7bd..0837eb5 100644 (file)
--- a/lustre/tests/cfg/ncli.sh
+++ b/lustre/tests/cfg/ncli.sh
@@ -14,7 +14,6 @@ init_clients_lists
 # for recovery scale tests
 # default boulder cluster iozone location
 export PATH=/opt/iozone/bin:$PATH
-SHARED_DIRECTORY=${SHARED_DIRECTORY:-""}       # bug 17839 comment 65
 LOADS=${LOADS:-"dd tar dbench iozone"}
 for i in $LOADS; do
     [ -f $LUSTRE/tests/run_${i}.sh ] || \

diff --git a/lustre/tests/recovery-double-scale.sh b/lustre/tests/recovery-double-scale.sh
index 4dff18f..c0bbad4 100644 (file)
--- a/lustre/tests/recovery-double-scale.sh
+++ b/lustre/tests/recovery-double-scale.sh
@@ -32,6 +32,9 @@ set -x
 [ "$SHARED_DIRECTORY" ] || \
     { FAIL_ON_ERROR=true skip_env "$0 Empty SHARED_DIRECTORY" && exit 0; }
 
+check_shared_dir $SHARED_DIRECTORY ||
+    error "$SHARED_DIRECTORY isn't a shared directory"
+
 [ -n "$CLIENTS" ] || \
     { FAIL_ON_ERROR=true skip_env "$0 Need two or more remote clients" && exit 0; }
 

diff --git a/lustre/tests/recovery-mds-scale.sh b/lustre/tests/recovery-mds-scale.sh
index 496c71c..6bbc86c 100644 (file)
--- a/lustre/tests/recovery-mds-scale.sh
+++ b/lustre/tests/recovery-mds-scale.sh
@@ -30,6 +30,9 @@ set -x
 [ "$SHARED_DIRECTORY" ] || \
     { FAIL_ON_ERROR=true skip_env "$0 Empty SHARED_DIRECTORY" && exit 0; }
 
+check_shared_dir $SHARED_DIRECTORY ||
+    error "$SHARED_DIRECTORY isn't a shared directory"
+
 [ -n "$CLIENTS" ] || \
     { FAIL_ON_ERROR=true skip_env "$0 Need two or more remote clients" && exit 0; }
 

diff --git a/lustre/tests/recovery-random-scale.sh b/lustre/tests/recovery-random-scale.sh
index fb281e1..5b79583 100644 (file)
--- a/lustre/tests/recovery-random-scale.sh
+++ b/lustre/tests/recovery-random-scale.sh
@@ -34,6 +34,9 @@ set -x
 [ "$SHARED_DIRECTORY" ] || \
     { FAIL_ON_ERROR=true skip_env "$0 Empty SHARED_DIRECTORY" && exit 0; }
 
+check_shared_dir $SHARED_DIRECTORY ||
+    error "$SHARED_DIRECTORY isn't a shared directory"
+
 [ -n "$CLIENTS" ] || \
     { FAIL_ON_ERROR=true skip_env "$0 Need two or more remote clients" && exit 0; }
 

diff --git a/lustre/tests/test-framework.sh b/lustre/tests/test-framework.sh
index d152a89..6de0445 100644 (file)
--- a/lustre/tests/test-framework.sh
+++ b/lustre/tests/test-framework.sh
@@ -122,9 +122,6 @@ init_test_env() {
 
     export LFSCK_ALWAYS=${LFSCK_ALWAYS:-"no"} # check fs after each test suite
     export FSCK_MAX_ERR=4   # File system errors left uncorrected
-    export SHARED_DIRECTORY=${SHARED_DIRECTORY:-"/tmp"}
-    export MDSDB=${MDSDB:-$SHARED_DIRECTORY/mdsdb}
-    export OSTDB=${OSTDB:-$SHARED_DIRECTORY/ostdb}
     declare -a OSTDEVS
 
     #[ -d /r ] && export ROOT=${ROOT:-/r}
@@ -2231,23 +2228,27 @@ run_e2fsck() {
     return 0
 }
 
+# verify a directory is shared among nodes.
+check_shared_dir() {
+    local dir=$1
+
+    [ -z "$dir" ] && return 1
+    do_rpc_nodes $(comma_list $(nodes_list)) check_logdir $dir
+    check_write_access $dir || return 1
+    return 0
+}
+
 # Run e2fsck on MDT and OST(s) to generate databases used for lfsck.
 generate_db() {
     local i
     local ostidx
     local dev
-    local tmp_file
-
-    tmp_file=$(mktemp -p $SHARED_DIRECTORY ||
-        error_exit "fail to create file in $SHARED_DIRECTORY")
 
-    # make sure everything gets to the backing store
-    local list=$(comma_list $CLIENTS $(facet_host mds) $(osts_nodes))
-    do_nodes $list "sync; sleep 2; sync"
+    check_shared_dir $SHARED_DIRECTORY ||
+        error "$SHARED_DIRECTORY isn't a shared directory"
 
-    do_nodes $list ls $tmp_file || \
-        error_exit "$SHARED_DIRECTORY is not a shared directory"
-    rm $tmp_file
+    export MDSDB=$SHARED_DIRECTORY/mdsdb
+    export OSTDB=$SHARED_DIRECTORY/ostdb
 
     run_e2fsck $(facet_host mds) $MDSDEV "--mdsdb $MDSDB"
 
@@ -3887,18 +3888,23 @@ check_logdir() {
         # Not found. Create local logdir
         mkdir -p $dir
     else
-        touch $dir/node.$(hostname).yml
+        touch $dir/check_file.$(hostname)
     fi
     return 0
 }
 
 check_write_access() {
     local dir=$1
+    local node
+    local file
+
     for node in $(nodes_list); do
-        if [ ! -f "$dir/node.${node}.yml" ]; then
+        file=$dir/check_file.$node
+        if [[ ! -f $file ]]; then
             # Logdir not accessible/writable from this node.
             return 1
         fi
+        rm -f $file || return 1
     done
     return 0
 }
@@ -3911,18 +3917,18 @@ init_logging() {
     mkdir -p $LOGDIR
     init_clients_lists
 
-    if [ ! -f $YAML_LOG ]; then       # If the yaml log already exists then we will just append to it
-      do_rpc_nodes $(comma_list $(nodes_list)) check_logdir $LOGDIR
-      if check_write_access $LOGDIR; then
-          touch $LOGDIR/shared
-          echo "Logging to shared log directory: $LOGDIR"
-      else
-          echo "Logging to local directory: $LOGDIR"
-      fi
+    # If the yaml log already exists then we will just append to it.
+    [[ -f $YAML_LOG ]] && return 0
 
-      yml_nodes_file $LOGDIR >> $YAML_LOG
-      yml_results_file >> $YAML_LOG
+    if check_shared_dir $LOGDIR; then
+        touch $LOGDIR/shared
+        echo "Logging to shared log directory: $LOGDIR"
+    else
+        echo "Logging to local directory: $LOGDIR"
     fi
+
+    yml_nodes_file $LOGDIR >> $YAML_LOG
+    yml_results_file >> $YAML_LOG
 }
 
 log_test() {