LU-7227 tests: check dmesg for errors after test

[fs/lustre-release.git] / lustre / tests / test-framework.sh

diff --git a/lustre/tests/test-framework.sh b/lustre/tests/test-framework.sh
index 57d4f24..eaa4878 100755 (executable)
--- a/lustre/tests/test-framework.sh
+++ b/lustre/tests/test-framework.sh
@@ -1116,7 +1116,7 @@ get_osd_param() {
        local name=$3
 
        do_nodes $nodes "$LCTL get_param -n obdfilter.$device.$name \
-               osd-*.$device.$name 2>&1" | grep -v 'Found no match'
+               osd-*.$device.$name 2>&1" | grep -v 'error:'
 }
 
 set_osd_param() {
@@ -1126,7 +1126,7 @@ set_osd_param() {
        local value=$4
 
        do_nodes $nodes "$LCTL set_param -n obdfilter.$device.$name=$value \
-               osd-*.$device.$name=$value 2>&1" | grep -v 'Found no match'
+               osd-*.$device.$name=$value 2>&1" | grep -v 'error:'
 }
 
 set_debug_size () {
@@ -4687,6 +4687,9 @@ error_noexit() {
                        echo "$@" > $LOGDIR/err
                fi
        fi
+
+       # cleanup the env for failed tests
+       reset_fail_loc
 }
 
 exit_status () {
@@ -4929,6 +4932,18 @@ banner() {
     log "$msg== $(date +"%H:%M:%S (%s)")"
 }
 
+check_dmesg_for_errors() {
+       local res
+       local errors="VFS: Busy inodes after unmount of\|\
+ldiskfs_check_descriptors: Checksum for group 0 failed\|\
+group descriptors corrupted"
+
+       res=$(do_nodes $(comma_list $(nodes_list)) "dmesg" | grep "$errors")
+       [ -z "$res" ] && return 0
+       echo "Kernel error detected: $res"
+       return 1
+}
+
 #
 # Run a single test function and cleanup after it.
 #
@@ -4950,6 +4965,7 @@ run_one() {
        reset_fail_loc
        check_grant ${testnum} || error "check_grant $testnum failed with $?"
        check_catastrophe || error "LBUG/LASSERT detected"
+       check_dmesg_for_errors || error "Error in dmesg detected"
        if [ "$PARALLEL" != "yes" ]; then
                ps auxww | grep -v grep | grep -q multiop &&
                                        error "multiop still running"
@@ -6070,19 +6086,21 @@ create_pool() {
 }
 
 add_pool_to_list () {
-    local fsname=${1%%.*}
-    local poolname=${1##$fsname.}
+       local fsname=${1%%.*}
+       local poolname=${1##$fsname.}
 
-    local listvar=${fsname}_CREATED_POOLS
-    eval export ${listvar}=$(expand_list ${!listvar} $poolname)
+       local listvar=${fsname}_CREATED_POOLS
+       local temp=${listvar}=$(expand_list ${!listvar} $poolname)
+       eval export $temp
 }
 
 remove_pool_from_list () {
-    local fsname=${1%%.*}
-    local poolname=${1##$fsname.}
+       local fsname=${1%%.*}
+       local poolname=${1##$fsname.}
 
-    local listvar=${fsname}_CREATED_POOLS
-    eval export ${listvar}=$(exclude_items_from_list ${!listvar} $poolname)
+       local listvar=${fsname}_CREATED_POOLS
+       local temp=${listvar}=$(exclude_items_from_list ${!listvar} $poolname)
+       eval export $temp
 }
 
 destroy_pool_int() {
@@ -6219,6 +6237,31 @@ max_recovery_time() {
        echo -n $service_time
 }
 
+recovery_time_min() {
+       local connection_switch_min=5
+       local connection_switch_inc=5
+       local connection_switch_max
+       local reconnect_delay_max
+       local initial_connect_timeout
+       local max
+       local timout_20
+
+       #connection_switch_max=min(50, max($connection_switch_min,$TIMEOUT)
+       (($connection_switch_min > $TIMEOUT)) &&
+               max=$connection_switch_min || max=$TIMEOUT
+       (($max < 50)) && connection_switch_max=$max || connection_switch_max=50
+
+       #initial_connect_timeout = max(connection_switch_min, obd_timeout/20)
+       timeout_20=$((TIMEOUT/20))
+       (($connection_switch_min > $timeout_20)) &&
+               initial_connect_timeout=$connection_switch_min ||
+               initial_connect_timeout=$timeout_20
+
+       reconnect_delay_max=$((connection_switch_max + connection_switch_inc + \
+                              initial_connect_timeout))
+       echo $((2 * reconnect_delay_max))
+}
+
 get_clients_mount_count () {
     local clients=${CLIENTS:-`hostname`}