LU-9795 tests: exclude several tests which conflict with SSK

When SSK is activated by setting SHARED_KEY to true,
some tests in various suites fail, often because components
are manually halted or failed.
This patch excludes these tests under SSK and makes minor changes
for SSK compatibility.

Also reconnect client-to-OST connections if they became idle.
This new idle ability is brought by patch
https://review.whamcloud.com/16682 and it prevents idle OSCs from
taking into account new GSS flavor.

Change-Id: I998ae9bf1998f206914ff425e1f6e27741443e9c
Test-Parameters: testlist=sanity-gss envdefinitions=ONLY=1,SHARED_KEY=true
Signed-off-by: Chris Hanna <hannac@iu.edu>
Signed-off-by: Sebastien Buisson <sbuisson@ddn.com>
Reviewed-on: https://review.whamcloud.com/28662
Reviewed-by: Andreas Dilger <adilger@whamcloud.com>
Reviewed-by: James Simmons <uja.ornl@yahoo.com>
Tested-by: Jenkins
Tested-by: Maloo <hpdd-maloo@intel.com>
Reviewed-by: Oleg Drokin <green@whamcloud.com>

diff --git a/lustre/tests/conf-sanity.sh b/lustre/tests/conf-sanity.sh
index 394fe54..90c5cd1 100644 (file)
--- a/lustre/tests/conf-sanity.sh
+++ b/lustre/tests/conf-sanity.sh
@@ -11,6 +11,12 @@ ALWAYS_EXCEPT="$CONF_SANITY_EXCEPT"
 # bug number for skipped test:
 # a tool to create lustre filesystem images
 ALWAYS_EXCEPT="32newtarball $ALWAYS_EXCEPT"
+if $SHARED_KEY; then
+# bug number for skipped tests:                LU-9795 (all below)
+       ALWAYS_EXCEPT="$ALWAYS_EXCEPT   0       31      32a     32d     35a"
+       ALWAYS_EXCEPT="$ALWAYS_EXCEPT   53a     53b     54b     76a     76b"
+       ALWAYS_EXCEPT="$ALWAYS_EXCEPT   76c     76d     78      103"
+fi
 
 SRCDIR=$(dirname $0)
 PATH=$PWD/$SRCDIR:$SRCDIR:$SRCDIR/../utils:$PATH

diff --git a/lustre/tests/replay-dual.sh b/lustre/tests/replay-dual.sh
index 45cd5de..5e75e4f 100755 (executable)
--- a/lustre/tests/replay-dual.sh
+++ b/lustre/tests/replay-dual.sh
@@ -27,6 +27,10 @@ remote_mds_nodsh && skip "remote MDS with nodsh" && exit 0
 [[ $(facet_fstype $SINGLEMDS) == zfs ]] &&
 # bug number for skipped test:       LU-2230
        ALWAYS_EXCEPT="$ALWAYS_EXCEPT 21b"
+if $SHARED_KEY; then
+# bug number for skipped tests:        LU-9795 LU-9795
+       ALWAYS_EXCEPT="         0a      0b      $ALWAYS_EXCEPT"
+fi
 
 build_test_filter
 

diff --git a/lustre/tests/replay-single.sh b/lustre/tests/replay-single.sh
index 32838a9..b86f862 100755 (executable)
--- a/lustre/tests/replay-single.sh
+++ b/lustre/tests/replay-single.sh
@@ -36,6 +36,13 @@ if [ $(facet_fstype $SINGLEMDS) = "zfs" ]; then
                ALWAYS_EXCEPT+=" 2d       70d      80c     80d"
        fi
 fi
+if $SHARED_KEY; then
+# bug number for skipped tests: LU-9795 (all below)
+       ALWAYS_EXCEPT="$ALWAYS_EXCEPT   0b      0c      0d      34      45"
+       ALWAYS_EXCEPT="$ALWAYS_EXCEPT   47      58b     58c     71a     85a"
+       ALWAYS_EXCEPT="$ALWAYS_EXCEPT   85b     86      88      89      90"
+       ALWAYS_EXCEPT="$ALWAYS_EXCEPT   93a     100a    100b    120"
+fi
 
 build_test_filter
 

diff --git a/lustre/tests/sanity-gss.sh b/lustre/tests/sanity-gss.sh
index e10687a..457c835 100755 (executable)
--- a/lustre/tests/sanity-gss.sh
+++ b/lustre/tests/sanity-gss.sh
@@ -10,6 +10,10 @@ ONLY=${ONLY:-"$*"}
 # bug number for skipped test:
 ALWAYS_EXCEPT=${ALWAYS_EXCEPT:-"$SANITY_GSS_EXCEPT"}
 # UPDATE THE COMMENT ABOVE WITH BUG NUMBERS WHEN CHANGING ALWAYS_EXCEPT!
+if $SHARED_KEY; then
+# bug number for skipped tests:        LU-9795 LU-9795
+       ALWAYS_EXCEPT="         8       90      $ALWAYS_EXCEPT"
+fi
 
 SRCDIR=`dirname $0`
 
@@ -102,6 +106,9 @@ stop_dbench()
 calc_connection_cnt
 umask 077
 
+# Stop previously existing gss daemons
+stop_gss_daemons
+
 echo "bring up gss daemons..."
 # start gss daemon with -z flag for gssnull
 start_gss_daemons $(comma_list $(mdts_nodes)) "$LSVCGSSD -z -vv" ||
@@ -588,6 +595,11 @@ test_151() {
 run_test 151 "secure mgs connection: server flavor control"
 
 stop_gss_daemons
+if $GSS_KRB5 || $GSS_SK; then
+       start_gss_daemons
+fi
+
+restore_to_default_flavor
 
 complete $SECONDS
 check_and_cleanup_lustre

diff --git a/lustre/tests/sanity-hsm.sh b/lustre/tests/sanity-hsm.sh
index 904f2df..e355a12 100755 (executable)
--- a/lustre/tests/sanity-hsm.sh
+++ b/lustre/tests/sanity-hsm.sh
@@ -14,6 +14,10 @@ ONLY=${ONLY:-"$*"}
 # bug number for skipped test:
 ALWAYS_EXCEPT="$SANITY_HSM_EXCEPT"
 # UPDATE THE COMMENT ABOVE WITH BUG NUMBERS WHEN CHANGING ALWAYS_EXCEPT!
+if $SHARED_KEY; then
+# bug number for skipped tests:        LU-9795 LU-9795
+       ALWAYS_EXCEPT="         13      402b    $ALWAYS_EXCEPT"
+fi
 
 LUSTRE=${LUSTRE:-$(cd $(dirname $0)/..; echo $PWD)}
 

diff --git a/lustre/tests/sanity-sec.sh b/lustre/tests/sanity-sec.sh
index 9018c3f..57311e2 100755 (executable)
--- a/lustre/tests/sanity-sec.sh
+++ b/lustre/tests/sanity-sec.sh
@@ -9,10 +9,6 @@ set -e
 ONLY=${ONLY:-"$*"}
 # bug number for skipped test:
 ALWAYS_EXCEPT="              $SANITY_SEC_EXCEPT"
-if $SHARED_KEY; then
-# bug number for skipped test: 9145 9145 9671 9145 9145 9145 9145 9245
-       ALWAYS_EXCEPT="        17   18   19   20   21   22   23   27 $ALWAYS_EXCEPT"
-fi
 # UPDATE THE COMMENT ABOVE WITH BUG NUMBERS WHEN CHANGING ALWAYS_EXCEPT!
 
 SRCDIR=$(dirname $0)
@@ -1751,10 +1747,11 @@ test_23a() {
 run_test 23a "test mapped regular ACLs"
 
 test_23b() { #LU-9929
-       remote_mgs_nodsh && skip "remote MGS with nodsh" && return
+       [ $num_clients -lt 2 ] && skip "Need 2 clients at least" && return
        [ $(lustre_version_code mgs) -lt $(version_code 2.10.53) ] &&
                skip "Need MGS >= 2.10.53" && return
 
+       export SK_UNIQUE_NM=true
        nodemap_test_setup
        trap nodemap_test_cleanup EXIT
 
@@ -1766,11 +1763,16 @@ test_23b() { #LU-9929
 
        do_facet mgs $LCTL nodemap_modify --name c0 --property admin --value 1
        wait_nm_sync c0 admin_nodemap
+       do_facet mgs $LCTL nodemap_modify --name c1 --property admin --value 1
+       wait_nm_sync c1 admin_nodemap
+       do_facet mgs $LCTL nodemap_modify --name c1 --property trusted --value 1
+       wait_nm_sync c1 trusted_nodemap
 
        # Add idmap $ID0:$fs_id (500:60010)
        do_facet mgs $LCTL nodemap_add_idmap --name c0 --idtype gid \
                --idmap $ID0:$fs_id ||
                error "add idmap $ID0:$fs_id to nodemap c0 failed"
+       wait_nm_sync c0 idmap
 
        # set/getfacl default acl on client0 (unmapped gid=500)
        rm -rf $testdir
@@ -1783,20 +1785,18 @@ test_23b() { #LU-9929
        [ "$unmapped_id" = "$USER0" ] ||
                error "gid=$ID0 was not unmapped correctly on ${clients_arr[0]}"
 
-       # getfacl default acl on MGS (mapped gid=60010)
-       zconf_mount $mgs_HOST $MOUNT
-       do_rpc_nodes $mgs_HOST is_mounted $MOUNT ||
-               error "mount lustre on MGS failed"
-       mapped_id=$(do_node $mgs_HOST getfacl $testdir |
+       # getfacl default acl on client2 (mapped gid=60010)
+       mapped_id=$(do_node ${clients_arr[1]} getfacl $testdir |
                        grep -E "default:group:.*:rwx" | awk -F: '{print $3}')
-       fs_user=$(do_facet mgs getent passwd |
+       fs_user=$(do_node ${clients_arr[1]} getent passwd |
                        grep :$fs_id:$fs_id: | cut -d: -f1)
+       [ -z "$fs_user" ] && fs_user=$fs_id
        [ $mapped_id -eq $fs_id -o "$mapped_id" = "$fs_user" ] ||
-               error "Should return gid=$fs_id or $fs_user on MGS"
+               error "Should return gid=$fs_id or $fs_user on client2"
 
        rm -rf $testdir
-       do_facet mgs umount $MOUNT
        nodemap_test_cleanup
+       export SK_UNIQUE_NM=false
 }
 run_test 23b "test mapped default ACLs"
 
@@ -1997,6 +1997,11 @@ test_27a() {
                local subdir="subdir_${nm}"
                local subsubdir="subsubdir_${nm}"
 
+               if [ "$nm" == "default" ] && [ "$SHARED_KEY" == "true" ]; then
+                       echo "Skipping nodemap $nm with SHARED_KEY";
+                       continue;
+               fi
+
                echo "Exercising fileset for nodemap $nm"
                nodemap_exercise_fileset "$nm"
        done

diff --git a/lustre/tests/sanity.sh b/lustre/tests/sanity.sh
index c77114f..45920b4 100755 (executable)
--- a/lustre/tests/sanity.sh
+++ b/lustre/tests/sanity.sh
@@ -15,9 +15,13 @@ ALWAYS_EXCEPT="$SANITY_EXCEPT  42a     42b     42c     77k"
 # skipped tests: LU-8411 LU-9096 LU-9054 ..
 ALWAYS_EXCEPT="  407     253     312     $ALWAYS_EXCEPT"
 
+if $SHARED_KEY; then
+# bug number for skipped tests:        LU-9795 (all below)
+       ALWAYS_EXCEPT="$ALWAYS_EXCEPT   17n     60a     133g    300f"
+fi
+
 # Check Grants after these tests
 GRANT_CHECK_LIST="$GRANT_CHECK_LIST 42a 42b 42c 42d 42e 63a 63b 64a 64b 64c"
-
 SRCDIR=$(cd $(dirname $0); echo $PWD)
 export PATH=$PATH:/sbin
 

diff --git a/lustre/tests/test-framework.sh b/lustre/tests/test-framework.sh
index 7b0fe56..381c091 100755 (executable)
--- a/lustre/tests/test-framework.sh
+++ b/lustre/tests/test-framework.sh
@@ -22,6 +22,7 @@ export SK_UNIQUE_NM=${SK_UNIQUE_NM:-false}
 export SK_S2S=${SK_S2S:-false}
 export SK_S2SNM=${SK_S2SNM:-TestFrameNM}
 export SK_S2SNMCLI=${SK_S2SNMCLI:-TestFrameNMCli}
+export SK_SKIPFIRST=${SK_SKIPFIRST:-true}
 export IDENTITY_UPCALL=default
 export QUOTA_AUTO=1
 export FLAKEY=${FLAKEY:-true}
@@ -943,7 +944,11 @@ init_gss() {
                # security ctx config for keyring
                SK_NO_KEY=false
                mkdir -p $SK_OM_PATH
-               mount -o bind $SK_OM_PATH /etc/request-key.d/
+               if grep -q request-key /proc/mounts > /dev/null; then
+                       echo "SSK: Request key already mounted."
+               else
+                       mount -o bind $SK_OM_PATH /etc/request-key.d/
+               fi
                local lgssc_conf_line='create lgssc * * '
                lgssc_conf_line+=$(which lgss_keyring)
                lgssc_conf_line+=' %o %k %t %d %c %u %g %T %P %S'
@@ -1054,9 +1059,12 @@ cleanup_sk() {
                $RPC_MODE || echo "Cleaning up Shared Key.."
                do_nodes $(comma_list $(all_nodes)) "rm -f \
                        $SK_PATH/$FSNAME*.key $SK_PATH/nodemap/$FSNAME*.key"
+               do_nodes $(comma_list $(all_nodes)) "keyctl show | \
+                 awk '/lustre/ { print \\\$1 }' | xargs -IX keyctl unlink X"
                # Remove the mount and clean up the files we added to SK_PATH
-               do_nodes $(comma_list $(all_nodes)) "umount \
-                       /etc/request-key.d/"
+               do_nodes $(comma_list $(all_nodes)) "while grep -q \
+                       request-key.d /proc/mounts; do umount \
+                       /etc/request-key.d/; done"
                do_nodes $(comma_list $(all_nodes)) "rm -f \
                        $SK_OM_PATH/lgssc.conf"
                do_nodes $(comma_list $(all_nodes)) "rmdir $SK_OM_PATH"
@@ -4631,8 +4639,17 @@ setupall() {
                if $GSS_SK; then
                        set_rule $FSNAME any cli2mdt $SK_FLAVOR
                        set_rule $FSNAME any cli2ost $SK_FLAVOR
-                       wait_flavor cli2mdt $SK_FLAVOR
-                       wait_flavor cli2ost $SK_FLAVOR
+                       if $SK_SKIPFIRST; then
+                               export SK_SKIPFIRST=false
+
+                               sleep 30
+                               do_nodes $CLIENTS \
+                                        "lctl set_param osc.*.idle_connect=1"
+                               return
+                       else
+                               wait_flavor cli2mdt $SK_FLAVOR
+                               wait_flavor cli2ost $SK_FLAVOR
+                       fi
                else
                        set_flavor_all $SEC
                fi
@@ -5043,7 +5060,6 @@ check_and_setup_lustre() {
                fi
        fi
 
-       init_gss
        if $GSS_SK; then
                set_flavor_all null
        elif $GSS; then
@@ -7609,6 +7625,8 @@ flvr_cnt_cli2ost()
     local clients=${CLIENTS:-$HOSTNAME}
 
     for c in ${clients//,/ }; do
+       # reconnect if idle
+       do_node $c lctl set_param osc.*.idle_connect=1 >/dev/null 2>&1
        local output=$(do_node $c lctl get_param -n \
                 osc.*OST*-osc-[^M][^D][^T]*.$PROC_CLI 2>/dev/null)
        local tmpcnt=$(count_flvr "$output" $flavor)