LU-12131 tests: fix SSK handling in tests

SSK can be activated for Lustre tests by setting SHARED_KEY env
variable to true.
In setup_all() an additional env variable SK_MOUNTED is used to avoid
mounting an SSK file system twice. But this variable has to be set
back to false in stopall() for consistency.
Some tests are incompatible with SSK, so skip them in case SHARED_KEY
is true. Some other tests playing with nodemaps have to take SSK into
account.

Lustre-change: https://review.whamcloud.com/34521
Lustre-commit: ee6904e312f7d7446f390cff0ec3c6e48b98e32b

Whamcloud-bug-id: ATM-1283
Test-Parameters: envdefinitions=SHARED_KEY=true testlist=sanity,recovery-small,sanity-sec
Signed-off-by: Sebastien Buisson <sbuisson@ddn.com>
Change-Id: I1016a459c42ffed1ab2b6f67d0a145ed2af9fa40
Reviewed-on: https://review.whamcloud.com/35556
Tested-by: jenkins <devops@whamcloud.com>
Reviewed-by: Andreas Dilger <adilger@whamcloud.com>
Tested-by: Maloo <maloo@whamcloud.com>

diff --git a/lustre/tests/sanity-selinux.sh b/lustre/tests/sanity-selinux.sh
index 4c1ffef..049d401 100755 (executable)
--- a/lustre/tests/sanity-selinux.sh
+++ b/lustre/tests/sanity-selinux.sh
@@ -456,7 +456,24 @@ test_21a() {
        fi
 
        # create nodemap entry with sepol
-       create_nodemap nm1
+       create_nodemap c0
+
+       if $GSS_SK; then
+               # update mount option with skpath
+               MOUNT_OPTS=$(add_sk_mntflag $MOUNT_OPTS)
+               export SK_UNIQUE_NM=true
+
+               # load specific key on servers
+               do_nodes $(comma_list $(all_server_nodes)) "lgss_sk -t server \
+                                                   -l $SK_PATH/nodemap/c0.key"
+
+               # set perms for per-nodemap keys else permission denied
+               do_nodes $(comma_list $(all_server_nodes)) \
+                "keyctl show | grep lustre | cut -c1-11 |
+                               sed -e 's/ //g;' |
+                               xargs -IX keyctl setperm X 0x3f3f3f3f"
+
+       fi
 
        # mount client without sending sepol
        mount_client $MOUNT $MOUNT_OPTS &&
@@ -472,16 +489,20 @@ test_21a() {
 
        # store wrong sepol in nodemap
        sepol="0:policy:0:0000000000000000000000000000000000000000000000000000000000000000"
-       do_facet mgs $LCTL set_param nodemap.nm1.sepol="$sepol"
-       do_facet mgs $LCTL set_param -P nodemap.nm1.sepol="$sepol"
-       check_nodemap nm1 sepol $sepol
+       do_facet mgs $LCTL set_param nodemap.c0.sepol="$sepol"
+       do_facet mgs $LCTL set_param -P nodemap.c0.sepol="$sepol"
+       check_nodemap c0 sepol $sepol
 
        # mount client with sepol
        mount_client $MOUNT $MOUNT_OPTS &&
                error "client mount without matching sepol should be refused"
 
        # remove nodemap
-       remove_nodemap nm1
+       remove_nodemap c0
+
+       if $GSS_SK; then
+               export SK_UNIQUE_NM=false
+       fi
 
        # remount client normally
        echo 0 > /sys/module/ptlrpc/parameters/send_sepol
@@ -523,7 +544,22 @@ test_21b() {
        echo 3 > /proc/sys/vm/drop_caches
 
        # create nodemap entry with sepol
-       create_nodemap nm1
+       create_nodemap c0
+
+       if $GSS_SK; then
+               export SK_UNIQUE_NM=true
+
+               # load specific key on servers
+               do_nodes $(comma_list $(all_server_nodes)) "lgss_sk -t server \
+                                                   -l $SK_PATH/nodemap/c0.key"
+
+               # set perms for per-nodemap keys else permission denied
+               do_nodes $(comma_list $(all_server_nodes)) \
+                "keyctl show | grep lustre | cut -c1-11 |
+                               sed -e 's/ //g;' |
+                               xargs -IX keyctl setperm X 0x3f3f3f3f"
+
+       fi
 
        # metadata ops without sending sepol
        touch $DIR/$tdir/f0 && error "touch (1)"
@@ -567,9 +603,9 @@ test_21b() {
 
        # store wrong sepol in nodemap
        sepol="0:policy:0:0000000000000000000000000000000000000000000000000000000000000000"
-       do_facet mgs $LCTL set_param nodemap.nm1.sepol="$sepol"
-       do_facet mgs $LCTL set_param -P nodemap.nm1.sepol="$sepol"
-       check_nodemap nm1 sepol $sepol
+       do_facet mgs $LCTL set_param nodemap.c0.sepol="$sepol"
+       do_facet mgs $LCTL set_param -P nodemap.c0.sepol="$sepol"
+       check_nodemap c0 sepol $sepol
 
        # metadata ops with sepol
        touch $DIR/$tdir/f4 && error "touch (3)"
@@ -592,9 +628,9 @@ test_21b() {
 
        # reset correct sepol
        sepol=$(l_getsepol | cut -d':' -f2- | xargs)
-       do_facet mgs $LCTL set_param nodemap.nm1.sepol="$sepol"
-       do_facet mgs $LCTL set_param -P nodemap.nm1.sepol="$sepol"
-       check_nodemap nm1 sepol $sepol
+       do_facet mgs $LCTL set_param nodemap.c0.sepol="$sepol"
+       do_facet mgs $LCTL set_param -P nodemap.c0.sepol="$sepol"
+       check_nodemap c0 sepol $sepol
 
        # metadata ops with sepol every 10 seconds only
        echo 10 > /sys/module/ptlrpc/parameters/send_sepol
@@ -674,8 +710,12 @@ test_21b() {
        fi
 
        # remove nodemap
-       remove_nodemap nm1
+       remove_nodemap c0
        echo 0 > /sys/module/ptlrpc/parameters/send_sepol
+
+       if $GSS_SK; then
+               export SK_UNIQUE_NM=false
+       fi
 }
 run_test 21b "Send sepol for metadata ops"
 

diff --git a/lustre/tests/sanity.sh b/lustre/tests/sanity.sh
index cb7a686..ce671aa 100755 (executable)
--- a/lustre/tests/sanity.sh
+++ b/lustre/tests/sanity.sh
@@ -20294,6 +20294,8 @@ run_test 811 "orphan name stub can be cleaned up in startup"
 test_812() {
        [ $OST1_VERSION -lt $(version_code 2.12.51) ] &&
                skip "OST < 2.12.51 doesn't support this fail_loc"
+       [ "$SHARED_KEY" = true ] &&
+               skip "OSC connections never go IDLE with Shared-Keys enabled"
 
        $LFS setstripe -c 1 -i 0 $DIR/$tfile
        # ensure ost1 is connected

diff --git a/lustre/tests/test-framework.sh b/lustre/tests/test-framework.sh
index 48ef0ec..7ee0ee7 100755 (executable)
--- a/lustre/tests/test-framework.sh
+++ b/lustre/tests/test-framework.sh
@@ -4140,40 +4140,44 @@ unmount_fstype() {
 ## MountConf setup
 
 stopall() {
-    # make sure we are using the primary server, so test-framework will
-    # be able to clean up properly.
-    activemds=`facet_active mds1`
-    if [ $activemds != "mds1" ]; then
-        fail mds1
-    fi
+       # make sure we are using the primary server, so test-framework will
+       # be able to clean up properly.
+       activemds=`facet_active mds1`
+       if [ $activemds != "mds1" ]; then
+               fail mds1
+       fi
 
-    local clients=$CLIENTS
-    [ -z $clients ] && clients=$(hostname)
+       local clients=$CLIENTS
+       [ -z $clients ] && clients=$(hostname)
 
-    zconf_umount_clients $clients $MOUNT "$*" || true
-    [ -n "$MOUNT2" ] && zconf_umount_clients $clients $MOUNT2 "$*" || true
+       zconf_umount_clients $clients $MOUNT "$*" || true
+       [ -n "$MOUNT2" ] && zconf_umount_clients $clients $MOUNT2 "$*" || true
 
-    [ -n "$CLIENTONLY" ] && return
+       [ -n "$CLIENTONLY" ] && return
 
-    # The add fn does rm ${facet}active file, this would be enough
-    # if we use do_facet <facet> only after the facet added, but
-    # currently we use do_facet mds in local.sh
-    for num in `seq $MDSCOUNT`; do
-        stop mds$num -f
-        rm -f ${TMP}/mds${num}active
-    done
-    combined_mgs_mds && rm -f $TMP/mgsactive
+       # The add fn does rm ${facet}active file, this would be enough
+       # if we use do_facet <facet> only after the facet added, but
+       # currently we use do_facet mds in local.sh
+       for num in `seq $MDSCOUNT`; do
+               stop mds$num -f
+               rm -f ${TMP}/mds${num}active
+       done
+       combined_mgs_mds && rm -f $TMP/mgsactive
 
-    for num in `seq $OSTCOUNT`; do
-        stop ost$num -f
-        rm -f $TMP/ost${num}active
-    done
+       for num in `seq $OSTCOUNT`; do
+               stop ost$num -f
+               rm -f $TMP/ost${num}active
+       done
 
-    if ! combined_mgs_mds ; then
-        stop mgs
-    fi
+       if ! combined_mgs_mds ; then
+               stop mgs
+       fi
 
-    return 0
+       if $SHARED_KEY; then
+               export SK_MOUNTED=false
+       fi
+
+       return 0
 }
 
 cleanup_echo_devs () {