From: Chris Hanna <hannac@iu.edu>
Date: Wed, 19 Jul 2017 14:25:10 +0000 (-0400)
Subject: LU-8275 tests: add flag to enable secret shared key for tests
X-Git-Tag: 2.10.52~46
X-Git-Url: https://git.whamcloud.com/?p=fs%2Flustre-release.git;a=commitdiff_plain;h=62ed4f22e21075daa074f2c7f92be6509d76e51c;hp=c45d7ba4c023e76d445b97792a64c3594cec20a0

LU-8275 tests: add flag to enable secret shared key for tests

When the SHARED_KEY environment variable is set to true,
test-framework will set up a shared key between the nodes and start
Lustre with shared key enabled. Three tests (28,29,30) are also
added to sanity-sec in order to test shared key features.

Signed-off-by: Kit Westneat <kit.westneat@gmail.com>
Signed-off-by: Chris Hanna <hannac@iu.edu>
Change-Id: I2abff35fbb7dd4f227c6cd12a66cb0b77f610157
Reviewed-on: https://review.whamcloud.com/20780
Tested-by: Jenkins
Tested-by: Maloo <hpdd-maloo@intel.com>
Reviewed-by: James Nunez <james.a.nunez@intel.com>
Reviewed-by: John L. Hammond <john.hammond@intel.com>
Reviewed-by: Oleg Drokin <oleg.drokin@intel.com>
---

diff --git a/lustre/tests/sanity-sec.sh b/lustre/tests/sanity-sec.sh
index 5df8e27..eec2f88 100755
--- a/lustre/tests/sanity-sec.sh
+++ b/lustre/tests/sanity-sec.sh
@@ -9,6 +9,10 @@ set -e
 ONLY=${ONLY:-"$*"}
 # bug number for skipped test: 19430 19967 19967
 ALWAYS_EXCEPT="                2     5     6    $SANITY_SEC_EXCEPT"
+if $SHARED_KEY; then
+# bug number for skipped test: 9145 9145 9671 9145 9145 9145 9145 9245
+	ALWAYS_EXCEPT="        17   18   19   20   21   22   23   27 $ALWAYS_EXCEPT"
+fi
 # UPDATE THE COMMENT ABOVE WITH BUG NUMBERS WHEN CHANGING ALWAYS_EXCEPT!
 
 SRCDIR=$(dirname $0)
@@ -114,6 +118,7 @@ sec_login() {
 	local user=$1
 	local group=$2
 
+	$GSS_KRB5 || return
 	if ! $RUNAS_CMD -u $user krb5_login.sh; then
 		error "$user login kerberos failed."
 		exit 1
@@ -1342,6 +1347,7 @@ nodemap_test_cleanup() {
 	do_facet mgs $LCTL nodemap_activate 0
 	wait_nm_sync active 0
 
+	export SK_UNIQUE_NM=false
 	return 0
 }
 
@@ -1588,6 +1594,8 @@ run_test 24 "check nodemap proc files for LBUGs and Oopses"
 test_25() {
 	local tmpfile=$(mktemp)
 	local tmpfile2=$(mktemp)
+	local tmpfile3=$(mktemp)
+	local tmpfile4=$(mktemp)
 	local subdir=c0dir
 	local client
 
@@ -1597,6 +1605,7 @@ test_25() {
 	zconf_umount_clients $CLIENTS $MOUNT ||
 	    error "unable to umount clients $CLIENTS"
 
+	export SK_UNIQUE_NM=true
 	nodemap_test_setup
 
 	# enable trusted/admin for setquota call in cleanup_and_setup_lustre()
@@ -1613,36 +1622,42 @@ test_25() {
 	trap nodemap_test_cleanup EXIT
 
 	# create a new, empty nodemap, and add fileset info to it
-	do_facet mgs $LCTL nodemap_add test26 ||
-		error "unable to create nodemap test26"
-	do_facet mgs $LCTL set_param -P nodemap.test26.fileset=/$subdir ||
-		error "unable to add fileset info to nodemap test26"
+	do_facet mgs $LCTL nodemap_add test25 ||
+		error "unable to create nodemap $testname"
+	do_facet mgs $LCTL set_param -P nodemap.$testname.fileset=/$subdir ||
+		error "unable to add fileset info to nodemap test25"
 
-	wait_nm_sync test26 id
+	wait_nm_sync test25 id
 
 	do_facet mgs $LCTL nodemap_info > $tmpfile
 	do_facet mds $LCTL nodemap_info > $tmpfile2
 
-	cleanup_and_setup_lustre
+	if ! $SHARED_KEY; then
+		# will conflict with SK's nodemaps
+		cleanup_and_setup_lustre
+	fi
 	# stop clients for this test
 	zconf_umount_clients $CLIENTS $MOUNT ||
 	    error "unable to umount clients $CLIENTS"
 
-	diff -q <(do_facet mgs $LCTL nodemap_info) $tmpfile >& /dev/null ||
+	do_facet mgs $LCTL nodemap_info > $tmpfile3
+	diff -q $tmpfile3 $tmpfile >& /dev/null ||
 		error "nodemap_info diff on MGS after remount"
 
-	diff -q <(do_facet mds $LCTL nodemap_info) $tmpfile2 >& /dev/null ||
+	do_facet mds $LCTL nodemap_info > $tmpfile4
+	diff -q $tmpfile4 $tmpfile2 >& /dev/null ||
 		error "nodemap_info diff on MDS after remount"
 
 	# cleanup nodemap
-	do_facet mgs $LCTL nodemap_del test26 ||
-	    error "cannot delete nodemap test26 from config"
+	do_facet mgs $LCTL nodemap_del test25 ||
+	    error "cannot delete nodemap test25 from config"
 	nodemap_test_cleanup
 	# restart clients previously stopped
 	zconf_mount_clients $CLIENTS $MOUNT ||
 	    error "unable to mount clients $CLIENTS"
 
 	rm -f $tmpfile $tmpfile2
+	export SK_UNIQUE_NM=false
 }
 run_test 25 "test save and reload nodemap config"
 
@@ -1666,7 +1681,12 @@ test_27() {
 	local loop=0
 
 	nodemap_test_setup
-	trap nodemap_test_cleanup EXIT
+	if $SHARED_KEY; then
+		export SK_UNIQUE_NM=true
+	else
+		# will conflict with SK's nodemaps
+		trap nodemap_test_cleanup EXIT
+	fi
 
 	fileset_test_setup
 
@@ -1678,8 +1698,11 @@ test_27() {
 	# re-mount client
 	zconf_umount_clients ${clients_arr[0]} $MOUNT ||
 		error "unable to umount client ${clients_arr[0]}"
+	# set some generic fileset to trigger SSK code
+	export FILESET=/
 	zconf_mount_clients ${clients_arr[0]} $MOUNT $MOUNT_OPTS ||
 		error "unable to remount client ${clients_arr[0]}"
+	unset FILESET
 
 	# test mount point content
 	do_node ${clients_arr[0]} test -f $MOUNT/this_is_$subdir ||
@@ -1726,11 +1749,115 @@ test_27() {
 	do_node ${clients_arr[0]} test -d $MOUNT/$subdir ||
 		(ls $MOUNT ; error "fileset not cleared on nodemap c0")
 
+	# back to non-nodemap setup
+	if $SHARED_KEY; then
+		export SK_UNIQUE_NM=false
+		zconf_umount_clients ${clients_arr[0]} $MOUNT ||
+			error "unable to umount client ${clients_arr[0]}"
+	fi
 	fileset_test_cleanup
 	nodemap_test_cleanup
+	if $SHARED_KEY; then
+		zconf_mount_clients ${clients_arr[0]} $MOUNT $MOUNT_OPTS ||
+			error "unable to remount client ${clients_arr[0]}"
+	fi
 }
 run_test 27 "test fileset in nodemap"
 
+test_28() {
+	if ! $SHARED_KEY; then
+		skip "need shared key feature for this test" && return
+	fi
+	mkdir -p $DIR/$tdir || error "mkdir failed"
+	touch $DIR/$tdir/$tdir.out || error "touch failed"
+	if [ ! -f $DIR/$tdir/$tdir.out ]; then
+		error "read before rotation failed"
+	fi
+	# store top key identity to ensure rotation has occurred
+	SK_IDENTITY_OLD=$(lctl get_param *.*.*srpc* | grep "expire" |
+		head -1 | awk '{print $15}' | cut -c1-8)
+	do_facet $SINGLEMDS lfs flushctx ||
+		 error "could not run flushctx on $SINGLEMDS"
+	sleep 5
+	lfs flushctx || error "could not run flushctx on client"
+	sleep 5
+	# verify new key is in place
+	SK_IDENTITY_NEW=$(lctl get_param *.*.*srpc* | grep "expire" |
+		head -1 | awk '{print $15}' | cut -c1-8)
+	if [ $SK_IDENTITY_OLD == $SK_IDENTITY_NEW ]; then
+		error "key did not rotate correctly"
+	fi
+	if [ ! -f $DIR/$tdir/$tdir.out ]; then
+		error "read after rotation failed"
+	fi
+}
+run_test 28 "check shared key rotation method"
+
+test_29() {
+	if ! $SHARED_KEY; then
+		skip "need shared key feature for this test" && return
+	fi
+	if [ $SK_FLAVOR != "ski" ] && [ $SK_FLAVOR != "skpi" ]; then
+		skip "test only valid if integrity is active"
+	fi
+	rm -r $DIR/$tdir
+	mkdir $DIR/$tdir || error "mkdir"
+	touch $DIR/$tdir/$tfile || error "touch"
+	zconf_umount_clients ${clients_arr[0]} $MOUNT ||
+		error "unable to umount clients"
+	keyctl show | awk '/lustre/ { print $1 }' |
+		xargs -IX keyctl unlink X
+	OLD_SK_PATH=$SK_PATH
+	export SK_PATH=/dev/null
+	if zconf_mount_clients ${clients_arr[0]} $MOUNT; then
+		export SK_PATH=$OLD_SK_PATH
+		if [ -e $DIR/$tdir/$tfile ]; then
+			error "able to mount and read without key"
+		else
+			error "able to mount without key"
+		fi
+	else
+		export SK_PATH=$OLD_SK_PATH
+		keyctl show | awk '/lustre/ { print $1 }' |
+			xargs -IX keyctl unlink X
+	fi
+}
+run_test 29 "check for missing shared key"
+
+test_30() {
+	if ! $SHARED_KEY; then
+		skip "need shared key feature for this test" && return
+	fi
+	if [ $SK_FLAVOR != "ski" ] && [ $SK_FLAVOR != "skpi" ]; then
+		skip "test only valid if integrity is active"
+	fi
+	mkdir -p $DIR/$tdir || error "mkdir failed"
+	touch $DIR/$tdir/$tdir.out || error "touch failed"
+	zconf_umount_clients ${clients_arr[0]} $MOUNT ||
+		error "unable to umount clients"
+	# unload keys from ring
+	keyctl show | awk '/lustre/ { print $1 }' |
+		xargs -IX keyctl unlink X
+	# invalidate the key with bogus filesystem name
+	lgss_sk -w $SK_PATH/$FSNAME-bogus.key -f $FSNAME.bogus \
+		-t client -d /dev/urandom || error "lgss_sk failed (1)"
+	do_facet $SINGLEMDS lfs flushctx || error "could not run flushctx"
+	OLD_SK_PATH=$SK_PATH
+	export SK_PATH=$SK_PATH/$FSNAME-bogus.key
+	if zconf_mount_clients ${clients_arr[0]} $MOUNT; then
+		SK_PATH=$OLD_SK_PATH
+		if [ -a $DIR/$tdir/$tdir.out ]; then
+			error "mount and read file with invalid key"
+		else
+			error "mount with invalid key"
+		fi
+	fi
+	SK_PATH=$OLD_SK_PATH
+	zconf_umount_clients ${clients_arr[0]} $MOUNT ||
+		error "unable to umount clients"
+}
+run_test 30 "check for invalid shared key"
+
 log "cleanup: ======================================================"
 
 sec_unsetup() {
diff --git a/lustre/tests/test-framework.sh b/lustre/tests/test-framework.sh
index 7776af0..e94f941 100755
--- a/lustre/tests/test-framework.sh
+++ b/lustre/tests/test-framework.sh
@@ -3,15 +3,25 @@
 trap 'print_summary && print_stack_trace | tee $TF_FAIL && \
     echo "$TESTSUITE: FAIL: test-framework exiting on error"' ERR
 set -e
-#set -x
 
 export LANG=en_US
 export REFORMAT=${REFORMAT:-""}
 export WRITECONF=${WRITECONF:-""}
 export VERBOSE=${VERBOSE:-false}
-export GSS=false
+export GSS=${GSS:-false}
+export GSS_SK=${GSS_SK:-false}
 export GSS_KRB5=false
 export GSS_PIPEFS=false
+export SHARED_KEY=${SHARED_KEY:-false}
+export SK_PATH=${SK_PATH:-/tmp/test-framework-keys}
+export SK_OM_PATH=$SK_PATH'/tmp-request-mount'
+export SK_MOUNTED=${SK_MOUNTED:-false}
+export SK_FLAVOR=${SK_FLAVOR:-ski}
+export SK_NO_KEY=${SK_NO_KEY:-true}
+export SK_UNIQUE_NM=${SK_UNIQUE_NM:-false}
+export SK_S2S=${SK_S2S:-false}
+export SK_S2SNM=${SK_S2SNM:-TestFrameNM}
+export SK_S2SNMCLI=${SK_S2SNMCLI:-TestFrameNMCli}
 export IDENTITY_UPCALL=default
 export QUOTA_AUTO=1
 # specify environment variable containing batch job name for server statistics
@@ -145,59 +155,59 @@ init_test_env() {
 	export DO_CLEANUP=${DO_CLEANUP:-true}
 	export KEEP_ZPOOL=${KEEP_ZPOOL:-false}
 
-    export MKE2FS=$MKE2FS
-    if [ -z "$MKE2FS" ]; then
-        if which mkfs.ldiskfs >/dev/null 2>&1; then
-            export MKE2FS=mkfs.ldiskfs
-        else
-            export MKE2FS=mke2fs
-        fi
-    fi
+	export MKE2FS=$MKE2FS
+	if [ -z "$MKE2FS" ]; then
+		if which mkfs.ldiskfs >/dev/null 2>&1; then
+			export MKE2FS=mkfs.ldiskfs
+		else
+			export MKE2FS=mke2fs
+		fi
+	fi
 
-    export DEBUGFS=$DEBUGFS
-    if [ -z "$DEBUGFS" ]; then
-        if which debugfs.ldiskfs >/dev/null 2>&1; then
-            export DEBUGFS=debugfs.ldiskfs
-        else
-            export DEBUGFS=debugfs
-        fi
-    fi
+	export DEBUGFS=$DEBUGFS
+	if [ -z "$DEBUGFS" ]; then
+		if which debugfs.ldiskfs >/dev/null 2>&1; then
+			export DEBUGFS=debugfs.ldiskfs
+		else
+			export DEBUGFS=debugfs
+		fi
+	fi
 
-    export TUNE2FS=$TUNE2FS
-    if [ -z "$TUNE2FS" ]; then
-        if which tunefs.ldiskfs >/dev/null 2>&1; then
-            export TUNE2FS=tunefs.ldiskfs
-        else
-            export TUNE2FS=tune2fs
-        fi
-    fi
+	export TUNE2FS=$TUNE2FS
+	if [ -z "$TUNE2FS" ]; then
+		if which tunefs.ldiskfs >/dev/null 2>&1; then
+			export TUNE2FS=tunefs.ldiskfs
+		else
+			export TUNE2FS=tune2fs
+		fi
+	fi
 
-    export E2LABEL=$E2LABEL
-    if [ -z "$E2LABEL" ]; then
-        if which label.ldiskfs >/dev/null 2>&1; then
-            export E2LABEL=label.ldiskfs
-        else
-            export E2LABEL=e2label
-        fi
-    fi
+	export E2LABEL=$E2LABEL
+	if [ -z "$E2LABEL" ]; then
+		if which label.ldiskfs >/dev/null 2>&1; then
+			export E2LABEL=label.ldiskfs
+		else
+			export E2LABEL=e2label
+		fi
+	fi
 
-    export DUMPE2FS=$DUMPE2FS
-    if [ -z "$DUMPE2FS" ]; then
-        if which dumpfs.ldiskfs >/dev/null 2>&1; then
-            export DUMPE2FS=dumpfs.ldiskfs
-        else
-            export DUMPE2FS=dumpe2fs
-        fi
-    fi
+	export DUMPE2FS=$DUMPE2FS
+	if [ -z "$DUMPE2FS" ]; then
+		if which dumpfs.ldiskfs >/dev/null 2>&1; then
+			export DUMPE2FS=dumpfs.ldiskfs
+		else
+			export DUMPE2FS=dumpe2fs
+		fi
+	fi
 
-    export E2FSCK=$E2FSCK
-    if [ -z "$E2FSCK" ]; then
-        if which fsck.ldiskfs >/dev/null 2>&1; then
-            export E2FSCK=fsck.ldiskfs
-        else
-            export E2FSCK=e2fsck
-        fi
-    fi
+	export E2FSCK=$E2FSCK
+	if [ -z "$E2FSCK" ]; then
+		if which fsck.ldiskfs >/dev/null 2>&1; then
+			export E2FSCK=fsck.ldiskfs
+		else
+			 export E2FSCK=e2fsck
+		fi
+	fi
 
 	export RESIZE2FS=$RESIZE2FS
 	if [ -z "$RESIZE2FS" ]; then
@@ -216,119 +226,129 @@ init_test_env() {
 	export ZDB=${ZDB:-zdb}
 	export PARTPROBE=${PARTPROBE:-partprobe}
 
-    #[ -d /r ] && export ROOT=${ROOT:-/r}
-    export TMP=${TMP:-$ROOT/tmp}
-    export TESTSUITELOG=${TMP}/${TESTSUITE}.log
-    export LOGDIR=${LOGDIR:-${TMP}/test_logs/$(date +%s)}
-    export TESTLOG_PREFIX=$LOGDIR/$TESTSUITE
+	#[ -d /r ] && export ROOT=${ROOT:-/r}
+	export TMP=${TMP:-$ROOT/tmp}
+	export TESTSUITELOG=${TMP}/${TESTSUITE}.log
+	export LOGDIR=${LOGDIR:-${TMP}/test_logs/$(date +%s)}
+	export TESTLOG_PREFIX=$LOGDIR/$TESTSUITE
 
-    export HOSTNAME=${HOSTNAME:-$(hostname -s)}
-    if ! echo $PATH | grep -q $LUSTRE/utils; then
-        export PATH=$LUSTRE/utils:$PATH
-    fi
-    if ! echo $PATH | grep -q $LUSTRE/utils/gss; then
-        export PATH=$LUSTRE/utils/gss:$PATH
-    fi
-    if ! echo $PATH | grep -q $LUSTRE/tests; then
-        export PATH=$LUSTRE/tests:$PATH
-    fi
-    if ! echo $PATH | grep -q $LUSTRE/../lustre-iokit/sgpdd-survey; then
-        export PATH=$LUSTRE/../lustre-iokit/sgpdd-survey:$PATH
-    fi
-    export LST=${LST:-"$LUSTRE/../lnet/utils/lst"}
-    [ ! -f "$LST" ] && export LST=$(which lst)
-    export SGPDDSURVEY=${SGPDDSURVEY:-"$LUSTRE/../lustre-iokit/sgpdd-survey/sgpdd-survey")}
-    [ ! -f "$SGPDDSURVEY" ] && export SGPDDSURVEY=$(which sgpdd-survey)
+	export HOSTNAME=${HOSTNAME:-$(hostname -s)}
+	if ! echo $PATH | grep -q $LUSTRE/utils; then
+		export PATH=$LUSTRE/utils:$PATH
+	fi
+	if ! echo $PATH | grep -q $LUSTRE/utils/gss; then
+		export PATH=$LUSTRE/utils/gss:$PATH
+	fi
+	if ! echo $PATH | grep -q $LUSTRE/tests; then
+		export PATH=$LUSTRE/tests:$PATH
+	fi
+	if ! echo $PATH | grep -q $LUSTRE/../lustre-iokit/sgpdd-survey; then
+		export PATH=$LUSTRE/../lustre-iokit/sgpdd-survey:$PATH
+	fi
+	export LST=${LST:-"$LUSTRE/../lnet/utils/lst"}
+	[ ! -f "$LST" ] && export LST=$(which lst)
+	export SGPDDSURVEY=${SGPDDSURVEY:-"$LUSTRE/../lustre-iokit/sgpdd-survey/sgpdd-survey")}
+	[ ! -f "$SGPDDSURVEY" ] && export SGPDDSURVEY=$(which sgpdd-survey)
 	export MCREATE=${MCREATE:-mcreate}
-    # Ubuntu, at least, has a truncate command in /usr/bin
-    # so fully path our truncate command.
-    export TRUNCATE=${TRUNCATE:-$LUSTRE/tests/truncate}
+	# Ubuntu, at least, has a truncate command in /usr/bin
+	# so fully path our truncate command.
+	export TRUNCATE=${TRUNCATE:-$LUSTRE/tests/truncate}
 	export FSX=${FSX:-$LUSTRE/tests/fsx}
-    export MDSRATE=${MDSRATE:-"$LUSTRE/tests/mpi/mdsrate"}
-    [ ! -f "$MDSRATE" ] && export MDSRATE=$(which mdsrate 2> /dev/null)
-    if ! echo $PATH | grep -q $LUSTRE/tests/racer; then
-        export PATH=$LUSTRE/tests/racer:$PATH:
-    fi
-    if ! echo $PATH | grep -q $LUSTRE/tests/mpi; then
-        export PATH=$LUSTRE/tests/mpi:$PATH
-    fi
-    export RSYNC_RSH=${RSYNC_RSH:-rsh}
-
-    export LCTL=${LCTL:-"$LUSTRE/utils/lctl"}
-    [ ! -f "$LCTL" ] && export LCTL=$(which lctl)
-    export LFS=${LFS:-"$LUSTRE/utils/lfs"}
-    [ ! -f "$LFS" ] && export LFS=$(which lfs)
-    SETSTRIPE=${SETSTRIPE:-"$LFS setstripe"}
-    GETSTRIPE=${GETSTRIPE:-"$LFS getstripe"}
-
-    export L_GETIDENTITY=${L_GETIDENTITY:-"$LUSTRE/utils/l_getidentity"}
-    if [ ! -f "$L_GETIDENTITY" ]; then
-        if `which l_getidentity > /dev/null 2>&1`; then
-            export L_GETIDENTITY=$(which l_getidentity)
-        else
-            export L_GETIDENTITY=NONE
-        fi
-    fi
-    export LL_DECODE_FILTER_FID=${LL_DECODE_FILTER_FID:-"$LUSTRE/utils/ll_decode_filter_fid"}
-    [ ! -f "$LL_DECODE_FILTER_FID" ] && export LL_DECODE_FILTER_FID="ll_decode_filter_fid"
-    export LL_DECODE_LINKEA=${LL_DECODE_LINKEA:-"$LUSTRE/utils/ll_decode_linkea"}
-    [ ! -f "$LL_DECODE_LINKEA" ] && export LL_DECODE_LINKEA="ll_decode_linkea"
-    export MKFS=${MKFS:-"$LUSTRE/utils/mkfs.lustre"}
-    [ ! -f "$MKFS" ] && export MKFS="mkfs.lustre"
-    export TUNEFS=${TUNEFS:-"$LUSTRE/utils/tunefs.lustre"}
-    [ ! -f "$TUNEFS" ] && export TUNEFS="tunefs.lustre"
-    export CHECKSTAT="${CHECKSTAT:-"checkstat -v"} "
-    export LUSTRE_RMMOD=${LUSTRE_RMMOD:-$LUSTRE/scripts/lustre_rmmod}
-    [ ! -f "$LUSTRE_RMMOD" ] &&
-        export LUSTRE_RMMOD=$(which lustre_rmmod 2> /dev/null)
-    export LFS_MIGRATE=${LFS_MIGRATE:-$LUSTRE/scripts/lfs_migrate}
-    [ ! -f "$LFS_MIGRATE" ] &&
-        export LFS_MIGRATE=$(which lfs_migrate 2> /dev/null)
-    export LR_READER=${LR_READER:-"$LUSTRE/utils/lr_reader"}
-    [ ! -f "$LR_READER" ] && export LR_READER=$(which lr_reader 2> /dev/null)
-    [ -z "$LR_READER" ] && export LR_READER="/usr/sbin/lr_reader"
-    export NAME=${NAME:-local}
-    export LGSSD=${LGSSD:-"$LUSTRE/utils/gss/lgssd"}
-    [ "$GSS_PIPEFS" = "true" ] && [ ! -f "$LGSSD" ] && \
-        export LGSSD=$(which lgssd)
-    export LSVCGSSD=${LSVCGSSD:-"$LUSTRE/utils/gss/lsvcgssd"}
-    [ ! -f "$LSVCGSSD" ] && export LSVCGSSD=$(which lsvcgssd 2> /dev/null)
-    export KRB5DIR=${KRB5DIR:-"/usr/kerberos"}
-    export DIR2
-    export SAVE_PWD=${SAVE_PWD:-$LUSTRE/tests}
-    export AT_MAX_PATH
-    export LDEV=${LDEV:-"$LUSTRE/scripts/ldev"}
-    [ ! -f "$LDEV" ] && export LDEV=$(which ldev 2> /dev/null)
-
-    if [ "$ACCEPTOR_PORT" ]; then
-        export PORT_OPT="--port $ACCEPTOR_PORT"
-    fi
+	export MDSRATE=${MDSRATE:-"$LUSTRE/tests/mpi/mdsrate"}
+	[ ! -f "$MDSRATE" ] && export MDSRATE=$(which mdsrate 2> /dev/null)
+	if ! echo $PATH | grep -q $LUSTRE/tests/racer; then
+		export PATH=$LUSTRE/tests/racer:$PATH:
+	fi
+	if ! echo $PATH | grep -q $LUSTRE/tests/mpi; then
+		export PATH=$LUSTRE/tests/mpi:$PATH
+	fi
+	export RSYNC_RSH=${RSYNC_RSH:-rsh}
+
+	export LCTL=${LCTL:-"$LUSTRE/utils/lctl"}
+	[ ! -f "$LCTL" ] && export LCTL=$(which lctl)
+	export LFS=${LFS:-"$LUSTRE/utils/lfs"}
+	[ ! -f "$LFS" ] && export LFS=$(which lfs)
+	SETSTRIPE=${SETSTRIPE:-"$LFS setstripe"}
+	GETSTRIPE=${GETSTRIPE:-"$LFS getstripe"}
+
+	export L_GETIDENTITY=${L_GETIDENTITY:-"$LUSTRE/utils/l_getidentity"}
+	if [ ! -f "$L_GETIDENTITY" ]; then
+		if `which l_getidentity > /dev/null 2>&1`; then
+			export L_GETIDENTITY=$(which l_getidentity)
+		else
+			export L_GETIDENTITY=NONE
+		fi
+	fi
+	export LL_DECODE_FILTER_FID=${LL_DECODE_FILTER_FID:-"$LUSTRE/utils/ll_decode_filter_fid"}
+	[ ! -f "$LL_DECODE_FILTER_FID" ] && export LL_DECODE_FILTER_FID="ll_decode_filter_fid"
+	export LL_DECODE_LINKEA=${LL_DECODE_LINKEA:-"$LUSTRE/utils/ll_decode_linkea"}
+	[ ! -f "$LL_DECODE_LINKEA" ] && export LL_DECODE_LINKEA="ll_decode_linkea"
+	export MKFS=${MKFS:-"$LUSTRE/utils/mkfs.lustre"}
+	[ ! -f "$MKFS" ] && export MKFS="mkfs.lustre"
+	export TUNEFS=${TUNEFS:-"$LUSTRE/utils/tunefs.lustre"}
+	[ ! -f "$TUNEFS" ] && export TUNEFS="tunefs.lustre"
+	export CHECKSTAT="${CHECKSTAT:-"checkstat -v"} "
+	export LUSTRE_RMMOD=${LUSTRE_RMMOD:-$LUSTRE/scripts/lustre_rmmod}
+	[ ! -f "$LUSTRE_RMMOD" ] &&
+		export LUSTRE_RMMOD=$(which lustre_rmmod 2> /dev/null)
+	export LFS_MIGRATE=${LFS_MIGRATE:-$LUSTRE/scripts/lfs_migrate}
+	[ ! -f "$LFS_MIGRATE" ] &&
+		export LFS_MIGRATE=$(which lfs_migrate 2> /dev/null)
+	export LR_READER=${LR_READER:-"$LUSTRE/utils/lr_reader"}
+	[ ! -f "$LR_READER" ] &&
+		export LR_READER=$(which lr_reader 2> /dev/null)
+	[ -z "$LR_READER" ] && export LR_READER="/usr/sbin/lr_reader"
+	export NAME=${NAME:-local}
+	export LGSSD=${LGSSD:-"$LUSTRE/utils/gss/lgssd"}
+	[ "$GSS_PIPEFS" = "true" ] && [ ! -f "$LGSSD" ] &&
+		export LGSSD=$(which lgssd)
+	export LSVCGSSD=${LSVCGSSD:-"$LUSTRE/utils/gss/lsvcgssd"}
+	[ ! -f "$LSVCGSSD" ] && export LSVCGSSD=$(which lsvcgssd 2> /dev/null)
+	export KRB5DIR=${KRB5DIR:-"/usr/kerberos"}
+	export DIR2
+	export SAVE_PWD=${SAVE_PWD:-$LUSTRE/tests}
+	export AT_MAX_PATH
+	export LDEV=${LDEV:-"$LUSTRE/scripts/ldev"}
+	[ ! -f "$LDEV" ] && export LDEV=$(which ldev 2> /dev/null)
+
+	if [ "$ACCEPTOR_PORT" ]; then
+		export PORT_OPT="--port $ACCEPTOR_PORT"
+	fi
+
+	if $SHARED_KEY; then
+		$RPC_MODE || echo "Using GSS shared-key feature"
+		which lgss_sk > /dev/null 2>&1 ||
+			error_exit "built with lgss_sk disabled! SEC=$SEC"
+		GSS=true
+		GSS_SK=true
+		SEC=$SK_FLAVOR
+	fi
+
+	case "x$SEC" in
+		xkrb5*)
+		$RPC_MODE || echo "Using GSS/krb5 ptlrpc security flavor"
+		which lgss_keyring > /dev/null 2>&1 ||
+			error_exit "built with gss disabled! SEC=$SEC"
+		GSS=true
+		GSS_KRB5=true
+		;;
+	esac
+
+	case "x$IDUP" in
+		xtrue)
+			IDENTITY_UPCALL=true
+			;;
+		xfalse)
+			IDENTITY_UPCALL=false
+			;;
+	esac
+
+	export LOAD_MODULES_REMOTE=${LOAD_MODULES_REMOTE:-false}
 
-    case "x$SEC" in
-        xkrb5*)
-            echo "Using GSS/krb5 ptlrpc security flavor"
-            which lgss_keyring > /dev/null 2>&1 || \
-                error_exit "built with gss disabled! SEC=$SEC"
-            GSS=true
-            GSS_KRB5=true
-            ;;
-    esac
-
-    case "x$IDUP" in
-        xtrue)
-            IDENTITY_UPCALL=true
-            ;;
-        xfalse)
-            IDENTITY_UPCALL=false
-            ;;
-    esac
-
-    export LOAD_MODULES_REMOTE=${LOAD_MODULES_REMOTE:-false}
-
-    # Paths on remote nodes, if different
-    export RLUSTRE=${RLUSTRE:-$LUSTRE}
-    export RPWD=${RPWD:-$PWD}
-    export I_MOUNTED=${I_MOUNTED:-"no"}
+	# Paths on remote nodes, if different
+	export RLUSTRE=${RLUSTRE:-$LUSTRE}
+	export RPWD=${RPWD:-$PWD}
+	export I_MOUNTED=${I_MOUNTED:-"no"}
 	if [ ! -f /lib/modules/$(uname -r)/kernel/fs/lustre/mdt.ko -a \
 	     ! -f /lib/modules/$(uname -r)/updates/kernel/fs/lustre/mdt.ko -a \
 	     ! -f /lib/modules/$(uname -r)/extra/kernel/fs/lustre/mdt.ko -a \
@@ -776,88 +796,234 @@ send_sigint() {
     do_nodes $list "killall -2 $@ 2>/dev/null || true"
 }
 
-# start gss daemons on all nodes, or
-# "daemon" on "list" if set
+# start gss daemons on all nodes, or "daemon" on "nodes" if set
 start_gss_daemons() {
-    local list=$1
-    local daemon=$2
+	local nodes=$1
+	local daemon=$2
 
-    if [ "$list" ] && [ "$daemon" ] ; then
-        echo "Starting gss daemon on nodes: $list"
-        do_nodes $list "$daemon" || return 8
-        return 0
-    fi
+	if [ "$nodes" ] && [ "$daemon" ] ; then
+		echo "Starting gss daemon on nodes: $nodes"
+		do_nodes $nodes "$daemon" || return 8
+		return 0
+	fi
 
-    local list=$(comma_list $(mdts_nodes))
-    echo "Starting gss daemon on mds: $list"
-    do_nodes $list "$LSVCGSSD -v" || return 1
-    if $GSS_PIPEFS; then
-        do_nodes $list "$LGSSD -v" || return 2
-    fi
+	nodes=$(comma_list $(mdts_nodes))
+	echo "Starting gss daemon on mds: $nodes"
+	if $GSS_SK; then
+		# Start all versions, in case of switching
+		do_nodes $nodes "$LSVCGSSD -vvv -s -m -o -z" || return 1
+	else
+		do_nodes $nodes "$LSVCGSSD -v" || return 1
+	fi
+	if $GSS_PIPEFS; then
+		do_nodes $nodes "$LGSSD -v" || return 2
+	fi
 
-    list=$(comma_list $(osts_nodes))
-    echo "Starting gss daemon on ost: $list"
-    do_nodes $list "$LSVCGSSD -v" || return 3
-    # starting on clients
+	nodes=$(comma_list $(osts_nodes))
+	echo "Starting gss daemon on ost: $nodes"
+	if $GSS_SK; then
+		# Start all versions, in case of switching
+		do_nodes $nodes "$LSVCGSSD -vvv -s -m -o -z" || return 3
+	else
+		do_nodes $nodes "$LSVCGSSD -v" || return 3
+	fi
+	# starting on clients
 
-    local clients=${CLIENTS:-`hostname`}
-    if $GSS_PIPEFS; then
-        echo "Starting $LGSSD on clients $clients "
-        do_nodes $clients  "$LGSSD -v" || return 4
-    fi
+	local clients=${CLIENTS:-$HOSTNAME}
+	if $GSS_PIPEFS; then
+		echo "Starting $LGSSD on clients $clients "
+		do_nodes $clients  "$LGSSD -v" || return 4
+	fi
 
-    # wait daemons entering "stable" status
-    sleep 5
-
-    #
-    # check daemons are running
-    #
-    list=$(comma_list $(mdts_nodes) $(osts_nodes))
-    check_gss_daemon_nodes $list lsvcgssd || return 5
-    if $GSS_PIPEFS; then
-        list=$(comma_list $(mdts_nodes))
-        check_gss_daemon_nodes $list lgssd || return 6
-    fi
-    if $GSS_PIPEFS; then
-        check_gss_daemon_nodes $clients lgssd || return 7
-    fi
+	# wait daemons entering "stable" status
+	sleep 5
+
+	#
+	# check daemons are running
+	#
+	nodes=$(comma_list $(mdts_nodes) $(osts_nodes))
+	check_gss_daemon_nodes $nodes lsvcgssd || return 5
+	if $GSS_PIPEFS; then
+		nodes=$(comma_list $(mdts_nodes))
+		check_gss_daemon_nodes $nodes lgssd || return 6
+	fi
+	if $GSS_PIPEFS; then
+		check_gss_daemon_nodes $clients lgssd || return 7
+	fi
 }
 
 stop_gss_daemons() {
-    local list=$(comma_list $(mdts_nodes))
+	local nodes=$(comma_list $(mdts_nodes))
 
-    send_sigint $list lsvcgssd lgssd
+	send_sigint $nodes lsvcgssd lgssd
 
-    list=$(comma_list $(osts_nodes))
-    send_sigint $list lsvcgssd
+	nodes=$(comma_list $(osts_nodes))
+	send_sigint $nodes lsvcgssd
 
-    list=${CLIENTS:-`hostname`}
-    send_sigint $list lgssd
+	nodes=${CLIENTS:-$HOSTNAME}
+	send_sigint $nodes lgssd
+}
+
+add_sk_mntflag() {
+	# Add mount flags for shared key
+	local mt_opts=$@
+	if grep -q skpath <<< "$mt_opts" ; then
+		mt_opts=$(echo $mt_opts |
+			sed -e "s#skpath=[^ ,]*#skpath=$SK_PATH#")
+	else
+		if [ -z "$mt_opts" ]; then
+			mt_opts="-o skpath=$SK_PATH"
+		else
+			mt_opts="$mt_opts,skpath=$SK_PATH"
+		fi
+	fi
+	echo -n $mt_opts
 }
 
 init_gss() {
-    if $GSS; then
-        if ! module_loaded ptlrpc_gss; then
-            load_module ptlrpc/gss/ptlrpc_gss
-            module_loaded ptlrpc_gss ||
-                error_exit "init_gss : GSS=$GSS, but gss/krb5 is not supported!"
-        fi
-        if $GSS_KRB5; then
-                start_gss_daemons || error_exit "start gss daemon failed! rc=$?"
-        fi
+	if $SHARED_KEY; then
+		GSS=true
+		GSS_SK=true
+	fi
+
+	if ! $GSS; then
+		return
+	fi
+
+	if ! module_loaded ptlrpc_gss; then
+		load_module ptlrpc/gss/ptlrpc_gss
+		module_loaded ptlrpc_gss ||
+			error_exit "init_gss: GSS=$GSS, but gss/krb5 missing"
+	fi
+
+	if $GSS_KRB5 || $GSS_SK; then
+		start_gss_daemons || error_exit "start gss daemon failed! rc=$?"
+	fi
+
+	if $GSS_SK && $SK_NO_KEY; then
+		local numclients=${1:-$CLIENTCOUNT}
+		local clients=${CLIENTS:-$HOSTNAME}
+
+		# security ctx config for keyring
+		SK_NO_KEY=false
+		mkdir -p $SK_OM_PATH
+		mount -o bind $SK_OM_PATH /etc/request-key.d/
+		local lgssc_conf_line='create lgssc * * '
+		lgssc_conf_line+=$(which lgss_keyring)
+		lgssc_conf_line+=' %o %k %t %d %c %u %g %T %P %S'
+
+		local lgssc_conf_file="/etc/request-key.d/lgssc.conf"
+		echo "$lgssc_conf_line" > $lgssc_conf_file
+		[ -e $lgssc_conf_file ] ||
+			error_exit "Could not find key options in $lgssc_conf_file"
 
-        if [ -n "$LGSS_KEYRING_DEBUG" ]; then
+		if ! local_mode; then
+			do_nodes $(comma_list $(all_nodes)) "mkdir -p \
+				$SK_OM_PATH"
+			do_nodes $(comma_list $(all_nodes)) "mount \
+				-o bind $SK_OM_PATH \
+				/etc/request-key.d/"
+			do_nodes $(comma_list $(all_nodes)) "rsync -aqv \
+				$HOSTNAME:$lgssc_conf_file \
+				$lgssc_conf_file >/dev/null 2>&1"
+		fi
+
+		# create shared key on all nodes
+		mkdir -p $SK_PATH/nodemap
+		rm -f $SK_PATH/$FSNAME.key $SK_PATH/nodemap/c*.key \
+			$SK_PATH/$FSNAME-*.key
+		# for nodemap testing each client may need own key,
+		# and S2S now requires keys as well, both for "client"
+		# and for "server"
+		if $SK_S2S; then
+			lgss_sk -t server -f$FSNAME -n $SK_S2SNMCLI \
+				-w $SK_PATH/$FSNAME-nmclient.key \
+				-d /dev/urandom >/dev/null 2>&1
+			lgss_sk -t mgs,server -f$FSNAME -n $SK_S2SNM \
+				-w $SK_PATH/$FSNAME-s2s-server.key \
+				-d /dev/urandom >/dev/null 2>&1
+		fi
+		# basic key create
+		lgss_sk -t server -f$FSNAME -w $SK_PATH/$FSNAME.key \
+			-d /dev/urandom >/dev/null 2>&1
+		# per-nodemap keys
+		for i in $(seq 0 $((numclients - 1))); do
+			lgss_sk -t server -f$FSNAME -n c$i \
+				-w $SK_PATH/nodemap/c$i.key -d /dev/urandom \
+				>/dev/null 2>&1
+		done
+		# Distribute keys
+		if ! local_mode; then
+			do_nodes $(comma_list $(all_nodes)) "rsync -av \
+				$HOSTNAME:$SK_PATH/ $SK_PATH >/dev/null 2>&1"
+		fi
+		# Set client keys to client type to generate prime P
+		if local_mode; then
+			do_nodes $(all_nodes) "lgss_sk -t client,server -m \
+				$SK_PATH/$FSNAME.key >/dev/null 2>&1"
+		else
+			do_nodes $clients "lgss_sk -t client -m \
+				$SK_PATH/$FSNAME.key >/dev/null 2>&1"
+			do_nodes $clients "find $SK_PATH/nodemap -name \*.key | \
+				xargs -IX lgss_sk -t client -m X >/dev/null 2>&1"
+		fi
+		# This is required for servers as well, if S2S in use
+		if $SK_S2S; then
+			do_nodes $(comma_list $(mdts_nodes)) \
+				"cp $SK_PATH/$FSNAME-s2s-server.key \
+				$SK_PATH/$FSNAME-s2s-client.key; lgss_sk \
+				-t client -m $SK_PATH/$FSNAME-s2s-client.key \
+				>/dev/null 2>&1"
+			do_nodes $(comma_list $(osts_nodes)) \
+				"cp $SK_PATH/$FSNAME-s2s-server.key \
+				$SK_PATH/$FSNAME-s2s-client.key; lgss_sk \
+				-t client -m $SK_PATH/$FSNAME-s2s-client.key \
+				>/dev/null 2>&1"
+			do_nodes $clients "lgss_sk -t client \
+				-m $SK_PATH/$FSNAME-nmclient.key \
+				 >/dev/null 2>&1"
+		fi
+		# mount options for servers and clients
+		MGS_MOUNT_OPTS=$(add_sk_mntflag $MGS_MOUNT_OPTS)
+		MDS_MOUNT_OPTS=$(add_sk_mntflag $MDS_MOUNT_OPTS)
+		OST_MOUNT_OPTS=$(add_sk_mntflag $OST_MOUNT_OPTS)
+		MOUNT_OPTS=$(add_sk_mntflag $MOUNT_OPTS)
+		SEC=$SK_FLAVOR
+	fi
+
+	if [ -n "$LGSS_KEYRING_DEBUG" ]; then
 		lctl set_param -n \
-		    sptlrpc.gss.lgss_keyring.debug_level=$LGSS_KEYRING_DEBUG
-        fi
-    fi
+			sptlrpc.gss.lgss_keyring.debug_level=$LGSS_KEYRING_DEBUG
+	fi
 }
 
 cleanup_gss() {
-    if $GSS; then
-        stop_gss_daemons
-        # maybe cleanup credential cache?
-    fi
+	if $GSS; then
+		stop_gss_daemons
+		# maybe cleanup credential cache?
+	fi
+}
+
+cleanup_sk() {
+	if $GSS_SK; then
+		if $SK_S2S; then
+			do_node $(mgs_node) "$LCTL nodemap_del $SK_S2SNM"
+			do_node $(mgs_node) "$LCTL nodemap_del $SK_S2SNMCLI"
+			$RPC_MODE || echo "Sleeping for 10 sec for Nodemap.."
+			sleep 10
+		fi
+		stop_gss_daemons
+		$RPC_MODE || echo "Cleaning up Shared Key.."
+		do_nodes $(comma_list $(all_nodes)) "rm -f \
+			$SK_PATH/$FSNAME*.key $SK_PATH/nodemap/$FSNAME*.key"
+		# Remove the mount and clean up the files we added to SK_PATH
+		do_nodes $(comma_list $(all_nodes)) "umount \
+			/etc/request-key.d/"
+		do_nodes $(comma_list $(all_nodes)) "rm -f \
+			$SK_OM_PATH/lgssc.conf"
+		do_nodes $(comma_list $(all_nodes)) "rmdir $SK_OM_PATH"
+		SK_NO_KEY=true
+	fi
 }
 
 facet_svc() {
@@ -1653,7 +1819,20 @@ zconf_mount() {
 		do_node $client "! grep -q $mnt' ' /proc/mounts ||
 			umount $mnt"
 	fi
-	do_node $client $MOUNT_CMD $flags $opts $device $mnt || return 1
+	if $GSS_SK && ($SK_UNIQUE_NM || $SK_S2S); then
+		# Mount using nodemap key
+		local mountkey=$SK_PATH/$FSNAME-nmclient.key
+		if $SK_UNIQUE_NM; then
+			mountkey=$SK_PATH/nodemap/c0.key
+		fi
+		local prunedopts=$(echo $opts |
+				sed -e "s#skpath=[^,^ ]*#skpath=$mountkey#g")
+		do_node $client $MOUNT_CMD $flags $prunedopts $device $mnt ||
+				return 1
+	else
+		do_node $client $MOUNT_CMD $flags $opts $device $mnt ||
+				return 1
+	fi
 
 	set_default_debug_nodes $client
 
@@ -1752,7 +1931,6 @@ zconf_mount_clients() {
 	local opts=${3:-$MOUNT_OPTS}
 	opts=${opts:+-o $opts}
 	local flags=${4:-$MOUNT_FLAGS}
-
 	local device=$MGSNID:/$FSNAME$FILESET
 	if [ -z "$mnt" -o -z "$FSNAME" ]; then
 		echo "Bad conf mount command: opt=$flags $opts dev=$device " \
@@ -1762,10 +1940,46 @@ zconf_mount_clients() {
 
 	echo "Starting client $clients: $flags $opts $device $mnt"
 	if [ -n "$FILESET" -a ! -n "$SKIP_FILESET" ]; then
-		do_nodes $clients "! grep -q $mnt' ' /proc/mounts ||
-			umount $mnt"
-		do_nodes $clients $MOUNT_CMD $flags $opts $MGSNID:/$FSNAME \
-			$mnt || return 1
+		if $GSS_SK && ($SK_UNIQUE_NM || $SK_S2S); then
+			# Mount with own nodemap key
+			local i=0
+			# Mount all server nodes first with per-NM keys
+			for nmclient in ${clients//,/ }; do
+#				do_nodes $(comma_list $(all_server_nodes)) "lgss_sk -t server -l $SK_PATH/nodemap/c$i.key -n c$i"
+				do_nodes $(comma_list $(all_server_nodes)) "lgss_sk -t server -l $SK_PATH/nodemap/c$i.key"
+				i=$((i + 1))
+			done
+			# set perms for per-nodemap keys else permission denied
+			do_nodes $(comma_list $(all_nodes)) \
+				"keyctl show | grep lustre | cut -c1-11 |
+				sed -e 's/ //g;' |
+				xargs -IX keyctl setperm X 0x3f3f3f3f"
+			local mountkey=$SK_PATH/$FSNAME-nmclient.key
+			i=0
+			for nmclient in ${clients//,/ }; do
+				if $SK_UNIQUE_NM; then
+					mountkey=$SK_PATH/nodemap/c$i.key
+				fi
+				do_node $nmclient "! grep -q $mnt' ' \
+					/proc/mounts || umount $mnt"
+				local prunedopts=$(add_sk_mntflag $prunedopts);
+				prunedopts=$(echo $prunedopts | sed -e \
+					"s#skpath=[^ ^,]*#skpath=$mountkey#g")
+				set -x
+				do_nodes $(comma_list $(all_server_nodes)) \
+					"keyctl show"
+				set +x
+				do_node $nmclient $MOUNT_CMD $flags \
+					$prunedopts $MGSNID:/$FSNAME $mnt ||
+					return 1
+				i=$((i + 1))
+			done
+		else
+			do_nodes $clients "! grep -q $mnt' ' /proc/mounts ||
+					umount $mnt"
+			do_nodes $clients $MOUNT_CMD $flags $opts \
+					$MGSNID:/$FSNAME $mnt || return 1
+		fi
 		#disable FILESET if not supported
 		do_nodes $clients lctl get_param -n \
 			mdc.$FSNAME-MDT0000*.import | grep -q subtree ||
@@ -1775,12 +1989,24 @@ zconf_mount_clients() {
 			umount $mnt"
 	fi
 
-	do_nodes $clients "
+	if $GSS_SK && ($SK_UNIQUE_NM || $SK_S2S); then
+		# Mount with nodemap key
+		local i=0
+		local mountkey=$SK_PATH/$FSNAME-nmclient.key
+		for nmclient in ${clients//,/ }; do
+			if $SK_UNIQUE_NM; then
+				mountkey=$SK_PATH/nodemap/c$i.key
+			fi
+			local prunedopts=$(echo $opts | sed -e \
+				"s#skpath=[^ ^,]*#skpath=$mountkey#g");
+			do_node $nmclient "! grep -q $mnt' ' /proc/mounts ||
+				umount $mnt"
+			do_node $nmclient "
 		running=\\\$(mount | grep -c $mnt' ');
 		rc=0;
 		if [ \\\$running -eq 0 ] ; then
 			mkdir -p $mnt;
-			$MOUNT_CMD $flags $opts $device $mnt;
+			$MOUNT_CMD $flags $prunedopts $device $mnt;
 			rc=\\\$?;
 		else
 			lustre_mnt_count=\\\$(mount | grep $mnt' ' | \
@@ -1795,6 +2021,25 @@ zconf_mount_clients() {
 		fi;
 	exit \\\$rc" || return ${PIPESTATUS[0]}
 
+			i=$((i + 1))
+		done
+	else
+
+		local tmpopts=$opts
+		if $SHARED_KEY; then
+			tmpopts=$(add_sk_mntflag $opts)
+		fi
+		do_nodes $clients "
+running=\\\$(mount | grep -c $mnt' ');
+rc=0;
+if [ \\\$running -eq 0 ] ; then
+	mkdir -p $mnt;
+	$MOUNT_CMD $flags $tmpopts $device $mnt;
+	rc=\\\$?;
+fi;
+exit \\\$rc" || return ${PIPESTATUS[0]}
+	fi
+
 	echo "Started clients $clients: "
 	do_nodes $clients "mount | grep $mnt' '"
 
@@ -2541,7 +2786,6 @@ lfs_df_check() {
 	fi
 }
 
-
 clients_up() {
 	# not every config has many clients
 	sleep 1
@@ -3502,14 +3746,15 @@ cleanup_echo_devs () {
 }
 
 cleanupall() {
-    nfs_client_mode && return
+	nfs_client_mode && return
 	cifs_client_mode && return
 
-    stopall $*
-    cleanup_echo_devs
+	stopall $*
+	cleanup_echo_devs
 
-    unload_modules
-    cleanup_gss
+	unload_modules
+	cleanup_sk
+	cleanup_gss
 }
 
 combined_mgs_mds () {
@@ -3754,11 +3999,11 @@ formatall() {
 }
 
 mount_client() {
-    grep " $1 " /proc/mounts || zconf_mount $HOSTNAME $*
+	grep " $1 " /proc/mounts || zconf_mount $HOSTNAME $*
 }
 
 umount_client() {
-    grep " $1 " /proc/mounts && zconf_umount `hostname` $*
+	grep " $1 " /proc/mounts && zconf_umount $HOSTNAME $*
 }
 
 # return value:
@@ -3794,8 +4039,8 @@ switch_identity() {
 
 remount_client()
 {
-        zconf_umount `hostname` $1 || error "umount failed"
-        zconf_mount `hostname` $1 || error "mount failed"
+	zconf_umount $HOSTNAME $1 || error "umount failed"
+	zconf_mount $HOSTNAME $1 || error "mount failed"
 }
 
 writeconf_facet() {
@@ -3825,6 +4070,75 @@ writeconf_all () {
 	return $rc
 }
 
+mountmgs() {
+	if ! combined_mgs_mds ; then
+		start mgs $(mgsdevname) $MGS_MOUNT_OPTS
+	fi
+}
+
+mountmds() {
+	for num in $(seq $MDSCOUNT); do
+		DEVNAME=$(mdsdevname $num)
+		start mds$num $DEVNAME $MDS_MOUNT_OPTS
+
+		# We started mds, now we should set failover variables properly.
+		# Set mds${num}failover_HOST if unset (the default
+		# failnode).
+		local varname=mds${num}failover_HOST
+		if [ -z "${!varname}" ]; then
+			eval mds${num}failover_HOST=$(facet_host mds$num)
+		fi
+
+		if [ $IDENTITY_UPCALL != "default" ]; then
+			switch_identity $num $IDENTITY_UPCALL
+		fi
+	done
+}
+
+mountoss() {
+	for num in $(seq $OSTCOUNT); do
+		DEVNAME=$(ostdevname $num)
+		start ost$num $DEVNAME $OST_MOUNT_OPTS
+
+		# We started ost$num, now we should set ost${num}failover
+		# variable properly. Set ost${num}failover_HOST if it is not
+		# set (the default failnode).
+		varname=ost${num}failover_HOST
+		if [ -z "${!varname}" ]; then
+			eval ost${num}failover_HOST=$(facet_host ost${num})
+		fi
+
+	done
+}
+
+mountcli() {
+	[ "$DAEMONFILE" ] && $LCTL debug_daemon start $DAEMONFILE $DAEMONSIZE
+	if [ ! -z $arg1 ]; then
+		[ "$arg1" = "server_only" ] && return
+	fi
+	mount_client $MOUNT
+	[ -n "$CLIENTS" ] && zconf_mount_clients $CLIENTS $MOUNT
+	clients_up
+
+	if [ "$MOUNT_2" ]; then
+		mount_client $MOUNT2
+		[ -n "$CLIENTS" ] && zconf_mount_clients $CLIENTS $MOUNT2
+	fi
+}
+
+sk_nodemap_setup() {
+	local sk_map_name=${1:-$SK_S2SNM}
+	local sk_map_nodes=${2:-$HOSTNAME}
+	do_node $(mgs_node) "$LCTL nodemap_add $sk_map_name"
+	for servernode in $sk_map_nodes; do
+		local nids=$(do_nodes $servernode "$LCTL list_nids")
+		for nid in $nids; do
+			do_node $(mgs_node) "$LCTL nodemap_add_range --name \
+				$sk_map_name --range $nid"
+		done
+	done
+}
+
 setupall() {
 	local arg1=$1
 
@@ -3835,78 +4149,73 @@ setupall() {
 
 	load_modules
 
+	init_gss
+
 	if [ -z "$CLIENTONLY" ]; then
 		echo Setup mgs, mdt, osts
 		echo $WRITECONF | grep -q "writeconf" && writeconf_all
-		if ! combined_mgs_mds ; then
-			start mgs $(mgsdevname) $MGS_MOUNT_OPTS
-		fi
-
-        for num in `seq $MDSCOUNT`; do
-            DEVNAME=$(mdsdevname $num)
-            start mds$num $DEVNAME $MDS_MOUNT_OPTS
 
-            # We started mds, now we should set failover variables properly.
-            # Set mds${num}failover_HOST if it is not set (the default failnode).
-            local varname=mds${num}failover_HOST
-            if [ -z "${!varname}" ]; then
-                eval mds${num}failover_HOST=$(facet_host mds$num)
-            fi
-
-            if [ $IDENTITY_UPCALL != "default" ]; then
-                switch_identity $num $IDENTITY_UPCALL
-            fi
-        done
-        for num in `seq $OSTCOUNT`; do
-            DEVNAME=$(ostdevname $num)
-            start ost$num $DEVNAME $OST_MOUNT_OPTS
-
-            # We started ost$num, now we should set ost${num}failover variable properly.
-            # Set ost${num}failover_HOST if it is not set (the default failnode).
-            varname=ost${num}failover_HOST
-            if [ -z "${!varname}" ]; then
-                eval ost${num}failover_HOST=$(facet_host ost${num})
-            fi
-
-        done
-    fi
-
-    init_gss
-
-    # wait a while to allow sptlrpc configuration be propogated to targets,
-    # only needed when mounting new target devices.
-    if $GSS; then
-        sleep 10
-    fi
-
-    [ "$DAEMONFILE" ] && $LCTL debug_daemon start $DAEMONFILE $DAEMONSIZE
-
-	if [ ! -z $arg1 ]; then
-		[ "$arg1" = "server_only" ] && return
+		if $SK_MOUNTED; then
+			echo "Shared Key file system already mounted"
+		else
+			mountmgs
+			mountmds
+			mountoss
+			if $SHARED_KEY; then
+				export SK_MOUNTED=true
+			fi
+		fi
+		if $GSS_SK; then
+			echo "GSS_SK: setting kernel keyring perms"
+			do_nodes $(comma_list $(all_nodes)) \
+				"keyctl show | grep lustre | cut -c1-11 |
+				sed -e 's/ //g;' |
+				xargs -IX keyctl setperm X 0x3f3f3f3f"
+
+			if $SK_S2S; then
+				# Need to start one nodemap for servers,
+				# and one for clients.
+				sk_nodemap_setup $SK_S2SNM \
+					$(comma_list $(all_server_nodes))
+				mountcli
+				sk_nodemap_setup $SK_S2SNMCLI \
+					${CLIENTS:-$HOSTNAME}
+				echo "Nodemap set up for SK S2S, remounting."
+				stopall
+				mountmgs
+				mountmds
+				mountoss
+			fi
+		fi
 	fi
 
-    mount_client $MOUNT
-    [ -n "$CLIENTS" ] && zconf_mount_clients $CLIENTS $MOUNT
-    clients_up
-
-    if [ "$MOUNT_2" ]; then
-        mount_client $MOUNT2
-        [ -n "$CLIENTS" ] && zconf_mount_clients $CLIENTS $MOUNT2
-    fi
+	# wait a while to allow sptlrpc configuration be propogated to targets,
+	# only needed when mounting new target devices.
+	if $GSS; then
+		sleep 10
+	fi
 
-    init_param_vars
+	mountcli
+	init_param_vars
 
-    # by remounting mdt before ost, initial connect from mdt to ost might
-    # timeout because ost is not ready yet. wait some time to its fully
-    # recovery. initial obd_connect timeout is 5s; in GSS case it's preceeded
-    # by a context negotiation rpc with $TIMEOUT.
-    # FIXME better by monitoring import status.
-    if $GSS; then
-        set_flavor_all $SEC
-        sleep $((TIMEOUT + 5))
-    else
-        sleep 5
-    fi
+	# by remounting mdt before ost, initial connect from mdt to ost might
+	# timeout because ost is not ready yet. wait some time to its fully
+	# recovery. initial obd_connect timeout is 5s; in GSS case it's
+	# preceeded by a context negotiation rpc with $TIMEOUT.
+	# FIXME better by monitoring import status.
+	if $GSS; then
+		if $GSS_SK; then
+			set_rule $FSNAME any cli2mdt $SK_FLAVOR
+			set_rule $FSNAME any cli2ost $SK_FLAVOR
+			wait_flavor cli2mdt $SK_FLAVOR
+			wait_flavor cli2ost $SK_FLAVOR
+		else
+			set_flavor_all $SEC
+		fi
+		sleep $((TIMEOUT + 5))
+	else
+		sleep 5
+	fi
 }
 
 mounted_lustre_filesystems() {
@@ -4264,7 +4573,9 @@ check_and_setup_lustre() {
 	fi
 
 	init_gss
-	if $GSS; then
+	if $GSS_SK; then
+		set_flavor_all null
+	elif $GSS; then
 		set_flavor_all $SEC
 	fi
 
@@ -5533,6 +5844,11 @@ facets_nodes () {
 	echo -n $nodes_sort
 }
 
+# Get name of the active MGS node.
+mgs_node () {
+	echo -n $(facets_nodes $(get_facets MGS))
+}
+
 # Get all of the active MDS nodes.
 mdts_nodes () {
 	echo -n $(facets_nodes $(get_facets MDS))
@@ -5574,7 +5890,7 @@ remote_nodes_list () {
 all_mdts_nodes () {
 	local host
 	local failover_host
-	local nodes
+	local nodes="${mds_HOST} ${mdsfailover_HOST}"
 	local nodes_sort
 	local i
 
@@ -5592,7 +5908,7 @@ all_mdts_nodes () {
 all_osts_nodes () {
 	local host
 	local failover_host
-	local nodes
+	local nodes="${ost_HOST} ${ostfailover_HOST}"
 	local nodes_sort
 	local i
 
@@ -6542,6 +6858,7 @@ gather_logs () {
     do_nodesv $list \
         "$LCTL dk > ${prefix}.debug_log.\\\$(hostname -s).${suffix};
          dmesg > ${prefix}.dmesg.\\\$(hostname -s).${suffix}"
+
     if [ ! -f $LOGDIR/shared ]; then
         do_nodes $list rsync -az "${prefix}.*.${suffix}" $HOSTNAME:$LOGDIR
     fi
@@ -6615,15 +6932,17 @@ recovery_time_min() {
 }
 
 get_clients_mount_count () {
-    local clients=${CLIENTS:-`hostname`}
+	local clients=${CLIENTS:-$HOSTNAME}
 
-    # we need to take into account the clients mounts and
-    # exclude mds/ost mounts if any;
-    do_nodes $clients cat /proc/mounts | grep lustre | grep $MOUNT | wc -l
+	# we need to take into account the clients mounts and
+	# exclude mds/ost mounts if any;
+	do_nodes $clients cat /proc/mounts | grep lustre |
+		grep -w $MOUNT | wc -l
 }
 
 # gss functions
 PROC_CLI="srpc_info"
+PROC_CON="srpc_contexts"
 
 combination()
 {
@@ -6646,28 +6965,39 @@ combination()
 }
 
 calc_connection_cnt() {
-    local dir=$1
+	local dir=$1
 
-    # MDT->MDT = 2 * C(M, 2)
-    # MDT->OST = M * O
-    # CLI->OST = C * O
-    # CLI->MDT = C * M
-    comb_m2=$(combination $MDSCOUNT 2)
+	# MDT->MDT = 2 * C(M, 2)
+	# MDT->OST = M * O
+	# CLI->OST = C * O
+	# CLI->MDT = C * M
+	comb_m2=$(combination $MDSCOUNT 2)
 
-    local num_clients=$(get_clients_mount_count)
+	local num_clients=$(get_clients_mount_count)
 
-    local cnt_mdt2mdt=$((comb_m2 * 2))
-    local cnt_mdt2ost=$((MDSCOUNT * OSTCOUNT))
-    local cnt_cli2ost=$((num_clients * OSTCOUNT))
-    local cnt_cli2mdt=$((num_clients * MDSCOUNT))
-    local cnt_all2ost=$((cnt_mdt2ost + cnt_cli2ost))
-    local cnt_all2mdt=$((cnt_mdt2mdt + cnt_cli2mdt))
-    local cnt_all2all=$((cnt_mdt2ost + cnt_mdt2mdt + cnt_cli2ost + cnt_cli2mdt))
+	local cnt_mdt2mdt=$((comb_m2 * 2))
+	local cnt_mdt2ost=$((MDSCOUNT * OSTCOUNT))
+	local cnt_cli2ost=$((num_clients * OSTCOUNT))
+	local cnt_cli2mdt=$((num_clients * MDSCOUNT))
+	if is_mounted $MOUNT2; then
+		cnt_cli2mdt=$((cnt_cli2mdt * 2))
+		cnt_cli2ost=$((cnt_cli2ost * 2))
+	fi
+	if local_mode; then
+		cnt_mdt2mdt=0
+		cnt_mdt2ost=0
+		cnt_cli2ost=2
+		cnt_cli2mdt=1
+	fi
+	local cnt_all2ost=$((cnt_mdt2ost + cnt_cli2ost))
+	local cnt_all2mdt=$((cnt_mdt2mdt + cnt_cli2mdt))
+	local cnt_all2all=$((cnt_mdt2ost + cnt_mdt2mdt \
+		+ cnt_cli2ost + cnt_cli2mdt))
 
-    local var=cnt_$dir
-    local res=${!var}
+	local var=cnt_$dir
+	local res=${!var}
 
-    echo $res
+	echo $res
 }
 
 set_rule()
@@ -6692,6 +7022,13 @@ set_rule()
     do_facet mgs "$LCTL conf_param $cmd"
 }
 
+count_contexts()
+{
+	local output=$1
+	local total_ctx=$(echo "$output" | grep -c "expire.*key.*hdl")
+	echo $total_ctx
+}
+
 count_flvr()
 {
     local output=$1
@@ -6733,12 +7070,22 @@ flvr_cnt_cli2mdt()
     local flavor=$1
     local cnt
 
-    local clients=${CLIENTS:-`hostname`}
+    local clients=${CLIENTS:-$HOSTNAME}
 
     for c in ${clients//,/ }; do
-        output=`do_node $c lctl get_param -n mdc.*-MDT*-mdc-*.$PROC_CLI 2>/dev/null`
-        tmpcnt=`count_flvr "$output" $flavor`
-        cnt=$((cnt + tmpcnt))
+	local output=$(do_node $c lctl get_param -n \
+		 mdc.*-*-mdc-*.$PROC_CLI 2>/dev/null)
+	local tmpcnt=$(count_flvr "$output" $flavor)
+	if $GSS_SK && [ $flavor != "null" ]; then
+		# tmpcnt=min(contexts,flavors) to ensure SK context is on
+		output=$(do_node $c lctl get_param -n \
+			 mdc.*-MDT*-mdc-*.$PROC_CON 2>/dev/null)
+		local outcon=$(count_contexts "$output")
+		if [ "$outcon" -lt "$tmpcnt" ]; then
+			tmpcnt=$outcon
+		fi
+	fi
+	cnt=$((cnt + tmpcnt))
     done
     echo $cnt
 }
@@ -6748,11 +7095,21 @@ flvr_cnt_cli2ost()
     local flavor=$1
     local cnt
 
-    local clients=${CLIENTS:-`hostname`}
+    local clients=${CLIENTS:-$HOSTNAME}
 
     for c in ${clients//,/ }; do
-        output=`do_node $c lctl get_param -n osc.*OST*-osc-[^M][^D][^T]*.$PROC_CLI 2>/dev/null`
-        tmpcnt=`count_flvr "$output" $flavor`
+	local output=$(do_node $c lctl get_param -n \
+		 osc.*OST*-osc-[^M][^D][^T]*.$PROC_CLI 2>/dev/null)
+	local tmpcnt=$(count_flvr "$output" $flavor)
+	if $GSS_SK && [ $flavor != "null" ]; then
+		# tmpcnt=min(contexts,flavors) to ensure SK context is on
+		output=$(do_node $c lctl get_param -n \
+			 osc.*OST*-osc-[^M][^D][^T]*.$PROC_CON 2>/dev/null)
+		local outcon=$(count_contexts "$output")
+		if [ "$outcon" -lt "$tmpcnt" ]; then
+			tmpcnt=$outcon
+		fi
+	fi
         cnt=$((cnt + tmpcnt))
     done
     echo $cnt
@@ -6769,8 +7126,18 @@ flvr_cnt_mdt2mdt()
     fi
 
     for num in `seq $MDSCOUNT`; do
-        output=`do_facet mds$num lctl get_param -n mdc.*-MDT*-mdc[0-9]*.$PROC_CLI 2>/dev/null`
-        tmpcnt=`count_flvr "$output" $flavor`
+	local output=$(do_facet mds$num lctl get_param -n \
+		osp.*-MDT*osp-MDT*.$PROC_CLI 2>/dev/null)
+	local tmpcnt=$(count_flvr "$output" $flavor)
+	if $GSS_SK && [ $flavor != "null" ]; then
+		# tmpcnt=min(contexts,flavors) to ensure SK context is on
+		output=$(do_facet mds$num lctl get_param -n \
+			osp.*-MDT*osp-MDT*.$PROC_CON 2>/dev/null)
+		local outcon=$(count_contexts "$output")
+		if [ "$outcon" -lt "$tmpcnt" ]; then
+			tmpcnt=$outcon
+		fi
+	fi
         cnt=$((cnt + tmpcnt))
     done
     echo $cnt;
@@ -6785,9 +7152,18 @@ flvr_cnt_mdt2ost()
     for num in `seq $MDSCOUNT`; do
         mdtosc=$(get_mdtosc_proc_path mds$num)
         mdtosc=${mdtosc/-MDT*/-MDT\*}
-        output=$(do_facet mds$num lctl get_param -n \
-            osc.$mdtosc.$PROC_CLI 2>/dev/null)
-        tmpcnt=`count_flvr "$output" $flavor`
+	local output=$(do_facet mds$num lctl get_param -n \
+		 osc.$mdtosc.$PROC_CLI 2>/dev/null)
+	local tmpcnt=$(count_flvr "$output" $flavor)
+	if $GSS_SK && [ $flavor != "null" ]; then
+		# tmpcnt=min(contexts,flavors) to ensure SK context is on
+		output=$(do_facet mds$num lctl get_param -n \
+			 osc.$mdtosc.$PROC_CON 2>/dev/null)
+		local outcon=$(count_contexts "$output")
+		if [ "$outcon" -lt "$tmpcnt" ]; then
+			tmpcnt=$outcon
+		fi
+	fi
         cnt=$((cnt + tmpcnt))
     done
     echo $cnt;
@@ -6797,7 +7173,8 @@ flvr_cnt_mgc2mgs()
 {
     local flavor=$1
 
-    output=`do_facet client lctl get_param -n mgc.*.$PROC_CLI 2>/dev/null`
+    local output=$(do_facet client lctl get_param -n mgc.*.$PROC_CLI \
+			2>/dev/null)
     count_flvr "$output" $flavor
 }
 
@@ -6836,75 +7213,109 @@ do_check_flavor()
 
 wait_flavor()
 {
-    local dir=$1        # from to
-    local flavor=$2     # flavor expected
-    local expect=${3:-$(calc_connection_cnt $dir)}     # number expected
-
-    local res=0
-
-    for ((i=0;i<20;i++)); do
-        echo -n "checking $dir..."
-        res=$(do_check_flavor $dir $flavor)
-        echo "found $res/$expect $flavor connections"
-        [ $res -ge $expect ] && return 0
-        sleep 4
-    done
+	local dir=$1        # from to
+	local flavor=$2     # flavor expected
+	local expect=${3:-$(calc_connection_cnt $dir)} # number expected
+	local WAITFLAVOR_MAX=20 # how many retries before abort?
+
+	local res=0
+	for ((i = 0; i < $WAITFLAVOR_MAX; i++)); do
+		echo -n "checking $dir..."
+		res=$(do_check_flavor $dir $flavor)
+		echo "found $res/$expect $flavor connections"
+		[ $res -ge $expect ] && return 0
+		sleep 4
+	done
 
-    echo "Error checking $flavor of $dir: expect $expect, actual $res"
-    return 1
+	echo "Error checking $flavor of $dir: expect $expect, actual $res"
+#	echo "Dumping additional logs for SK debug.."
+	do_nodes $(comma_list $(all_server_nodes)) "keyctl show"
+	if $dump; then
+		gather_logs $(comma_list $(nodes_list))
+	fi
+	return 1
 }
 
 restore_to_default_flavor()
 {
-    local proc="mgs.MGS.live.$FSNAME"
+	local proc="mgs.MGS.live.$FSNAME"
 
-    echo "restoring to default flavor..."
+	echo "restoring to default flavor..."
 
-    nrule=`do_facet mgs lctl get_param -n $proc 2>/dev/null | grep ".srpc.flavor." | wc -l`
+	local nrule=$(do_facet mgs lctl get_param -n $proc 2>/dev/null |
+		grep ".srpc.flavor" | wc -l)
 
-    # remove all existing rules if any
-    if [ $nrule -ne 0 ]; then
-        echo "$nrule existing rules"
-        for rule in `do_facet mgs lctl get_param -n $proc 2>/dev/null | grep ".srpc.flavor."`; do
-            echo "remove rule: $rule"
-            spec=`echo $rule | awk -F = '{print $1}'`
-            do_facet mgs "$LCTL conf_param -d $spec"
-        done
-    fi
+	# remove all existing rules if any
+	if [ $nrule -ne 0 ]; then
+		echo "$nrule existing rules"
+		for rule in $(do_facet mgs lctl get_param -n $proc 2>/dev/null |
+		    grep ".srpc.flavor."); do
+			echo "remove rule: $rule"
+			spec=`echo $rule | awk -F = '{print $1}'`
+			do_facet mgs "$LCTL conf_param -d $spec"
+		done
+	fi
 
-    # verify no rules left
-    nrule=`do_facet mgs lctl get_param -n $proc 2>/dev/null | grep ".srpc.flavor." | wc -l`
-    [ $nrule -ne 0 ] && error "still $nrule rules left"
+	# verify no rules left
+	nrule=$(do_facet mgs lctl get_param -n $proc 2>/dev/null |
+		grep ".srpc.flavor." | wc -l)
+	[ $nrule -ne 0 ] && error "still $nrule rules left"
 
-    # wait for default flavor to be applied
-    # currently default flavor for all connections are 'null'
-    wait_flavor all2all null
-    echo "now at default flavor settings"
+	# wait for default flavor to be applied
+	if $GSS_SK; then
+		if $SK_S2S; then
+			set_rule $FSNAME any any $SK_FLAVOR
+			wait_flavor all2all $SK_FLAVOR
+		else
+			set_rule $FSNAME any cli2mdt $SK_FLAVOR
+			set_rule $FSNAME any cli2ost $SK_FLAVOR
+			wait_flavor cli2mdt $SK_FLAVOR
+			wait_flavor cli2ost $SK_FLAVOR
+		fi
+		echo "GSS_SK now at default flavor: $SK_FLAVOR"
+	else
+		wait_flavor all2all null
+	fi
 }
 
 set_flavor_all()
 {
-    local flavor=${1:-null}
+	local flavor=${1:-null}
 
-    echo "setting all flavor to $flavor"
+	echo "setting all flavor to $flavor"
 
-    # FIXME need parameter to this fn
-    # and remove global vars
-    local cnt_all2all=$(calc_connection_cnt all2all)
+	# FIXME need parameter to this fn
+	# and remove global vars
+	local cnt_all2all=$(calc_connection_cnt all2all)
 
-    local res=$(do_check_flavor all2all $flavor)
-    if [ $res -eq $cnt_all2all ]; then
-        echo "already have total $res $flavor connections"
-        return
-    fi
+	local res=$(do_check_flavor all2all $flavor)
+	if [ $res -eq $cnt_all2all ]; then
+		echo "already have total $res $flavor connections"
+		return
+	fi
 
-    echo "found $res $flavor out of total $cnt_all2all connections"
-    restore_to_default_flavor
+	echo "found $res $flavor out of total $cnt_all2all connections"
+	restore_to_default_flavor
 
-    [[ $flavor = null ]] && return 0
+	[[ $flavor = null ]] && return 0
 
-    set_rule $FSNAME any any $flavor
-    wait_flavor all2all $flavor
+	if $GSS_SK && [ $flavor != "null" ]; then
+		if $SK_S2S; then
+			set_rule $FSNAME any any $flavor
+			wait_flavor all2all $flavor
+		else
+			set_rule $FSNAME any cli2mdt $flavor
+			set_rule $FSNAME any cli2ost $flavor
+			set_rule $FSNAME any mdt2ost null
+			set_rule $FSNAME any mdt2mdt null
+			wait_flavor cli2mdt $flavor
+			wait_flavor cli2ost $flavor
+		fi
+		echo "GSS_SK now at flavor: $flavor"
+	else
+		set_rule $FSNAME any any $flavor
+		wait_flavor all2all $flavor
+	fi
 }