LU-12131 tests: properly handle GSS in server failover

In case of server failover, a number of aspects must be handled when
GSS based features (SSK or Kerberos) are activated:
- lsvcgssd daemon must be restarted;
- targets must be mounted with proper skpath option;
- permissions on keys must be adjusted.
When service is initially started, all that is managed in setupall().
fail() and facet_failover() have to be improved to take GSS aspects
into account.

Test-Parameters: envdefinitions=SHARED_KEY=true testlist=sanity,recovery-small,sanity-sec
Signed-off-by: Sebastien Buisson <sbuisson@ddn.com>
Change-Id: I8db686f406629c7eec655496cf83c0539c1bfb33
Reviewed-on: https://review.whamcloud.com/35041
Tested-by: jenkins <devops@whamcloud.com>
Tested-by: Maloo <maloo@whamcloud.com>
Reviewed-by: Andreas Dilger <adilger@whamcloud.com>
Reviewed-by: Wei Liu <sarah@whamcloud.com>
Reviewed-by: Oleg Drokin <green@whamcloud.com>

diff --git a/lustre/tests/test-framework.sh b/lustre/tests/test-framework.sh
index 858d222..c90f6db 100755 (executable)
--- a/lustre/tests/test-framework.sh
+++ b/lustre/tests/test-framework.sh
@@ -3517,6 +3517,10 @@ facet_failover() {
                change_active ${affecteds[index]}
 
                wait_for_facet ${affecteds[index]}
+               if $GSS_SK; then
+                       init_gss
+                       init_facets_vars_simple
+               fi
                # start mgs first if it is affected
                if ! combined_mgs_mds &&
                        list_member ${affecteds[index]} mgs; then
@@ -3526,6 +3530,12 @@ facet_failover() {
                affected=$(exclude_items_from_list ${affecteds[index]} mgs)
                echo mount facets: ${affecteds[index]}
                mount_facets ${affecteds[index]}
+               if $GSS_SK; then
+                       do_nodes $(comma_list $(all_nodes)) \
+                               "keyctl show | grep lustre | cut -c1-11 |
+                               sed -e 's/ //g;' |
+                               xargs -IX keyctl setperm X 0x3f3f3f3f"
+               fi
        done
 }
 
@@ -3619,7 +3629,12 @@ fail() {
        local facets=$1
        local clients=${CLIENTS:-$HOSTNAME}
 
+       SK_NO_KEY_save=$SK_NO_KEY
+       if $GSS_SK; then
+               export SK_NO_KEY=false
+       fi
        facet_failover $* || error "failover: $?"
+       export SK_NO_KEY=$SK_NO_KEY_save
        # to initiate all OSC idling connections
        clients_up
        wait_clients_import_state "$clients" "$facets" "\(FULL\|IDLE\)"
@@ -4984,6 +4999,31 @@ init_facets_vars () {
        fi
 }
 
+init_facets_vars_simple () {
+       local devname
+
+       if ! remote_mds_nodsh; then
+               for num in $(seq $MDSCOUNT); do
+                       devname=$(mdsdevname $num)
+                       eval export mds${num}_dev=${devname}
+                       eval export mds${num}_opt=\"${MDS_MOUNT_OPTS}\"
+               done
+       fi
+
+       if ! combined_mgs_mds ; then
+               eval export mgs_dev=$(mgsdevname)
+               eval export mgs_opt=\"${MGS_MOUNT_OPTS}\"
+       fi
+
+       if ! remote_ost_nodsh; then
+               for num in $(seq $OSTCOUNT); do
+                       devname=$(ostdevname $num)
+                       eval export ost${num}_dev=${devname}
+                       eval export ost${num}_opt=\"${OST_MOUNT_OPTS}\"
+               done
+       fi
+}
+
 osc_ensure_active () {
     local facet=$1
     local timeout=$2