LU-16307 tests: fix sanity-sec test_31

In order to improve sanity-sec test_31 resiliency, reorganize the way
the new LNet '999' is handled. And make sure everything is correctly
cleaned up after the test.

Test-Parameters: trivial
Test-Parameters: testgroup=review-dne-part-2
Signed-off-by: Sebastien Buisson <sbuisson@ddn.com>
Change-Id: Idd657c7555e598d0ebc08387eac537b1c73e35bd

diff --git a/lustre/tests/sanity-sec.sh b/lustre/tests/sanity-sec.sh
index 4f1b8ba..81701a9 100755 (executable)
--- a/lustre/tests/sanity-sec.sh
+++ b/lustre/tests/sanity-sec.sh
@@ -2369,6 +2369,7 @@ cleanup_31() {
        KZPOOL=$KEEP_ZPOOL
        export KEEP_ZPOOL="true"
        stopall
+       LOAD_MODULES_REMOTE=true unload_modules
 
        do_facet mds1 $TUNEFS --erase-param failover.node $(mdsdevname 1)
        if [ -n "$failover_mds1" ]; then
@@ -2410,6 +2411,20 @@ test_31() {
                skip "Conflicting test with SSK"
        fi
 
+       # build list of interface on nodes
+       for node in $(all_nodes); do
+               infname=inf_$(echo $node | cut -d'.' -f1 | sed s+-+_+g)
+               itf=$(do_node $node $LNETCTL net show --net $net |
+                     awk 'BEGIN{inf=0} \
+                     {if (inf==1) print $2; fi; inf=0} /interfaces/{inf=1}')
+               eval $infname=\$itf
+       done
+
+       # backup MGSNID
+       local mgsnid_orig=$MGSNID
+       # compute new MGSNID
+       local mgsnid_new=${MGSNID%@*}@$net2
+
        # save mds failover nids for restore at cleanup
        failover_mds1=$(do_facet mds1 $TUNEFS --dryrun $(mdsdevname 1))
        if [ -n "$failover_mds1" ]; then
@@ -2429,6 +2444,7 @@ test_31() {
        fi
 
        # check exports on servers are empty for client
+       do_facet mgs "lctl get_param *.MGS*.exports.*.export"
        do_facet mgs "lctl get_param -n *.MGS*.exports.'$nid'.uuid 2>/dev/null |
                      grep -q -" && error "export on MGS should be empty"
        do_nodes $(comma_list $(mdts_nodes) $(osts_nodes)) \
@@ -2436,19 +2452,22 @@ test_31() {
                  2>/dev/null | grep -q -" &&
                error "export on servers should be empty"
 
+       KZPOOL=$KEEP_ZPOOL
+       export KEEP_ZPOOL="true"
+       stopall
+       LOAD_MODULES_REMOTE=true unload_modules
+
        # add network $net2 on all nodes
-       do_nodes $(comma_list $(all_nodes)) \
-                "$LNETCTL lnet configure && $LNETCTL net add --if \
-                 \$($LNETCTL net show --net $net | awk 'BEGIN{inf=0} \
-                 {if (inf==1) print \$2; fi; inf=0} /interfaces/{inf=1}') \
-                 --net $net2" ||
-               error "unable to configure NID $net2"
+       for node in $(all_nodes); do
+               infname=inf_$(echo $node | cut -d'.' -f1 | sed s+-+_+g)
+               do_node $node "modprobe lnet ; modprobe ptlrpc ; \
+                              $LNETCTL lnet configure &&
+                              $LNETCTL net add --if ${!infname} --net $net2" ||
+                       error "unable to configure NID $net2"
+       done
 
        # necessary to do writeconf in order to register
        # new @$net2 nid for targets
-       KZPOOL=$KEEP_ZPOOL
-       export KEEP_ZPOOL="true"
-       stopall
        export SK_MOUNTED=false
        writeconf_all
 
@@ -2459,18 +2478,9 @@ test_31() {
        setupall server_only || echo 1
        export KEEP_ZPOOL="$KZPOOL"
 
-       # backup MGSNID
-       local mgsnid_orig=$MGSNID
-       # compute new MGSNID
-       MGSNID=$(do_facet mgs "$LCTL list_nids | grep $net2")
-
-       # on client, turn LNet Dynamic Discovery on
-       lnetctl set discovery 1
-
-       # mount client with -o network=$net2 option:
-       # should fail because of LNet Dynamic Discovery
-       mount_client $MOUNT ${MOUNT_OPTS},network=$net2 &&
-               error "client mount with '-o network' option should be refused"
+       # update MGSNID
+       MGSNID=$mgsnid_new
+       stack_trap "MGSNID=$mgsnid_orig" EXIT
 
        # on client, reconfigure LNet and turn LNet Dynamic Discovery off
        $LNETCTL net del --net $net2 && lnetctl lnet unconfigure
@@ -2478,20 +2488,18 @@ test_31() {
        modprobe lnet
        lnetctl set discovery 0
        modprobe ptlrpc
-       $LNETCTL lnet configure && $LNETCTL net add --if \
-         $($LNETCTL net show --net $net | awk 'BEGIN{inf=0} \
-         {if (inf==1) print $2; fi; inf=0} /interfaces/{inf=1}') \
-         --net $net2 ||
+       sleep 10
+       infname=inf_$(echo $(hostname -s) | sed s+-+_+g)
+       $LNETCTL lnet configure
+       $LNETCTL net add --if ${!infname} --net $net2 ||
                error "unable to configure NID $net2 on client"
 
        # mount client with -o network=$net2 option
        mount_client $MOUNT ${MOUNT_OPTS},network=$net2 ||
                error "unable to remount client"
 
-       # restore MGSNID
-       MGSNID=$mgsnid_orig
-
        # check export on MGS
+       do_facet mgs "lctl get_param *.MGS*.exports.*.export"
        do_facet mgs "lctl get_param -n *.MGS*.exports.'$nid'.uuid 2>/dev/null |
                      grep -"
        [ $? -ne 0 ] || error "export for $nid on MGS should not exist"
@@ -2521,6 +2529,28 @@ test_31() {
        lctl get_param mdc.${FSNAME}-*.import | grep failover_nids |
            grep ${addr2}@$net2 ||
                error "MDC import should have failnid ${addr2}@$net2"
+
+       # unmount client
+       zconf_umount $HOSTNAME $MOUNT || error "unable to umount client"
+
+       # on client, reconfigure LNet and turn LNet Dynamic Discovery on
+       $LNETCTL net del --net $net2 && lnetctl lnet unconfigure
+       lustre_rmmod
+       modprobe lnet
+       lnetctl set discovery 1
+       modprobe ptlrpc
+       sleep 10
+       infname=inf_$(echo $(hostname -s) | sed s+-+_+g)
+       $LNETCTL lnet configure
+       $LNETCTL net add --if ${!infname} --net $net2 ||
+               error "unable to configure NID $net2 on client"
+
+       # mount client with -o network=$net2 option:
+       # should fail because of LNet Dynamic Discovery
+       mount_client $MOUNT ${MOUNT_OPTS},network=$net2 &&
+               error "client mount with '-o network' option should be refused"
+
+       echo
 }
 run_test 31 "client mount option '-o network'"