LU-16307 tests: fix sanity-sec test_31 better

Patch "LU-16307 tests: fix sanity-sec test_31" was landed to this
branch before the master version, which got further improvements.
In particular, the test has been updated to handle IPv6 and numeric
NIDs, and it has been tweaked to run out of tree.

Lustre-change: https://review.whamcloud.com/53818
Lustre-commit: 28b4d02161c38e624efb10d4815856cb9df3dc07

Fixes: 331a0e20dd ("LU-16307 tests: fix sanity-sec test_31")
Test-Parameters: trivial
Test-Parameters: testgroup=review-dne-part-2
Signed-off-by: Sebastien Buisson <sbuisson@ddn.com>
Change-Id: Idd657c7555e598d0ebc08387eac537b1c73e35be
Reviewed-by: Jian Yu <yujian@whamcloud.com>
Reviewed-on: https://review.whamcloud.com/c/ex/lustre-release/+/54279
Tested-by: jenkins <devops@whamcloud.com>
Tested-by: Maloo <maloo@whamcloud.com>
Reviewed-by: Andreas Dilger <adilger@whamcloud.com>

diff --git a/lustre/tests/sanity-sec.sh b/lustre/tests/sanity-sec.sh
index 71320e1..54490e1 100755 (executable)
--- a/lustre/tests/sanity-sec.sh
+++ b/lustre/tests/sanity-sec.sh
@@ -2445,17 +2445,13 @@ cleanup_31() {
        # unmount client
        zconf_umount $HOSTNAME $MOUNT || error "unable to umount client"
 
-       # remove ${NETTYPE}999 network on all nodes
-       do_nodes $(comma_list $(all_nodes)) \
-                "$LNETCTL net del --net ${NETTYPE}999 && \
-                 $LNETCTL lnet unconfigure 2>/dev/null || true"
-
        # necessary to do writeconf in order to de-register
        # @${NETTYPE}999 nid for targets
        KZPOOL=$KEEP_ZPOOL
        export KEEP_ZPOOL="true"
        stopall
        LOAD_MODULES_REMOTE=true unload_modules
+       LOAD_MODULES_REMOTE=true load_modules
 
        do_facet mds1 $TUNEFS --erase-param failover.node $(mdsdevname 1)
        if [ -n "$failover_mds1" ]; then
@@ -2485,8 +2481,7 @@ test_31() {
        local net2=${NETTYPE}999
        local mdsnid=$(do_facet mds1 $LCTL list_nids | head -1)
        local addr1=${mdsnid%@*}
-       local addr2=${addr1%.*}.$(((${addr1##*.} + 11) % 256))
-       local failover_mds1
+       local addr2 failover_mds1
 
        export LNETCTL=$(which lnetctl 2> /dev/null)
 
@@ -2497,12 +2492,22 @@ test_31() {
                skip "Conflicting test with SSK"
        fi
 
+       if [[ $addr1 =~ ^([0-9a-f]{0,4}:){2,7}[0-9a-f]{0,4}$ ]]; then
+               local tmp=$(printf "%x" $(((0x${addr1##*:} + 11) % 65536)))
+
+               addr2=${addr1%:*}:${tmp}
+       elif [[ $addr1 =~ ^([0-9]{1,3}\.){3,3}[0-9]{1,3}$ ]]; then
+               addr2=${addr1%.*}.$(((${addr1##*.} + 11) % 256))
+       elif [[ $addr1 =~ ^[0-9]+$ ]]; then
+               addr2=$((addr1 + 11))
+       fi
+
        # build list of interface on nodes
        for node in $(all_nodes); do
                infname=inf_$(echo $node | cut -d'.' -f1 | sed s+-+_+g)
                itf=$(do_node $node $LNETCTL net show --net $net |
                      awk 'BEGIN{inf=0} \
-                     {if (inf==1) print $2; fi; inf=0} /interfaces/{inf=1}')
+                     {if (inf==1) { print $2; exit; } fi} /interfaces/{inf=1}')
                eval $infname=\$itf
        done
 
@@ -2540,28 +2545,32 @@ test_31() {
 
        KZPOOL=$KEEP_ZPOOL
        export KEEP_ZPOOL="true"
-       stopall
-       LOAD_MODULES_REMOTE=true unload_modules
+       stopall || error "stopall failed"
+       LOAD_MODULES_REMOTE=true unload_modules ||
+               error "Failed to unload modules"
 
        # add network $net2 on all nodes
+       do_rpc_nodes $(comma_list $(all_nodes)) load_modules ||
+               error "unable to load modules on $(all_nodes)"
        for node in $(all_nodes); do
+               do_node $node "$LNETCTL lnet configure"
                infname=inf_$(echo $node | cut -d'.' -f1 | sed s+-+_+g)
-               do_node $node "modprobe lnet ; modprobe ptlrpc ; \
-                              $LNETCTL lnet configure &&
-                              $LNETCTL net add --if ${!infname} --net $net2" ||
-                       error "unable to configure NID $net2"
+               do_node $node "$LNETCTL net add --if ${!infname} --net $net2" ||
+                       error "unable to configure NID on $net2 for node $node"
        done
 
+       LOAD_MODULES_REMOTE=true load_modules || error "failed to load modules"
+
        # necessary to do writeconf in order to register
        # new @$net2 nid for targets
        export SK_MOUNTED=false
-       writeconf_all
+       writeconf_all || error "writeconf failed"
 
        nids="${addr1}@$net,${addr1}@$net2:${addr2}@$net,${addr2}@$net2"
        do_facet mds1 "$TUNEFS --servicenode="$nids" $(mdsdevname 1)" ||
                error "tunefs failed"
 
-       setupall server_only || echo 1
+       setupall server_only || error "setupall failed"
        export KEEP_ZPOOL="$KZPOOL"
 
        # update MGSNID
@@ -2569,16 +2578,14 @@ test_31() {
        stack_trap "MGSNID=$mgsnid_orig" EXIT
 
        # on client, reconfigure LNet and turn LNet Dynamic Discovery off
-       $LNETCTL net del --net $net2 && lnetctl lnet unconfigure
-       lustre_rmmod
-       modprobe lnet
-       lnetctl set discovery 0
-       modprobe ptlrpc
-       sleep 10
+       $LUSTRE_RMMOD || error "$LUSTRE_RMMOD failed (1)"
+       load_modules || error "Failed to load modules"
+       $LNETCTL set discovery 0 || error "Failed to disable discovery"
+       $LNETCTL lnet configure ||
+               error "unable to configure lnet on client"
        infname=inf_$(echo $(hostname -s) | sed s+-+_+g)
-       $LNETCTL lnet configure
        $LNETCTL net add --if ${!infname} --net $net2 ||
-               error "unable to configure NID $net2 on client"
+               error "unable to configure NID on $net2 on client (1)"
 
        # mount client with -o network=$net2 option
        mount_client $MOUNT ${MOUNT_OPTS},network=$net2 ||
@@ -2619,17 +2626,14 @@ test_31() {
        # unmount client
        zconf_umount $HOSTNAME $MOUNT || error "unable to umount client"
 
-       # on client, reconfigure LNet and turn LNet Dynamic Discovery on
-       $LNETCTL net del --net $net2 && lnetctl lnet unconfigure
-       lustre_rmmod
-       modprobe lnet
-       lnetctl set discovery 1
-       modprobe ptlrpc
-       sleep 10
+       # on client, configure LNet and turn LNet Dynamic Discovery on (default)
+       $LUSTRE_RMMOD || error "$LUSTRE_RMMOD failed (2)"
+       load_modules || error "Failed to load modules"
+       $LNETCTL lnet configure ||
+               error "unable to configure lnet on client"
        infname=inf_$(echo $(hostname -s) | sed s+-+_+g)
-       $LNETCTL lnet configure
        $LNETCTL net add --if ${!infname} --net $net2 ||
-               error "unable to configure NID $net2 on client"
+               error "unable to configure NID on $net2 on client (2)"
 
        # mount client with -o network=$net2 option:
        # should fail because of LNet Dynamic Discovery