From 331a0e20dd3b92612f1517d9dd3a242462cf8a92 Mon Sep 17 00:00:00 2001 From: Sebastien Buisson Date: Tue, 23 Jan 2024 14:29:11 +0100 Subject: [PATCH] LU-16307 tests: fix sanity-sec test_31 In order to improve sanity-sec test_31 resiliency, reorganize the way the new LNet '999' is handled. And make sure everything is correctly cleaned up after the test. Lustre-change: https://review.whamcloud.com/53818 Lustre-commit: TBD (from f4a96799159fd662855542d471197ac4060d3295) Test-Parameters: trivial testgroup=review-dne-part-2 Signed-off-by: Sebastien Buisson Change-Id: Idd657c7555e598d0ebc08387eac537b1c73e35bd Reviewed-on: https://review.whamcloud.com/c/ex/lustre-release/+/53779 Reviewed-by: Andreas Dilger Tested-by: jenkins Tested-by: Maloo --- lustre/tests/sanity-sec.sh | 86 +++++++++++++++++++++++++++++++--------------- 1 file changed, 58 insertions(+), 28 deletions(-) diff --git a/lustre/tests/sanity-sec.sh b/lustre/tests/sanity-sec.sh index 2587a7d..147eed7 100755 --- a/lustre/tests/sanity-sec.sh +++ b/lustre/tests/sanity-sec.sh @@ -2366,6 +2366,7 @@ cleanup_31() { KZPOOL=$KEEP_ZPOOL export KEEP_ZPOOL="true" stopall + LOAD_MODULES_REMOTE=true unload_modules do_facet mds1 $TUNEFS --erase-param failover.node $(mdsdevname 1) if [ -n "$failover_mds1" ]; then @@ -2407,6 +2408,20 @@ test_31() { skip "Conflicting test with SSK" fi + # build list of interface on nodes + for node in $(all_nodes); do + infname=inf_$(echo $node | cut -d'.' -f1 | sed s+-+_+g) + itf=$(do_node $node $LNETCTL net show --net $net | + awk 'BEGIN{inf=0} \ + {if (inf==1) print $2; fi; inf=0} /interfaces/{inf=1}') + eval $infname=\$itf + done + + # backup MGSNID + local mgsnid_orig=$MGSNID + # compute new MGSNID + local mgsnid_new=${MGSNID%@*}@$net2 + # save mds failover nids for restore at cleanup failover_mds1=$(do_facet mds1 $TUNEFS --dryrun $(mdsdevname 1)) if [ -n "$failover_mds1" ]; then @@ -2426,6 +2441,7 @@ test_31() { fi # check exports on servers are empty for client + do_facet mgs "lctl get_param *.MGS*.exports.*.export" do_facet mgs "lctl get_param -n *.MGS*.exports.'$nid'.uuid 2>/dev/null | grep -q -" && error "export on MGS should be empty" do_nodes $(comma_list $(mdts_nodes) $(osts_nodes)) \ @@ -2433,19 +2449,22 @@ test_31() { 2>/dev/null | grep -q -" && error "export on servers should be empty" + KZPOOL=$KEEP_ZPOOL + export KEEP_ZPOOL="true" + stopall + LOAD_MODULES_REMOTE=true unload_modules + # add network $net2 on all nodes - do_nodes $(comma_list $(all_nodes)) \ - "$LNETCTL lnet configure && $LNETCTL net add --if \ - \$($LNETCTL net show --net $net | awk 'BEGIN{inf=0} \ - {if (inf==1) print \$2; fi; inf=0} /interfaces/{inf=1}') \ - --net $net2" || - error "unable to configure NID $net2" + for node in $(all_nodes); do + infname=inf_$(echo $node | cut -d'.' -f1 | sed s+-+_+g) + do_node $node "modprobe lnet ; modprobe ptlrpc ; \ + $LNETCTL lnet configure && + $LNETCTL net add --if ${!infname} --net $net2" || + error "unable to configure NID $net2" + done # necessary to do writeconf in order to register # new @$net2 nid for targets - KZPOOL=$KEEP_ZPOOL - export KEEP_ZPOOL="true" - stopall export SK_MOUNTED=false writeconf_all @@ -2456,18 +2475,9 @@ test_31() { setupall server_only || echo 1 export KEEP_ZPOOL="$KZPOOL" - # backup MGSNID - local mgsnid_orig=$MGSNID - # compute new MGSNID - MGSNID=$(do_facet mgs "$LCTL list_nids | grep $net2") - - # on client, turn LNet Dynamic Discovery on - lnetctl set discovery 1 - - # mount client with -o network=$net2 option: - # should fail because of LNet Dynamic Discovery - mount_client $MOUNT ${MOUNT_OPTS},network=$net2 && - error "client mount with '-o network' option should be refused" + # update MGSNID + MGSNID=$mgsnid_new + stack_trap "MGSNID=$mgsnid_orig" EXIT # on client, reconfigure LNet and turn LNet Dynamic Discovery off $LNETCTL net del --net $net2 && lnetctl lnet unconfigure @@ -2475,20 +2485,18 @@ test_31() { modprobe lnet lnetctl set discovery 0 modprobe ptlrpc - $LNETCTL lnet configure && $LNETCTL net add --if \ - $($LNETCTL net show --net $net | awk 'BEGIN{inf=0} \ - {if (inf==1) print $2; fi; inf=0} /interfaces/{inf=1}') \ - --net $net2 || + sleep 10 + infname=inf_$(echo $(hostname -s) | sed s+-+_+g) + $LNETCTL lnet configure + $LNETCTL net add --if ${!infname} --net $net2 || error "unable to configure NID $net2 on client" # mount client with -o network=$net2 option mount_client $MOUNT ${MOUNT_OPTS},network=$net2 || error "unable to remount client" - # restore MGSNID - MGSNID=$mgsnid_orig - # check export on MGS + do_facet mgs "lctl get_param *.MGS*.exports.*.export" do_facet mgs "lctl get_param -n *.MGS*.exports.'$nid'.uuid 2>/dev/null | grep -" [ $? -ne 0 ] || error "export for $nid on MGS should not exist" @@ -2518,6 +2526,28 @@ test_31() { lctl get_param mdc.${FSNAME}-*.import | grep failover_nids | grep ${addr2}@$net2 || error "MDC import should have failnid ${addr2}@$net2" + + # unmount client + zconf_umount $HOSTNAME $MOUNT || error "unable to umount client" + + # on client, reconfigure LNet and turn LNet Dynamic Discovery on + $LNETCTL net del --net $net2 && lnetctl lnet unconfigure + lustre_rmmod + modprobe lnet + lnetctl set discovery 1 + modprobe ptlrpc + sleep 10 + infname=inf_$(echo $(hostname -s) | sed s+-+_+g) + $LNETCTL lnet configure + $LNETCTL net add --if ${!infname} --net $net2 || + error "unable to configure NID $net2 on client" + + # mount client with -o network=$net2 option: + # should fail because of LNet Dynamic Discovery + mount_client $MOUNT ${MOUNT_OPTS},network=$net2 && + error "client mount with '-o network' option should be refused" + + echo } run_test 31 "client mount option '-o network'" -- 1.8.3.1