From 0543381b2f0ea6e2980315765ad34ae37411d36a Mon Sep 17 00:00:00 2001 From: Mikhail Pershin Date: Mon, 13 Feb 2023 12:07:45 +0300 Subject: [PATCH] LU-16557 client: -o network needs add_conn processing Mount option -o network restricts client import to use only selected network. It processes connection UUID/NIDs during 'setup' config command handling but skips any 'add_conn' command if its UUID has no mention about that network. Meahwhile connection UUID is just a name and may have many NIDs configured including those on restricted network which are skipped as well. Therefore client import configuration misses failover NIDs on restricted network. Patch makes import to save restricted network information after 'setup' command processing, so it is applied to any client_import_add_conn() call. The 'add_conn' command is always processed now and its NIDs will be filtered in the same way as for 'setup'. Test 31 in sanity-sec.sh is extended to check imports failover_nids has all and only NIDs on restricted network Lustre-change: https://review.whamcloud.com/49986 Lustre-commit: c508c9426838f16256223ab0bbd648bfbec25e46 Test-Parameters: env=ONLY=31 testlist=sanity-sec Signed-off-by: Mikhail Pershin Change-Id: Id70ebd836f061f154e3779b07b52f1baea9a1776 Reviewed-by: Sebastien Buisson Reviewed-by: Andreas Dilger Reviewed-by: Cyril Bordage Reviewed-on: https://review.whamcloud.com/c/fs/lustre-release/+/50187 Tested-by: jenkins Tested-by: Maloo Reviewed-by: Oleg Drokin --- lustre/include/lustre_import.h | 1 + lustre/ldlm/ldlm_lib.c | 20 ++++++++------ lustre/obdclass/obd_config.c | 21 --------------- lustre/tests/sanity-sec.sh | 59 ++++++++++++++++++++++++++++-------------- 4 files changed, 52 insertions(+), 49 deletions(-) diff --git a/lustre/include/lustre_import.h b/lustre/include/lustre_import.h index 24f2c16..aab8919 100644 --- a/lustre/include/lustre_import.h +++ b/lustre/include/lustre_import.h @@ -336,6 +336,7 @@ struct obd_import { /* adaptive timeout data */ struct imp_at imp_at; time64_t imp_last_reply_time; /* for health check */ + __u32 imp_conn_restricted_net; }; /* import.c : adaptive timeout handling. diff --git a/lustre/ldlm/ldlm_lib.c b/lustre/ldlm/ldlm_lib.c index 855269c..bf61555 100644 --- a/lustre/ldlm/ldlm_lib.c +++ b/lustre/ldlm/ldlm_lib.c @@ -61,6 +61,7 @@ static int import_set_conn(struct obd_import *imp, struct obd_uuid *uuid, struct ptlrpc_connection *ptlrpc_conn; struct obd_import_conn *imp_conn = NULL, *item; lnet_nid_t nid4refnet = LNET_NID_ANY; + u32 refnet = imp->imp_conn_restricted_net; int rc = 0; ENTRY; @@ -70,10 +71,14 @@ static int import_set_conn(struct obd_import *imp, struct obd_uuid *uuid, RETURN(-EINVAL); } - if (imp->imp_connection && - imp->imp_connection->c_remote_uuid.uuid[0] == 0) - /* nid4refnet is used to restrict network connections */ - nid4refnet = lnet_nid_to_nid4(&imp->imp_connection->c_self); + /* refnet is used to restrict network connections */ + if (refnet != LNET_NIDNET(LNET_NID_ANY)) { + CDEBUG(D_HA, "imp %s: restrict %s to %s net\n", + imp->imp_obd->obd_name, uuid->uuid, + libcfs_net2str(refnet)); + nid4refnet = LNET_MKNID(refnet, 0); + } + ptlrpc_conn = ptlrpc_uuid_to_connection(uuid, nid4refnet); if (!ptlrpc_conn) { CDEBUG(D_HA, "can't find connection %s\n", uuid->uuid); @@ -326,8 +331,6 @@ int client_obd_setup(struct obd_device *obd, struct lustre_cfg *lcfg) const char *name = obd->obd_type->typ_name; enum ldlm_ns_type ns_type = LDLM_NS_TYPE_UNKNOWN; char *cli_name = lustre_cfg_buf(lcfg, 0); - struct ptlrpc_connection fake_conn = { .c_self = {}, - .c_remote_uuid.uuid[0] = 0 }; int rc; ENTRY; @@ -550,8 +553,9 @@ int client_obd_setup(struct obd_device *obd, struct lustre_cfg *lcfg) rc); GOTO(err_import, rc); } - lnet_nid4_to_nid(LNET_MKNID(refnet, 0), &fake_conn.c_self); - imp->imp_connection = &fake_conn; + imp->imp_conn_restricted_net = refnet; + } else { + imp->imp_conn_restricted_net = LNET_NIDNET(LNET_NID_ANY); } rc = client_import_add_conn(imp, &server_uuid, 1); diff --git a/lustre/obdclass/obd_config.c b/lustre/obdclass/obd_config.c index 183d719..5773926 100644 --- a/lustre/obdclass/obd_config.c +++ b/lustre/obdclass/obd_config.c @@ -1953,27 +1953,6 @@ int class_config_llog_handler(const struct lu_env *env, } } - /* - * Skip add_conn command if uuid is - * not on restricted net - */ - if (cfg && cfg->cfg_sb && s2lsi(cfg->cfg_sb) && - !IS_SERVER(s2lsi(cfg->cfg_sb))) { - struct lustre_sb_info *lsi = s2lsi(cfg->cfg_sb); - char *uuid_str = lustre_cfg_string(lcfg, 1); - - if (lcfg->lcfg_command == LCFG_ADD_CONN && - lsi->lsi_lmd->lmd_nidnet && - LNET_NIDNET(libcfs_str2nid(uuid_str)) != - libcfs_str2net(lsi->lsi_lmd->lmd_nidnet)) { - CDEBUG(D_CONFIG, "skipping add_conn for %s\n", - uuid_str); - rc = 0; - /* No processing! */ - break; - } - } - OBD_ALLOC(lcfg_new, lustre_cfg_len(bufs.lcfg_bufcount, bufs.lcfg_buflen)); if (!lcfg_new) diff --git a/lustre/tests/sanity-sec.sh b/lustre/tests/sanity-sec.sh index d996661..7d4dabc 100755 --- a/lustre/tests/sanity-sec.sh +++ b/lustre/tests/sanity-sec.sh @@ -2310,6 +2310,10 @@ test_31() { local nid=$(lctl list_nids | grep ${NETTYPE} | head -n1) local addr=${nid%@*} local net=${nid#*@} + local net2=${NETTYPE}999 + local mdsnid=$(do_facet mds1 $LCTL list_nids | head -1) + local addr1=${mdsnid%@*} + local addr2=${addr1%.*}.$(((${addr1##*.} + 11) % 256)) export LNETCTL=$(which lnetctl 2> /dev/null) @@ -2334,39 +2338,44 @@ test_31() { 2>/dev/null | grep -q -" && error "export on servers should be empty" - # add network ${NETTYPE}999 on all nodes + # add network $net2 on all nodes do_nodes $(comma_list $(all_nodes)) \ "$LNETCTL lnet configure && $LNETCTL net add --if \ \$($LNETCTL net show --net $net | awk 'BEGIN{inf=0} \ {if (inf==1) print \$2; fi; inf=0} /interfaces/{inf=1}') \ - --net ${NETTYPE}999" || - error "unable to configure NID ${NETTYPE}999" + --net $net2" || + error "unable to configure NID $net2" # necessary to do writeconf in order to register - # new @${NETTYPE}999 nid for targets + # new @$net2 nid for targets KZPOOL=$KEEP_ZPOOL export KEEP_ZPOOL="true" stopall export SK_MOUNTED=false writeconf_all + + nids="${addr1}@$net,${addr1}@$net2:${addr2}@$net,${addr2}@$net2" + do_facet mds1 "$TUNEFS --servicenode="$nids" $(mdsdevname 1)" || + error "tunefs failed" + setupall server_only || echo 1 export KEEP_ZPOOL="$KZPOOL" # backup MGSNID local mgsnid_orig=$MGSNID # compute new MGSNID - MGSNID=$(do_facet mgs "$LCTL list_nids | grep ${NETTYPE}999") + MGSNID=$(do_facet mgs "$LCTL list_nids | grep $net2") # on client, turn LNet Dynamic Discovery on lnetctl set discovery 1 - # mount client with -o network=${NETTYPE}999 option: + # mount client with -o network=$net2 option: # should fail because of LNet Dynamic Discovery - mount_client $MOUNT ${MOUNT_OPTS},network=${NETTYPE}999 && + mount_client $MOUNT ${MOUNT_OPTS},network=$net2 && error "client mount with '-o network' option should be refused" # on client, reconfigure LNet and turn LNet Dynamic Discovery off - $LNETCTL net del --net ${NETTYPE}999 && lnetctl lnet unconfigure + $LNETCTL net del --net $net2 && lnetctl lnet unconfigure lustre_rmmod modprobe lnet lnetctl set discovery 0 @@ -2374,11 +2383,11 @@ test_31() { $LNETCTL lnet configure && $LNETCTL net add --if \ $($LNETCTL net show --net $net | awk 'BEGIN{inf=0} \ {if (inf==1) print $2; fi; inf=0} /interfaces/{inf=1}') \ - --net ${NETTYPE}999 || - error "unable to configure NID ${NETTYPE}999 on client" + --net $net2 || + error "unable to configure NID $net2 on client" - # mount client with -o network=${NETTYPE}999 option - mount_client $MOUNT ${MOUNT_OPTS},network=${NETTYPE}999 || + # mount client with -o network=$net2 option + mount_client $MOUNT ${MOUNT_OPTS},network=$net2 || error "unable to remount client" # restore MGSNID @@ -2386,24 +2395,34 @@ test_31() { # check export on MGS do_facet mgs "lctl get_param -n *.MGS*.exports.'$nid'.uuid 2>/dev/null | - grep -q -" + grep -" [ $? -ne 0 ] || error "export for $nid on MGS should not exist" do_facet mgs \ - "lctl get_param -n *.MGS*.exports.'${addr}@${NETTYPE}999'.uuid \ - 2>/dev/null | grep -q -" + "lctl get_param -n *.MGS*.exports.'${addr}@$net2'.uuid \ + 2>/dev/null | grep -" [ $? -eq 0 ] || - error "export for ${addr}@${NETTYPE}999 on MGS should exist" + error "export for ${addr}@$net2 on MGS should exist" # check {mdc,osc} imports lctl get_param mdc.${FSNAME}-*.import | grep current_connection | - grep -q ${NETTYPE}999 + grep $net2 [ $? -eq 0 ] || - error "import for mdc should use ${addr}@${NETTYPE}999" + error "import for mdc should use ${addr1}@$net2" lctl get_param osc.${FSNAME}-*.import | grep current_connection | - grep -q ${NETTYPE}999 + grep $net2 [ $? -eq 0 ] || - error "import for osc should use ${addr}@${NETTYPE}999" + error "import for osc should use ${addr1}@$net2" + + # no NIDs on other networks should be listed + lctl get_param mdc.${FSNAME}-*.import | grep failover_nids | + grep -w ".*@$net" && + error "MDC import shouldn't have failnids at @$net" + + # failover NIDs on net999 should be listed + lctl get_param mdc.${FSNAME}-*.import | grep failover_nids | + grep ${addr2}@$net2 || + error "MDC import should have failnid ${addr2}@$net2" } run_test 31 "client mount option '-o network'" -- 1.8.3.1