From cf9492e522ecdc6688ad267338725f025fe89a1a Mon Sep 17 00:00:00 2001 From: Sebastien Buisson Date: Wed, 15 Jan 2025 18:23:46 +0100 Subject: [PATCH] LU-18637 obd: add 'network' server mount option Just like the '-o network' client option, we want to control which LNet network is used for connections established by servers to remote peers (MGS, other servers). This controls the following connections: - osp - lwp - mgc This patch leverages the 'imp_conn_restricted_net' field on the 'struct obd_import'. Enhance sanity-sec test_31 to exercise this '-o network' option on server side. Signed-off-by: Sebastien Buisson Change-Id: I39cbab2645aeda6fe54efb377d1b1d9edfceb89f Reviewed-on: https://review.whamcloud.com/c/fs/lustre-release/+/57780 Tested-by: jenkins Tested-by: Maloo Reviewed-by: Cyril Bordage Reviewed-by: Oleg Drokin Reviewed-by: Mikhail Pershin Reviewed-by: Serguei Smirnov --- lustre/ldlm/ldlm_lib.c | 9 +++--- lustre/obdclass/obd_config.c | 9 +++--- lustre/obdclass/obd_mount.c | 13 +-------- lustre/osp/lwp_dev.c | 48 +++++++++++++++++++++++++------- lustre/ptlrpc/client.c | 4 +-- lustre/ptlrpc/events.c | 5 +++- lustre/target/tgt_mount.c | 27 +++++++++++++++--- lustre/tests/sanity-sec.sh | 66 ++++++++++++++++++++++++++++++++++++++++---- 8 files changed, 137 insertions(+), 44 deletions(-) diff --git a/lustre/ldlm/ldlm_lib.c b/lustre/ldlm/ldlm_lib.c index dde85be..119859a 100644 --- a/lustre/ldlm/ldlm_lib.c +++ b/lustre/ldlm/ldlm_lib.c @@ -60,9 +60,10 @@ static int import_set_conn(struct obd_import *imp, struct obd_uuid *uuid, libcfs_net2str(refnet)); ptlrpc_conn = ptlrpc_uuid_to_connection(uuid, refnet); - if (!ptlrpc_conn) { - CDEBUG(D_HA, "can't find connection %s\n", uuid->uuid); - RETURN(-ENOENT); + if (IS_ERR(ptlrpc_conn)) { + CDEBUG(D_HA, "can't find connection %s: rc=%ld\n", + uuid->uuid, PTR_ERR(ptlrpc_conn)); + RETURN(PTR_ERR(ptlrpc_conn)); } if (create) { @@ -132,7 +133,7 @@ int client_import_dyn_add_conn(struct obd_import *imp, struct obd_uuid *uuid, int rc; ptlrpc_conn = ptlrpc_uuid_to_connection(uuid, LNET_NID_NET(prim_nid)); - if (!ptlrpc_conn) { + if (IS_ERR(ptlrpc_conn)) { const char *str_uuid = obd_uuid2str(uuid); rc = class_add_uuid(str_uuid, prim_nid); diff --git a/lustre/obdclass/obd_config.c b/lustre/obdclass/obd_config.c index ee4612d..40aea75 100644 --- a/lustre/obdclass/obd_config.c +++ b/lustre/obdclass/obd_config.c @@ -1933,16 +1933,15 @@ int class_config_llog_handler(const struct lu_env *env, * [3]: inactive-on-startup * [4]: restrictive net */ - if (cfg && cfg->cfg_sb && s2lsi(cfg->cfg_sb) && - !IS_SERVER(s2lsi(cfg->cfg_sb))) { + if (cfg && cfg->cfg_sb && s2lsi(cfg->cfg_sb)) { struct lustre_sb_info *lsi = s2lsi(cfg->cfg_sb); char *nidnet = lsi->lsi_lmd->lmd_nidnet; if (lcfg->lcfg_command == LCFG_SETUP && lcfg->lcfg_bufcount != 2 && nidnet) { - CDEBUG(D_CONFIG, "Adding net %s info to setup " - "command for client %s\n", nidnet, - lustre_cfg_string(lcfg, 0)); + CDEBUG(D_CONFIG, + "Adding net %s info to setup command for client %s\n", + nidnet, lustre_cfg_string(lcfg, 0)); lustre_cfg_bufs_set_string(&bufs, 4, nidnet); } } diff --git a/lustre/obdclass/obd_mount.c b/lustre/obdclass/obd_mount.c index 4d91da0..7de849f 100644 --- a/lustre/obdclass/obd_mount.c +++ b/lustre/obdclass/obd_mount.c @@ -456,7 +456,7 @@ int lustre_start_mgc(struct super_block *sb) /* Start the MGC */ rc = lustre_start_simple(mgcname, LUSTRE_MGC_NAME, (char *)uuid->uuid, LUSTRE_MGS_OBDNAME, - niduuid, NULL, NULL); + niduuid, NULL, lsi->lsi_lmd->lmd_nidnet); if (rc) GOTO(out_free, rc); @@ -1744,17 +1744,6 @@ bad_string: GOTO(invalid, rc = -ENOMEM); strncat(lmd->lmd_fileset, s1, s2 - s1 + 1); } - } else { - /* server mount */ - if (lmd->lmd_nidnet != NULL) { - /* 'network=' mount option forbidden for server */ - OBD_FREE(lmd->lmd_nidnet, strlen(lmd->lmd_nidnet) + 1); - lmd->lmd_nidnet = NULL; - rc = -EINVAL; - CERROR("%s: option 'network=' not allowed for Lustre servers: rc = %d\n", - devname, rc); - GOTO(invalid, rc); - } } /* Save mount options */ diff --git a/lustre/osp/lwp_dev.c b/lustre/osp/lwp_dev.c index 8d54abf..8f39d87 100644 --- a/lustre/osp/lwp_dev.c +++ b/lustre/osp/lwp_dev.c @@ -58,16 +58,21 @@ static inline struct lu_device *lwp2lu_dev(struct lwp_device *d) static int lwp_setup(const struct lu_env *env, struct lwp_device *lwp, char *nidstring) { - struct lustre_cfg_bufs *bufs = NULL; - struct lustre_cfg *lcfg = NULL; - char *lwp_name = lwp->lpd_obd->obd_name; - char *server_uuid = NULL; - char *ptr; - int uuid_len = -1; - struct obd_import *imp; - int len = strlen(lwp_name) + 1; - int rc; - const char *lwp_marker = "-" LUSTRE_LWP_NAME "-"; + const char *lwp_marker = "-" LUSTRE_LWP_NAME "-"; + char *lwp_name = lwp->lpd_obd->obd_name; + struct lustre_mount_info *lmi = NULL; + struct lustre_cfg_bufs *bufs = NULL; + struct lustre_cfg *lcfg = NULL; + int len = strlen(lwp_name) + 1; + struct lustre_sb_info *lsi; + char *server_uuid = NULL; + struct obd_import *imp; + char *target = NULL; + char *nidnet = NULL; + int uuid_len = -1; + char *ptr; + int rc; + ENTRY; lwp->lpd_notify_task = NULL; @@ -95,6 +100,28 @@ static int lwp_setup(const struct lu_env *env, struct lwp_device *lwp, lustre_cfg_bufs_reset(bufs, lwp_name); lustre_cfg_bufs_set_string(bufs, 1, server_uuid); lustre_cfg_bufs_set_string(bufs, 2, nidstring); + + OBD_ALLOC(target, len); + if (!target) + GOTO(out, rc = -ENOMEM); + ptr = strchr(lwp_name, '-'); + memcpy(target, lwp_name, ptr - lwp_name); + target[ptr - lwp_name] = '\0'; + strlcat(target, strrchr(lwp_name, '-'), len); + lmi = server_get_mount(target); + if (lmi) { + lsi = s2lsi(lmi->lmi_sb); + if (lsi && lsi->lsi_lmd) + nidnet = lsi->lsi_lmd->lmd_nidnet; + if (nidnet) { + CDEBUG(D_CONFIG, + "Adding net %s info to setup command for %s\n", + nidnet, lwp->lpd_obd->obd_name); + lustre_cfg_bufs_set_string(bufs, 4, nidnet); + } + server_put_mount(target, false); + } + OBD_ALLOC(lcfg, lustre_cfg_len(bufs->lcfg_bufcount, bufs->lcfg_buflen)); if (!lcfg) GOTO(out, rc = -ENOMEM); @@ -111,6 +138,7 @@ static int lwp_setup(const struct lu_env *env, struct lwp_device *lwp, rc = ptlrpc_init_import(imp); out: OBD_FREE_PTR(bufs); + OBD_FREE(target, len); OBD_FREE(server_uuid, len); OBD_FREE(lcfg, lustre_cfg_len(lcfg->lcfg_bufcount, lcfg->lcfg_buflens)); diff --git a/lustre/ptlrpc/client.c b/lustre/ptlrpc/client.c index 1ffd67e..aa998b0 100644 --- a/lustre/ptlrpc/client.c +++ b/lustre/ptlrpc/client.c @@ -121,7 +121,7 @@ struct ptlrpc_connection *ptlrpc_uuid_to_connection(struct obd_uuid *uuid, err = ptlrpc_uuid_to_peer(uuid, &peer, &self, refnet); if (err != 0) { CNETERR("cannot find peer %s!\n", uuid->uuid); - return NULL; + return ERR_PTR(err); } c = ptlrpc_connection_get(&peer, &self, uuid); @@ -132,7 +132,7 @@ struct ptlrpc_connection *ptlrpc_uuid_to_connection(struct obd_uuid *uuid, CDEBUG(D_INFO, "%s -> %p\n", uuid->uuid, c); - return c; + return c ? c : ERR_PTR(-ENOENT); } /** diff --git a/lustre/ptlrpc/events.c b/lustre/ptlrpc/events.c index ba1269b..689b52b 100644 --- a/lustre/ptlrpc/events.c +++ b/lustre/ptlrpc/events.c @@ -525,8 +525,11 @@ int ptlrpc_uuid_to_peer(struct obd_uuid *uuid, /* Choose the matching UUID that's closest */ while (lustre_uuid_to_peer(uuid->uuid, &dst_nid, count++) == 0) { if (refnet != LNET_NET_ANY && - LNET_NID_NET(&dst_nid) != refnet) + LNET_NID_NET(&dst_nid) != refnet) { + if (rc < 0) + rc = -ENETUNREACH; continue; + } dist = LNetDist(&dst_nid, &src_nid, &order); if (dist < 0) diff --git a/lustre/target/tgt_mount.c b/lustre/target/tgt_mount.c index c5e5d57..28cbea1 100644 --- a/lustre/target/tgt_mount.c +++ b/lustre/target/tgt_mount.c @@ -621,9 +621,12 @@ static int lustre_lwp_setup(struct lustre_cfg *lcfg, struct lustre_sb_info *lsi, char *lwpname = NULL; char *lwpuuid = NULL; struct lnet_nid nid; + char *nidnet; + __u32 refnet; int rc; ENTRY; + if (lcfg->lcfg_nid) lnet_nid4_to_nid(lcfg->lcfg_nid, &nid); else { @@ -631,6 +634,12 @@ static int lustre_lwp_setup(struct lustre_cfg *lcfg, struct lustre_sb_info *lsi, if (rc) RETURN(rc); } + + nidnet = lsi->lsi_lmd->lmd_nidnet; + refnet = nidnet ? libcfs_str2net(nidnet) : LNET_NET_ANY; + if (refnet != LNET_NET_ANY && LNET_NID_NET(&nid) != refnet) + RETURN(-ENETUNREACH); + rc = class_add_uuid(lustre_cfg_string(lcfg, 1), &nid); if (rc != 0) { CERROR("%s: Can't add uuid: rc =%d\n", lsi->lsi_svname, rc); @@ -678,7 +687,7 @@ out: OBD_FREE(lwpname, MTI_NAME_MAXLEN); OBD_FREE(lwpuuid, MTI_NAME_MAXLEN); - return rc; + RETURN(rc); } /* the caller is responsible for memory free */ @@ -748,8 +757,14 @@ static int lustre_lwp_add_conn(struct lustre_cfg *cfg, lustre_cfg_init(lcfg, LCFG_ADD_CONN, bufs); rc = class_add_conn(lwp, lcfg); - if (rc < 0) + if (rc == -ENETUNREACH) { + CDEBUG(D_CONFIG, + "%s: ignore conn not on net %s: rc = %d\n", + lwpname, lsi->lsi_lmd->lmd_nidnet, rc); + rc = 0; + } else if (rc < 0) { CERROR("%s: can't add conn: rc = %d\n", lwpname, rc); + } OBD_FREE(lcfg, lustre_cfg_len(lcfg->lcfg_bufcount, lcfg->lcfg_buflens)); @@ -846,10 +861,13 @@ static int client_lwp_config_process(const struct lu_env *env, case LCFG_ADD_UUID: { if (cfg->cfg_flags == CFG_F_MARKER) { rc = lustre_lwp_setup(lcfg, lsi, cfg->cfg_lwp_idx); - /* XXX: process only the first nid as + /* XXX: process only the first nid if on restricted net, * we don't need another instance of lwp */ - cfg->cfg_flags |= CFG_F_SKIP; + if (rc == -ENETUNREACH) + rc = 0; + else + cfg->cfg_flags |= CFG_F_SKIP; } else if (cfg->cfg_flags == (CFG_F_MARKER | CFG_F_SKIP)) { struct lnet_nid nid; @@ -1065,6 +1083,7 @@ static int lustre_start_lwp(struct super_block *sb) cfg->cfg_callback = client_lwp_config_process; cfg->cfg_instance = ll_get_cfg_instance(sb); rc = lustre_process_log(sb, logname, cfg); + /* need to remove config llog from mgc */ lsi->lsi_lwp_started = 1; diff --git a/lustre/tests/sanity-sec.sh b/lustre/tests/sanity-sec.sh index 0c30000..ad6aa82 100755 --- a/lustre/tests/sanity-sec.sh +++ b/lustre/tests/sanity-sec.sh @@ -2699,6 +2699,18 @@ cleanup_31() { LOAD_MODULES_REMOTE=true unload_modules LOAD_MODULES_REMOTE=true load_modules + # restore mgsnid on targets + for ((num = 1; num <= $MDSCOUNT; num++)); do + do_facet mds$num $TUNEFS --erase-param mgsnode \ + $(mdsdevname $num) + do_facet mds$num $TUNEFS --mgsnode=$MGSNID $(mdsdevname $num) + done + for ((num = 1; num <= $OSTCOUNT; num++)); do + do_facet ost$num $TUNEFS --erase-param mgsnode \ + $(ostdevname $num) + do_facet ost$num $TUNEFS --mgsnode=$MGSNID $(ostdevname $num) + done + do_facet mds1 $TUNEFS --erase-param failover.node $(mdsdevname 1) if [ -n "$failover_mds1" ]; then do_facet mds1 $TUNEFS \ @@ -2794,7 +2806,7 @@ test_31() { cut -d'.' -f4-" '!=' $nid for node in ${tgts//,/ }; do wait_update_cond $node \ - "$LCTL get_param -N *.${FSNAME}*.exports | grep $nid | + "$LCTL get_param -N *.${FSNAME}*.exports.* | grep $nid | cut -d'.' -f4-" '!=' $nid done do_facet mgs "$LCTL get_param *.MGS*.exports.*.export" @@ -2813,6 +2825,8 @@ test_31() { # add network $net2 on all nodes do_rpc_nodes $all load_modules || error "unable to load modules on $all" for node in ${all//,/ }; do + do_node $node "$LNETCTL set discovery 0" || + error "Failed to disable discovery on $node" do_node $node "$LNETCTL lnet configure" || error "unable to configure lnet on node $node" infname=inf_$(echo $node | cut -d'.' -f1 | sed s+-+_+g) @@ -2822,6 +2836,22 @@ test_31() { LOAD_MODULES_REMOTE=true load_modules || error "failed to load modules" + # update MGSNID + MGSNID=$mgsnid_orig,$mgsnid_new + stack_trap "MGSNID=$mgsnid_orig" EXIT + + # add mgsnid on @$net2 to targets + for ((num = 1; num <= $MDSCOUNT; num++)); do + do_facet mds$num $TUNEFS --erase-param mgsnode \ + $(mdsdevname $num) + do_facet mds$num $TUNEFS --mgsnode=$MGSNID $(mdsdevname $num) + done + for ((num = 1; num <= $OSTCOUNT; num++)); do + do_facet ost$num $TUNEFS --erase-param mgsnode \ + $(ostdevname $num) + do_facet ost$num $TUNEFS --mgsnode=$MGSNID $(ostdevname $num) + done + # necessary to do writeconf in order to register # new @$net2 nid for targets export SK_MOUNTED=false @@ -2834,10 +2864,6 @@ test_31() { setupall server_only || error "setupall failed" export KEEP_ZPOOL="$KZPOOL" - # update MGSNID - MGSNID=$mgsnid_new - stack_trap "MGSNID=$mgsnid_orig" EXIT - # on client, reconfigure LNet and turn LNet Dynamic Discovery off $LUSTRE_RMMOD || error "$LUSTRE_RMMOD failed (1)" load_modules || error "Failed to load modules" @@ -2890,7 +2916,7 @@ test_31() { cut -d'.' -f4-" '!=' $nid2 for node in ${tgts//,/ }; do wait_update_cond $node \ - "$LCTL get_param -N *.${FSNAME}*.exports | grep $nid2 | + "$LCTL get_param -N *.${FSNAME}*.exports.* | grep $nid2| cut -d'.' -f4-" '!=' $nid2 done do_facet mgs "$LCTL get_param *.MGS*.exports.*.export" @@ -2908,6 +2934,34 @@ test_31() { mount_client $MOUNT ${MOUNT_OPTS},network=$net2 && error "client mount with '-o network' option should be refused" + # remount with '-o network' server side option + (( $MDS1_VERSION >= $(version_code 2.16.51) )) || return 0 + + KZPOOL=$KEEP_ZPOOL + export KEEP_ZPOOL="true" + stopall || error "stopall failed" + mountmgs + for ((num = 1; num <= $MDSCOUNT; num++)); do + start mds$num $(mdsdevname $num) $MDS_MOUNT_OPTS,network=$net2 + done + for ((num = 1; num <= $OSTCOUNT; num++)); do + start ost$num $(ostdevname $num) $OST_MOUNT_OPTS,network=$net2 + done + export KEEP_ZPOOL="$KZPOOL" + sleep 5 + + # check exports on servers are empty for $net + do_facet mgs "$LCTL get_param mgs.MGS.exports.*.export" + wait_update_facet_cond mgs \ + "$LCTL get_param -N mgs.MGS.exports.*.export | \ + grep ${net}.export | cut -d'@' -f2-" '!=' ${net}.export + do_nodes $tgts "$LCTL get_param *.${FSNAME}*.exports.*.export" + for node in ${tgts//,/ }; do + wait_update_cond $node \ + "$LCTL get_param -N *.${FSNAME}*.exports.*.export | \ + grep ${net}.export | cut -d'@' -f2-" '!=' ${net}.export + done + return 0 } run_test 31 "client mount option '-o network'" -- 1.8.3.1