From: Amir Shehata Date: Mon, 10 Aug 2020 23:09:22 +0000 (-0700) Subject: LU-10360 mgc: Use IR for client->MDS/OST connections X-Git-Tag: 2.13.56~27 X-Git-Url: https://git.whamcloud.com/?p=fs%2Flustre-release.git;a=commitdiff_plain;h=37be05eca3f4aee15c946656a77f56967c15253d LU-10360 mgc: Use IR for client->MDS/OST connections When a target registers with the MGS, the MGS sends an IR log to the client to speed up recovery. The IR log contains the updated NID information on the target which just registered. This patch allows clients to update their imports with the latest NIDs for the targets reported in the IR log. It also allows clients to create new connections for targets which were not added via the config log. For example if a target reboots and comes up with a new NID, then the client can continue using it. This functionality is disabled by default and can be enabled by setting a new file system specific module parameter, dynamic_nids. lctl set_param mgc.*.dynamic_nids=1 This parameters will need to be set on the clients and the MGS Signed-off-by: Amir Shehata Change-Id: I9e85438c39a46d4950ac50309c1bbae2a96d7d5d Reviewed-on: https://review.whamcloud.com/39613 Tested-by: jenkins Reviewed-by: Andreas Dilger Tested-by: Maloo Reviewed-by: Mike Pershin Reviewed-by: Oleg Drokin --- diff --git a/lustre/include/lustre_net.h b/lustre/include/lustre_net.h index 0240880..fa18f84 100644 --- a/lustre/include/lustre_net.h +++ b/lustre/include/lustre_net.h @@ -2646,6 +2646,10 @@ int client_connect_import(const struct lu_env *env, int client_disconnect_export(struct obd_export *exp); int client_import_add_conn(struct obd_import *imp, struct obd_uuid *uuid, int priority); +int client_import_dyn_add_conn(struct obd_import *imp, struct obd_uuid *uuid, + lnet_nid_t prim_nid, int priority); +int client_import_add_nids_to_conn(struct obd_import *imp, lnet_nid_t *nids, + int nid_count, struct obd_uuid *uuid); int client_import_del_conn(struct obd_import *imp, struct obd_uuid *uuid); int client_import_find_conn(struct obd_import *imp, lnet_nid_t peer, struct obd_uuid *uuid); diff --git a/lustre/include/obd.h b/lustre/include/obd.h index 5940d33..f21b060 100644 --- a/lustre/include/obd.h +++ b/lustre/include/obd.h @@ -631,7 +631,8 @@ struct obd_device { * (for /proc/status only!!) */ obd_no_ir:1, /* no imperative recovery. */ obd_process_conf:1, /* device is processing mgs config */ - obd_checksum_dump:1; /* dump pages upon cksum error */ + obd_checksum_dump:1, /* dump pages upon cksum error */ + obd_dynamic_nids:1; /* Allow dynamic NIDs on device */ #ifdef HAVE_SERVER_SUPPORT /* no committed-transno notification */ unsigned long obd_no_transno:1; diff --git a/lustre/include/obd_class.h b/lustre/include/obd_class.h index 8148dcc..0284b13 100644 --- a/lustre/include/obd_class.h +++ b/lustre/include/obd_class.h @@ -1840,6 +1840,8 @@ int lustre_check_exclusion(struct super_block *sb, char *svname); int lustre_uuid_to_peer(const char *uuid, lnet_nid_t *peer_nid, int index); int class_add_uuid(const char *uuid, __u64 nid); int class_del_uuid (const char *uuid); +int class_add_nids_to_uuid(struct obd_uuid *uuid, lnet_nid_t *nids, + int nid_count); int class_check_uuid(struct obd_uuid *uuid, __u64 nid); /* class_obd.c */ diff --git a/lustre/ldlm/ldlm_lib.c b/lustre/ldlm/ldlm_lib.c index 4715718..74d769b 100644 --- a/lustre/ldlm/ldlm_lib.c +++ b/lustre/ldlm/ldlm_lib.c @@ -141,6 +141,51 @@ int client_import_add_conn(struct obd_import *imp, struct obd_uuid *uuid, } EXPORT_SYMBOL(client_import_add_conn); +int client_import_dyn_add_conn(struct obd_import *imp, struct obd_uuid *uuid, + lnet_nid_t prim_nid, int priority) +{ + struct ptlrpc_connection *ptlrpc_conn; + int rc; + + ptlrpc_conn = ptlrpc_uuid_to_connection(uuid, prim_nid); + if (!ptlrpc_conn) { + const char *str_uuid = obd_uuid2str(uuid); + + rc = class_add_uuid(str_uuid, prim_nid); + if (rc) { + CERROR("%s: failed to add UUID '%s': rc = %d\n", + imp->imp_obd->obd_name, str_uuid, rc); + return rc; + } + } + return import_set_conn(imp, uuid, priority, 1); +} +EXPORT_SYMBOL(client_import_dyn_add_conn); + +int client_import_add_nids_to_conn(struct obd_import *imp, lnet_nid_t *nids, + int nid_count, struct obd_uuid *uuid) +{ + struct obd_import_conn *conn; + int rc = -ENOENT; + + ENTRY; + if (nid_count <= 0 || !nids) + return rc; + + spin_lock(&imp->imp_lock); + list_for_each_entry(conn, &imp->imp_conn_list, oic_item) { + if (class_check_uuid(&conn->oic_uuid, nids[0])) { + *uuid = conn->oic_uuid; + rc = class_add_nids_to_uuid(&conn->oic_uuid, nids, + nid_count); + break; + } + } + spin_unlock(&imp->imp_lock); + RETURN(rc); +} +EXPORT_SYMBOL(client_import_add_nids_to_conn); + int client_import_del_conn(struct obd_import *imp, struct obd_uuid *uuid) { struct obd_import_conn *imp_conn; diff --git a/lustre/mgc/lproc_mgc.c b/lustre/mgc/lproc_mgc.c index f277d3e..74b6ec8 100644 --- a/lustre/mgc/lproc_mgc.c +++ b/lustre/mgc/lproc_mgc.c @@ -74,10 +74,46 @@ LUSTRE_RO_ATTR(conn_uuid); LUSTRE_RW_ATTR(ping); +ssize_t dynamic_nids_show(struct kobject *kobj, struct attribute *attr, + char *buf) +{ + struct obd_device *obd = container_of(kobj, struct obd_device, + obd_kset.kobj); + ssize_t count; + + ENTRY; + count = snprintf(buf, PAGE_SIZE, "%u\n", obd->obd_dynamic_nids); + + RETURN(count); +} + +ssize_t dynamic_nids_store(struct kobject *kobj, struct attribute *attr, + const char *buffer, size_t count) +{ + struct obd_device *obd = container_of(kobj, struct obd_device, + obd_kset.kobj); + bool val; + int rc; + + ENTRY; + rc = kstrtobool(buffer, &val); + if (rc) + return rc; + + spin_lock(&obd->obd_dev_lock); + obd->obd_dynamic_nids = val; + spin_unlock(&obd->obd_dev_lock); + + RETURN(count); +} + +LUSTRE_RW_ATTR(dynamic_nids); + static struct attribute *mgc_attrs[] = { &lustre_attr_mgs_conn_uuid.attr, &lustre_attr_conn_uuid.attr, &lustre_attr_ping.attr, + &lustre_attr_dynamic_nids.attr, NULL, }; diff --git a/lustre/mgc/mgc_request.c b/lustre/mgc/mgc_request.c index ab709dd..4aee4ff 100644 --- a/lustre/mgc/mgc_request.c +++ b/lustre/mgc/mgc_request.c @@ -1386,11 +1386,15 @@ static int mgc_apply_recover_logs(struct obd_device *mgc, int pos = 0; int rc = 0; int off = 0; + unsigned long dynamic_nids; ENTRY; LASSERT(cfg->cfg_instance != 0); LASSERT(ll_get_cfg_instance(cfg->cfg_sb) == cfg->cfg_instance); + /* get dynamic nids setting */ + dynamic_nids = mgc->obd_dynamic_nids; + OBD_ALLOC(inst, PAGE_SIZE); if (inst == NULL) RETURN(-ENOMEM); @@ -1414,36 +1418,36 @@ static int mgc_apply_recover_logs(struct obd_device *mgc, #endif /* HAVE_SERVER_SUPPORT */ } - ++pos; - buf = inst + pos; + ++pos; + buf = inst + pos; bufsz = PAGE_SIZE - pos; - while (datalen > 0) { - int entry_len = sizeof(*entry); - int is_ost, i; - struct obd_device *obd; - char *obdname; - char *cname; - char *params; - char *uuid; - - rc = -EINVAL; - if (datalen < sizeof(*entry)) - break; + while (datalen > 0) { + int entry_len = sizeof(*entry); + int is_ost; + struct obd_device *obd; + char *obdname; + char *cname; + char *params; + char *uuid; + + rc = -EINVAL; + if (datalen < sizeof(*entry)) + break; - entry = (typeof(entry))(data + off); + entry = (typeof(entry))(data + off); - /* sanity check */ - if (entry->mne_nid_type != 0) /* only support type 0 for ipv4 */ - break; - if (entry->mne_nid_count == 0) /* at least one nid entry */ - break; - if (entry->mne_nid_size != sizeof(lnet_nid_t)) - break; + /* sanity check */ + if (entry->mne_nid_type != 0) /* only support type 0 for ipv4 */ + break; + if (entry->mne_nid_count == 0) /* at least one nid entry */ + break; + if (entry->mne_nid_size != sizeof(lnet_nid_t)) + break; - entry_len += entry->mne_nid_count * entry->mne_nid_size; - if (datalen < entry_len) /* must have entry_len at least */ - break; + entry_len += entry->mne_nid_count * entry->mne_nid_size; + if (datalen < entry_len) /* must have entry_len at least */ + break; /* Keep this swab for normal mixed endian handling. LU-1644 */ if (mne_swab) @@ -1528,24 +1532,55 @@ static int mgc_apply_recover_logs(struct obd_device *mgc, /* iterate all nids to find one */ /* find uuid by nid */ + /* create import entries if they don't exist */ rc = -ENOENT; - for (i = 0; i < entry->mne_nid_count; i++) { - rc = client_import_find_conn(obd->u.cli.cl_import, - entry->u.nids[i], - (struct obd_uuid *)uuid); - if (rc == 0) - break; + rc = client_import_add_nids_to_conn(obd->u.cli.cl_import, + entry->u.nids, + entry->mne_nid_count, + (struct obd_uuid *)uuid); + + if (rc == -ENOENT && dynamic_nids) { + /* create a new connection for this import */ + char *primary_nid = libcfs_nid2str(entry->u.nids[0]); + int prim_nid_len = strlen(primary_nid) + 1; + struct obd_uuid server_uuid; + + if (prim_nid_len > UUID_MAX) + goto fail; + strncpy(server_uuid.uuid, primary_nid, prim_nid_len); + + CDEBUG(D_INFO, "Adding a connection for %s\n", + primary_nid); + + rc = client_import_dyn_add_conn(obd->u.cli.cl_import, + &server_uuid, + entry->u.nids[0], 1); + if (rc < 0) { + CERROR("%s: Failed to add new connection with NID '%s' to import: rc = %d\n", + obd->obd_name, primary_nid, rc); + goto fail; + } + rc = client_import_add_nids_to_conn(obd->u.cli.cl_import, + entry->u.nids, + entry->mne_nid_count, + (struct obd_uuid *)uuid); + if (rc < 0) { + CERROR("%s: failed to lookup UUID: rc = %d\n", + obd->obd_name, rc); + goto fail; + } } +fail: up_read(&obd->u.cli.cl_sem); - if (rc < 0) { - CERROR("mgc: cannot find uuid by nid %s\n", - libcfs_nid2str(entry->u.nids[0])); - break; - } + if (rc < 0 && rc != -ENOSPC) { + CERROR("mgc: cannot find UUID by nid '%s': rc = %d\n", + libcfs_nid2str(entry->u.nids[0]), rc); + break; + } - CDEBUG(D_INFO, "Find uuid %s by nid %s\n", - uuid, libcfs_nid2str(entry->u.nids[0])); + CDEBUG(D_INFO, "Found UUID '%s' by NID '%s'\n", + uuid, libcfs_nid2str(entry->u.nids[0])); pos += strlen(uuid); pos += sprintf(buf + pos, "::%u", entry->mne_instance); diff --git a/lustre/obdclass/lustre_peer.c b/lustre/obdclass/lustre_peer.c index d8dc95a..d344222 100644 --- a/lustre/obdclass/lustre_peer.c +++ b/lustre/obdclass/lustre_peer.c @@ -40,13 +40,11 @@ #include #include -#define NIDS_MAX 32 - struct uuid_nid_data { struct list_head un_list; struct obd_uuid un_uuid; int un_nid_count; - lnet_nid_t un_nids[NIDS_MAX]; + lnet_nid_t un_nids[MTI_NIDS_MAX]; }; /* FIXME: This should probably become more elegant than a global linked list */ @@ -107,7 +105,7 @@ int class_add_uuid(const char *uuid, __u64 nid) break; if (i == entry->un_nid_count) { - LASSERT(entry->un_nid_count < NIDS_MAX); + LASSERT(entry->un_nid_count < MTI_NIDS_MAX); entry->un_nids[entry->un_nid_count++] = nid; } break; @@ -126,6 +124,7 @@ int class_add_uuid(const char *uuid, __u64 nid) } return 0; } +EXPORT_SYMBOL(class_add_uuid); /* Delete the nids for one uuid if specified, otherwise delete all */ int class_del_uuid(const char *uuid) @@ -168,6 +167,38 @@ int class_del_uuid(const char *uuid) return 0; } +int class_add_nids_to_uuid(struct obd_uuid *uuid, lnet_nid_t *nids, + int nid_count) +{ + struct uuid_nid_data *entry; + int i; + + ENTRY; + + if (nid_count >= MTI_NIDS_MAX) { + CDEBUG(D_NET, "too many NIDs (%d) for UUID '%s'\n", + nid_count, obd_uuid2str(uuid)); + return -ENOSPC; + } + + spin_lock(&g_uuid_lock); + list_for_each_entry(entry, &g_uuid_list, un_list) { + CDEBUG(D_NET, "Comparing %s with %s\n", + obd_uuid2str(uuid), obd_uuid2str(&entry->un_uuid)); + + if (!obd_uuid_equals(&entry->un_uuid, uuid)) + continue; + CDEBUG(D_NET, "Updating UUID '%s'\n", obd_uuid2str(uuid)); + for (i = 0; i < nid_count; i++) + entry->un_nids[i] = nids[i]; + entry->un_nid_count = nid_count; + break; + } + spin_unlock(&g_uuid_lock); + RETURN(0); +} +EXPORT_SYMBOL(class_add_nids_to_uuid); + /* check if @nid exists in nid list of @uuid */ int class_check_uuid(struct obd_uuid *uuid, __u64 nid) {