From 84118744979594a1a4694797722812b3c6f3ee64 Mon Sep 17 00:00:00 2001 From: Mikhail Pershin Date: Thu, 8 May 2025 12:40:54 +0300 Subject: [PATCH] LU-18986 mgs: new target registration protocol Patch adds new target registration request format with enhanced NIDs list handling. The idea is to don't overload mgs_target_info with extra flags and fields for NID list description but keep such information in new structure. NIDs list is arrays of string always and can be send in varios manners: inline buffer, bulk, compressed, appended, etc. It helps also to resolve compatibility issues. Patch includes: - new wire structure mgs_target_nidlist - new possible RPC format with mgs_target_nidlist buffer - new connect flag OBD_CONNECT_MGS_NIDLIST to replace obsoleted OBD_CONNECT_REQPORTAL removed in commit 1.6.0-159-gd2d56f38da ("make HEAD from b_post_cmd3") - corresponding swabber and wirecheck Test-Parameters: testlist=runtests clientversion=EXA6.3.2 Test-Parameters: testlist=runtests serverversion=EXA6.3.2 Signed-off-by: Mikhail Pershin Change-Id: I441de467a530137f76712273b9a5f814fdb562c1 Reviewed-on: https://review.whamcloud.com/c/fs/lustre-release/+/59205 Tested-by: Maloo Tested-by: jenkins Reviewed-by: Andreas Dilger Reviewed-by: Alex Zhuravlev Reviewed-by: James Simmons Reviewed-by: Oleg Drokin --- lustre/include/lustre_req_layout.h | 2 ++ lustre/include/lustre_swab.h | 1 + lustre/include/uapi/linux/lustre/lustre_idl.h | 20 +++++++++++++++++--- lustre/llite/llite_lib.c | 2 +- lustre/lod/lod_lov.c | 1 - lustre/obdclass/lprocfs_status.c | 2 +- lustre/obdecho/echo_client.c | 2 +- lustre/osc/osc_request.c | 5 ----- lustre/ptlrpc/layout.c | 17 +++++++++++++++++ lustre/ptlrpc/pack_generic.c | 7 +++++++ lustre/ptlrpc/wiretest.c | 23 +++++++++++++++++++++-- lustre/utils/wirecheck.c | 17 ++++++++++++++++- lustre/utils/wiretest.c | 23 +++++++++++++++++++++-- 13 files changed, 105 insertions(+), 17 deletions(-) diff --git a/lustre/include/lustre_req_layout.h b/lustre/include/lustre_req_layout.h index 8ac8f3a..c78c8ac 100644 --- a/lustre/include/lustre_req_layout.h +++ b/lustre/include/lustre_req_layout.h @@ -182,6 +182,7 @@ extern struct req_format RQF_SEC_CTX; extern struct req_format RQF_OBD_IDX_READ; /* MGS req_format */ extern struct req_format RQF_MGS_TARGET_REG; +extern struct req_format RQF_MGS_TARGET_REG_NIDLIST; extern struct req_format RQF_MGS_SET_INFO; extern struct req_format RQF_MGS_CONFIG_READ; /* fid/fld req_format */ @@ -366,6 +367,7 @@ extern struct req_msg_field RMF_LLOG_LOG_HDR; extern struct req_msg_field RMF_LLOGD_CONN_BODY; extern struct req_msg_field RMF_MGS_TARGET_INFO; +extern struct req_msg_field RMF_MGS_TARGET_NIDLIST; extern struct req_msg_field RMF_MGS_SEND_PARAM; extern struct req_msg_field RMF_OST_BODY; diff --git a/lustre/include/lustre_swab.h b/lustre/include/lustre_swab.h index 033c120..2b9cd20 100644 --- a/lustre/include/lustre_swab.h +++ b/lustre/include/lustre_swab.h @@ -81,6 +81,7 @@ void lustre_swab_ldlm_lock_desc(struct ldlm_lock_desc *l); void lustre_swab_ldlm_request(struct ldlm_request *rq); void lustre_swab_ldlm_reply(struct ldlm_reply *r); void lustre_swab_mgs_target_info(struct mgs_target_info *oinfo); +void lustre_swab_mgs_target_nidlist(struct mgs_target_nidlist *mtn); void lustre_swab_mgs_nidtbl_entry_header(struct mgs_nidtbl_entry *oinfo); void lustre_swab_mgs_nidtbl_entry_content(struct mgs_nidtbl_entry *oinfo); void lustre_swab_mgs_config_body(struct mgs_config_body *body); diff --git a/lustre/include/uapi/linux/lustre/lustre_idl.h b/lustre/include/uapi/linux/lustre/lustre_idl.h index f4c89c2..093ccc1 100644 --- a/lustre/include/uapi/linux/lustre/lustre_idl.h +++ b/lustre/include/uapi/linux/lustre/lustre_idl.h @@ -739,7 +739,7 @@ struct ptlrpc_body_v2 { #define OBD_CONNECT_GRANT 0x8ULL /* fetch grant connect */ #define OBD_CONNECT_SRVLOCK 0x10ULL /* server lock for RPC */ #define OBD_CONNECT_VERSION 0x20ULL /* versions in OCD */ -#define OBD_CONNECT_REQPORTAL 0x40ULL /* non-IO portal */ +#define OBD_CONNECT_MGS_NIDLIST 0x40ULL /* MGS nidlist protocol */ #define OBD_CONNECT_ACL 0x80ULL /* access control list */ #define OBD_CONNECT_XATTR 0x100ULL /* extended attributes */ #define OBD_CONNECT_LARGE_ACL 0x200ULL /* over 32 ACL entries */ @@ -925,8 +925,7 @@ struct ptlrpc_body_v2 { OBD_CONNECT2_MIRROR_ID_FIX) #define OST_CONNECT_SUPPORTED (OBD_CONNECT_SRVLOCK | OBD_CONNECT_GRANT | \ - OBD_CONNECT_REQPORTAL | OBD_CONNECT_VERSION | \ - OBD_CONNECT_INDEX | \ + OBD_CONNECT_VERSION | OBD_CONNECT_INDEX | \ OBD_CONNECT_BRW_SIZE | OBD_CONNECT_CANCELSET | \ OBD_CONNECT_AT | LRU_RESIZE_CONNECT_FLAG | \ OBD_CONNECT_CKSUM | OBD_CONNECT_VBR | \ @@ -2752,6 +2751,21 @@ struct mgs_target_info { char mti_nidlist[][LNET_NIDSTR_SIZE]; } __attribute__((packed)); +enum mgs_nidlist_flags { + NIDLIST_APPEND = 0x00000001, /* append NIDs to nidtable */ + NIDLIST_IN_BULK = 0x00000002, /* nidlist in bulk */ + NIDLIST_COMPRESSED = 0x00000004, /* nidlist is compressed */ +}; + +#define MTN_NIDSTR_SIZE LNET_NIDSTR_SIZE +#define NIDLIST_SIZE(count) (MTN_NIDSTR_SIZE * count) + +struct mgs_target_nidlist { + __u32 mtn_flags; /* flags from mgs_nids_info_flags */ + __u32 mtn_nids; /* amount of nids in list */ + char mtn_inline_list[][MTN_NIDSTR_SIZE]; +} __attribute__((packed)); + struct mgs_nidtbl_entry { __u64 mne_version; /* table version of this entry */ __u32 mne_instance; /* target instance # */ diff --git a/lustre/llite/llite_lib.c b/lustre/llite/llite_lib.c index a0119fc..6c13a68 100644 --- a/lustre/llite/llite_lib.c +++ b/lustre/llite/llite_lib.c @@ -594,7 +594,7 @@ retry_connect: /* indicate OST features supported by this client */ data->ocd_connect_flags = OBD_CONNECT_GRANT | OBD_CONNECT_VERSION | - OBD_CONNECT_REQPORTAL | OBD_CONNECT_BRW_SIZE | + OBD_CONNECT_BRW_SIZE | OBD_CONNECT_CANCELSET | OBD_CONNECT_FID | OBD_CONNECT_SRVLOCK | OBD_CONNECT_AT | OBD_CONNECT_OSS_CAPA | diff --git a/lustre/lod/lod_lov.c b/lustre/lod/lod_lov.c index 0c83bae..193b82c 100644 --- a/lustre/lod/lod_lov.c +++ b/lustre/lod/lod_lov.c @@ -166,7 +166,6 @@ int lod_add_device(const struct lu_env *env, struct lod_device *lod, OBD_CONNECT_LRU_RESIZE | #endif OBD_CONNECT_MDS | - OBD_CONNECT_REQPORTAL | OBD_CONNECT_SKIP_ORPHAN | OBD_CONNECT_FID | OBD_CONNECT_LVB_TYPE | diff --git a/lustre/obdclass/lprocfs_status.c b/lustre/obdclass/lprocfs_status.c index 259c1fc..8a02421 100644 --- a/lustre/obdclass/lprocfs_status.c +++ b/lustre/obdclass/lprocfs_status.c @@ -630,7 +630,7 @@ static const char *const obd_connect_names[] = { "write_grant", /* 0x04 */ "server_lock", /* 0x10 */ "version", /* 0x20 */ - "request_portal", /* 0x40 */ + "mgs_nidlist", /* 0x40 */ "acl", /* 0x80 */ "xattr", /* 0x100 */ "create_on_write", /* 0x200 */ diff --git a/lustre/obdecho/echo_client.c b/lustre/obdecho/echo_client.c index 0378c9b..801ab94 100644 --- a/lustre/obdecho/echo_client.c +++ b/lustre/obdecho/echo_client.c @@ -2464,7 +2464,7 @@ static int echo_client_setup(const struct lu_env *env, return -ENOMEM; } - ocd->ocd_connect_flags = OBD_CONNECT_VERSION | OBD_CONNECT_REQPORTAL | + ocd->ocd_connect_flags = OBD_CONNECT_VERSION | OBD_CONNECT_BRW_SIZE | OBD_CONNECT_GRANT | OBD_CONNECT_FULL20 | OBD_CONNECT_64BITHASH | OBD_CONNECT_LVB_TYPE | diff --git a/lustre/osc/osc_request.c b/lustre/osc/osc_request.c index 5f20e30..ab34b8c 100644 --- a/lustre/osc/osc_request.c +++ b/lustre/osc/osc_request.c @@ -3895,11 +3895,6 @@ static int osc_import_event(struct obd_device *obd, struct obd_import *imp, if (ocd->ocd_connect_flags & OBD_CONNECT_GRANT) osc_init_grant(&obd->u.cli, ocd); - /* See bug 7198 */ - if (ocd->ocd_connect_flags & OBD_CONNECT_REQPORTAL) - imp->imp_client->cli_request_portal = - OST_REQUEST_PORTAL; - rc = obd_notify_observer(obd, obd, OBD_NOTIFY_OCD); break; } diff --git a/lustre/ptlrpc/layout.c b/lustre/ptlrpc/layout.c index 6924ea4..1458430 100644 --- a/lustre/ptlrpc/layout.c +++ b/lustre/ptlrpc/layout.c @@ -52,6 +52,12 @@ static const struct req_msg_field *mgs_target_info_only[] = { &RMF_MGS_TARGET_INFO }; +static const struct req_msg_field *mgs_target_info_nidlist[] = { + &RMF_PTLRPC_BODY, + &RMF_MGS_TARGET_INFO, + &RMF_MGS_TARGET_NIDLIST, +}; + #if LUSTRE_VERSION_CODE < OBD_OCD_VERSION(2, 18, 53, 0) static const struct req_msg_field *mgs_set_info[] = { &RMF_PTLRPC_BODY, @@ -799,6 +805,7 @@ static struct req_format *req_formats[] = { &RQF_OBD_IDX_READ, &RQF_SEC_CTX, &RQF_MGS_TARGET_REG, + &RQF_MGS_TARGET_REG_NIDLIST, #if LUSTRE_VERSION_CODE < OBD_OCD_VERSION(2, 18, 53, 0) &RQF_MGS_SET_INFO, #endif @@ -964,6 +971,11 @@ struct req_msg_field RMF_MGS_TARGET_INFO = lustre_swab_mgs_target_info, NULL); EXPORT_SYMBOL(RMF_MGS_TARGET_INFO); +struct req_msg_field RMF_MGS_TARGET_NIDLIST = + DEFINE_MSGF("mgs_target_nidlist", 0, sizeof(struct mgs_target_nidlist), + lustre_swab_mgs_target_nidlist, NULL); +EXPORT_SYMBOL(RMF_MGS_TARGET_NIDLIST); + #if LUSTRE_VERSION_CODE < OBD_OCD_VERSION(2, 18, 53, 0) struct req_msg_field RMF_MGS_SEND_PARAM = DEFINE_MSGF("mgs_send_param", 0, @@ -1410,6 +1422,11 @@ struct req_format RQF_MGS_TARGET_REG = mgs_target_info_only); EXPORT_SYMBOL(RQF_MGS_TARGET_REG); +struct req_format RQF_MGS_TARGET_REG_NIDLIST = + DEFINE_REQ_FMT0("MGS_TARGET_REG_NIDLIST", mgs_target_info_nidlist, + mgs_target_info_only); +EXPORT_SYMBOL(RQF_MGS_TARGET_REG_NIDLIST); + #if LUSTRE_VERSION_CODE < OBD_OCD_VERSION(2, 18, 53, 0) struct req_format RQF_MGS_SET_INFO = DEFINE_REQ_FMT0("MGS_SET_INFO", mgs_set_info, diff --git a/lustre/ptlrpc/pack_generic.c b/lustre/ptlrpc/pack_generic.c index 91e95ff..3df5e8e 100644 --- a/lustre/ptlrpc/pack_generic.c +++ b/lustre/ptlrpc/pack_generic.c @@ -2089,6 +2089,13 @@ void lustre_swab_mgs_target_info(struct mgs_target_info *mti) __swab64s(&mti->mti_nids[i]); } +void lustre_swab_mgs_target_nidlist(struct mgs_target_nidlist *mtn) +{ + __swab32s(&mtn->mtn_flags); + __swab32s(&mtn->mtn_nids); + BUILD_BUG_ON(MTN_NIDSTR_SIZE != 64); +} + void lustre_swab_mgs_nidtbl_entry_header(struct mgs_nidtbl_entry *entry) { __swab64s(&entry->mne_version); diff --git a/lustre/ptlrpc/wiretest.c b/lustre/ptlrpc/wiretest.c index f1d2855..07da401 100644 --- a/lustre/ptlrpc/wiretest.c +++ b/lustre/ptlrpc/wiretest.c @@ -1272,8 +1272,8 @@ void lustre_assert_wire_constants(void) OBD_CONNECT_SRVLOCK); LASSERTF(OBD_CONNECT_VERSION == 0x20ULL, "found 0x%.16llxULL\n", OBD_CONNECT_VERSION); - LASSERTF(OBD_CONNECT_REQPORTAL == 0x40ULL, "found 0x%.16llxULL\n", - OBD_CONNECT_REQPORTAL); + LASSERTF(OBD_CONNECT_MGS_NIDLIST == 0x40ULL, "found 0x%.16llxULL\n", + OBD_CONNECT_MGS_NIDLIST); LASSERTF(OBD_CONNECT_ACL == 0x80ULL, "found 0x%.16llxULL\n", OBD_CONNECT_ACL); LASSERTF(OBD_CONNECT_XATTR == 0x100ULL, "found 0x%.16llxULL\n", @@ -4836,6 +4836,25 @@ void lustre_assert_wire_constants(void) LASSERTF((int)sizeof(((struct mgs_target_info *)0)->mti_nidlist[0]) == 64, "found %lld\n", (long long)(int)sizeof(((struct mgs_target_info *)0)->mti_nidlist[0])); + /* Checks for struct mgs_target_nidlist */ + LASSERTF((int)sizeof(struct mgs_target_nidlist) == 8, "found %lld\n", + (long long)(int)sizeof(struct mgs_target_nidlist)); + LASSERTF((int)offsetof(struct mgs_target_nidlist, mtn_flags) == 0, "found %lld\n", + (long long)(int)offsetof(struct mgs_target_nidlist, mtn_flags)); + LASSERTF((int)sizeof(((struct mgs_target_nidlist *)0)->mtn_flags) == 4, "found %lld\n", + (long long)(int)sizeof(((struct mgs_target_nidlist *)0)->mtn_flags)); + LASSERTF((int)offsetof(struct mgs_target_nidlist, mtn_nids) == 4, "found %lld\n", + (long long)(int)offsetof(struct mgs_target_nidlist, mtn_nids)); + LASSERTF((int)sizeof(((struct mgs_target_nidlist *)0)->mtn_nids) == 4, "found %lld\n", + (long long)(int)sizeof(((struct mgs_target_nidlist *)0)->mtn_nids)); + LASSERTF((int)offsetof(struct mgs_target_nidlist, mtn_inline_list[0]) == 8, "found %lld\n", + (long long)(int)offsetof(struct mgs_target_nidlist, mtn_inline_list[0])); + LASSERTF((int)sizeof(((struct mgs_target_nidlist *)0)->mtn_inline_list[0]) == 64, "found %lld\n", + (long long)(int)sizeof(((struct mgs_target_nidlist *)0)->mtn_inline_list[0])); + BUILD_BUG_ON(NIDLIST_APPEND != 0x00000001); + BUILD_BUG_ON(NIDLIST_IN_BULK != 0x00000002); + BUILD_BUG_ON(NIDLIST_COMPRESSED != 0x00000004); + /* Checks for struct mgs_nidtbl_entry */ LASSERTF((int)sizeof(struct mgs_nidtbl_entry) == 24, "found %lld\n", (long long)(int)sizeof(struct mgs_nidtbl_entry)); diff --git a/lustre/utils/wirecheck.c b/lustre/utils/wirecheck.c index 8a1f670..00e3a30 100644 --- a/lustre/utils/wirecheck.c +++ b/lustre/utils/wirecheck.c @@ -614,7 +614,7 @@ check_obd_connect_data(void) CHECK_DEFINE_64X(OBD_CONNECT_GRANT); CHECK_DEFINE_64X(OBD_CONNECT_SRVLOCK); CHECK_DEFINE_64X(OBD_CONNECT_VERSION); - CHECK_DEFINE_64X(OBD_CONNECT_REQPORTAL); + CHECK_DEFINE_64X(OBD_CONNECT_MGS_NIDLIST); CHECK_DEFINE_64X(OBD_CONNECT_ACL); CHECK_DEFINE_64X(OBD_CONNECT_XATTR); CHECK_DEFINE_64X(OBD_CONNECT_LARGE_ACL); @@ -2252,6 +2252,20 @@ check_mgs_target_info(void) } static void +check_mgs_target_nidlist(void) +{ + BLANK_LINE(); + CHECK_STRUCT(mgs_target_nidlist); + CHECK_MEMBER(mgs_target_nidlist, mtn_flags); + CHECK_MEMBER(mgs_target_nidlist, mtn_nids); + CHECK_MEMBER(mgs_target_nidlist, mtn_inline_list[0]); + + CHECK_CVALUE_X(NIDLIST_APPEND); + CHECK_CVALUE_X(NIDLIST_IN_BULK); + CHECK_CVALUE_X(NIDLIST_COMPRESSED); +} + +static void check_mgs_nidtbl_entry(void) { BLANK_LINE(); @@ -3788,6 +3802,7 @@ main(int argc, char **argv) check_quota_body(); #endif /* !HAVE_NATIVE_LINUX_CLIENT */ check_mgs_target_info(); + check_mgs_target_nidlist(); check_mgs_nidtbl_entry(); check_mgs_config_body(); check_mgs_config_res(); diff --git a/lustre/utils/wiretest.c b/lustre/utils/wiretest.c index 6019ecf..07679c9 100644 --- a/lustre/utils/wiretest.c +++ b/lustre/utils/wiretest.c @@ -1315,8 +1315,8 @@ void lustre_assert_wire_constants(void) OBD_CONNECT_SRVLOCK); LASSERTF(OBD_CONNECT_VERSION == 0x20ULL, "found 0x%.16llxULL\n", OBD_CONNECT_VERSION); - LASSERTF(OBD_CONNECT_REQPORTAL == 0x40ULL, "found 0x%.16llxULL\n", - OBD_CONNECT_REQPORTAL); + LASSERTF(OBD_CONNECT_MGS_NIDLIST == 0x40ULL, "found 0x%.16llxULL\n", + OBD_CONNECT_MGS_NIDLIST); LASSERTF(OBD_CONNECT_ACL == 0x80ULL, "found 0x%.16llxULL\n", OBD_CONNECT_ACL); LASSERTF(OBD_CONNECT_XATTR == 0x100ULL, "found 0x%.16llxULL\n", @@ -4881,6 +4881,25 @@ void lustre_assert_wire_constants(void) (long long)(int)sizeof(((struct mgs_target_info *)0)->mti_nidlist[0])); #endif /* HAVE_SERVER_SUPPORT */ + /* Checks for struct mgs_target_nidlist */ + LASSERTF((int)sizeof(struct mgs_target_nidlist) == 8, "found %lld\n", + (long long)(int)sizeof(struct mgs_target_nidlist)); + LASSERTF((int)offsetof(struct mgs_target_nidlist, mtn_flags) == 0, "found %lld\n", + (long long)(int)offsetof(struct mgs_target_nidlist, mtn_flags)); + LASSERTF((int)sizeof(((struct mgs_target_nidlist *)0)->mtn_flags) == 4, "found %lld\n", + (long long)(int)sizeof(((struct mgs_target_nidlist *)0)->mtn_flags)); + LASSERTF((int)offsetof(struct mgs_target_nidlist, mtn_nids) == 4, "found %lld\n", + (long long)(int)offsetof(struct mgs_target_nidlist, mtn_nids)); + LASSERTF((int)sizeof(((struct mgs_target_nidlist *)0)->mtn_nids) == 4, "found %lld\n", + (long long)(int)sizeof(((struct mgs_target_nidlist *)0)->mtn_nids)); + LASSERTF((int)offsetof(struct mgs_target_nidlist, mtn_inline_list[0]) == 8, "found %lld\n", + (long long)(int)offsetof(struct mgs_target_nidlist, mtn_inline_list[0])); + LASSERTF((int)sizeof(((struct mgs_target_nidlist *)0)->mtn_inline_list[0]) == 64, "found %lld\n", + (long long)(int)sizeof(((struct mgs_target_nidlist *)0)->mtn_inline_list[0])); + BUILD_BUG_ON(NIDLIST_APPEND != 0x00000001); + BUILD_BUG_ON(NIDLIST_IN_BULK != 0x00000002); + BUILD_BUG_ON(NIDLIST_COMPRESSED != 0x00000004); + /* Checks for struct mgs_nidtbl_entry */ LASSERTF((int)sizeof(struct mgs_nidtbl_entry) == 24, "found %lld\n", (long long)(int)sizeof(struct mgs_nidtbl_entry)); -- 1.8.3.1