LU-18986 mgs: new target registration protocol

Patch adds new target registration request format with
enhanced NIDs list handling. The idea is to don't overload
mgs_target_info with extra flags and fields for NID list
description but keep such information in new structure.

NIDs list is arrays of string always and can be send
in varios manners: inline buffer, bulk, compressed,
appended, etc.

It helps also to resolve compatibility issues.

Patch includes:
- new wire structure mgs_target_nidlist
- new possible RPC format with mgs_target_nidlist buffer
- new connect flag OBD_CONNECT_MGS_NIDLIST to replace
  obsoleted OBD_CONNECT_REQPORTAL removed in commit
  1.6.0-159-gd2d56f38da ("make HEAD from b_post_cmd3")
- corresponding swabber and wirecheck

Test-Parameters: testlist=runtests clientversion=EXA6.3.2
Test-Parameters: testlist=runtests serverversion=EXA6.3.2
Signed-off-by: Mikhail Pershin <mpershin@whamcloud.com>
Change-Id: I441de467a530137f76712273b9a5f814fdb562c1
Reviewed-on: https://review.whamcloud.com/c/fs/lustre-release/+/59205
Tested-by: Maloo <maloo@whamcloud.com>
Tested-by: jenkins <devops@whamcloud.com>
Reviewed-by: Andreas Dilger <adilger@whamcloud.com>
Reviewed-by: Alex Zhuravlev <bzzz@whamcloud.com>
Reviewed-by: James Simmons <jsimmons@infradead.org>
Reviewed-by: Oleg Drokin <green@whamcloud.com>

diff --git a/lustre/include/lustre_req_layout.h b/lustre/include/lustre_req_layout.h
index 8ac8f3a..c78c8ac 100644 (file)
--- a/lustre/include/lustre_req_layout.h
+++ b/lustre/include/lustre_req_layout.h
@@ -182,6 +182,7 @@ extern struct req_format RQF_SEC_CTX;
 extern struct req_format RQF_OBD_IDX_READ;
 /* MGS req_format */
 extern struct req_format RQF_MGS_TARGET_REG;
+extern struct req_format RQF_MGS_TARGET_REG_NIDLIST;
 extern struct req_format RQF_MGS_SET_INFO;
 extern struct req_format RQF_MGS_CONFIG_READ;
 /* fid/fld req_format */
@@ -366,6 +367,7 @@ extern struct req_msg_field RMF_LLOG_LOG_HDR;
 extern struct req_msg_field RMF_LLOGD_CONN_BODY;
 
 extern struct req_msg_field RMF_MGS_TARGET_INFO;
+extern struct req_msg_field RMF_MGS_TARGET_NIDLIST;
 extern struct req_msg_field RMF_MGS_SEND_PARAM;
 
 extern struct req_msg_field RMF_OST_BODY;

diff --git a/lustre/include/lustre_swab.h b/lustre/include/lustre_swab.h
index 033c120..2b9cd20 100644 (file)
--- a/lustre/include/lustre_swab.h
+++ b/lustre/include/lustre_swab.h
@@ -81,6 +81,7 @@ void lustre_swab_ldlm_lock_desc(struct ldlm_lock_desc *l);
 void lustre_swab_ldlm_request(struct ldlm_request *rq);
 void lustre_swab_ldlm_reply(struct ldlm_reply *r);
 void lustre_swab_mgs_target_info(struct mgs_target_info *oinfo);
+void lustre_swab_mgs_target_nidlist(struct mgs_target_nidlist *mtn);
 void lustre_swab_mgs_nidtbl_entry_header(struct mgs_nidtbl_entry *oinfo);
 void lustre_swab_mgs_nidtbl_entry_content(struct mgs_nidtbl_entry *oinfo);
 void lustre_swab_mgs_config_body(struct mgs_config_body *body);

diff --git a/lustre/include/uapi/linux/lustre/lustre_idl.h b/lustre/include/uapi/linux/lustre/lustre_idl.h
index f4c89c2..093ccc1 100644 (file)
--- a/lustre/include/uapi/linux/lustre/lustre_idl.h
+++ b/lustre/include/uapi/linux/lustre/lustre_idl.h
@@ -739,7 +739,7 @@ struct ptlrpc_body_v2 {
 #define OBD_CONNECT_GRANT                      0x8ULL /* fetch grant connect */
 #define OBD_CONNECT_SRVLOCK                   0x10ULL /* server lock for RPC */
 #define OBD_CONNECT_VERSION                   0x20ULL /* versions in OCD */
-#define OBD_CONNECT_REQPORTAL                 0x40ULL /* non-IO portal */
+#define OBD_CONNECT_MGS_NIDLIST                       0x40ULL /* MGS nidlist protocol */
 #define OBD_CONNECT_ACL                               0x80ULL /* access control list */
 #define OBD_CONNECT_XATTR                    0x100ULL /* extended attributes */
 #define OBD_CONNECT_LARGE_ACL                0x200ULL /* over 32 ACL entries */
@@ -925,8 +925,7 @@ struct ptlrpc_body_v2 {
                                OBD_CONNECT2_MIRROR_ID_FIX)
 
 #define OST_CONNECT_SUPPORTED  (OBD_CONNECT_SRVLOCK | OBD_CONNECT_GRANT | \
-                               OBD_CONNECT_REQPORTAL | OBD_CONNECT_VERSION | \
-                               OBD_CONNECT_INDEX | \
+                               OBD_CONNECT_VERSION | OBD_CONNECT_INDEX | \
                                OBD_CONNECT_BRW_SIZE | OBD_CONNECT_CANCELSET | \
                                OBD_CONNECT_AT | LRU_RESIZE_CONNECT_FLAG | \
                                OBD_CONNECT_CKSUM | OBD_CONNECT_VBR | \
@@ -2752,6 +2751,21 @@ struct mgs_target_info {
        char            mti_nidlist[][LNET_NIDSTR_SIZE];
 } __attribute__((packed));
 
+enum mgs_nidlist_flags {
+       NIDLIST_APPEND          = 0x00000001, /* append NIDs to nidtable */
+       NIDLIST_IN_BULK         = 0x00000002, /* nidlist in bulk */
+       NIDLIST_COMPRESSED      = 0x00000004, /* nidlist is compressed */
+};
+
+#define MTN_NIDSTR_SIZE LNET_NIDSTR_SIZE
+#define NIDLIST_SIZE(count) (MTN_NIDSTR_SIZE * count)
+
+struct mgs_target_nidlist {
+       __u32   mtn_flags; /* flags from mgs_nids_info_flags */
+       __u32   mtn_nids;  /* amount of nids in list */
+       char    mtn_inline_list[][MTN_NIDSTR_SIZE];
+} __attribute__((packed));
+
 struct mgs_nidtbl_entry {
        __u64           mne_version;    /* table version of this entry */
        __u32           mne_instance;   /* target instance # */

diff --git a/lustre/llite/llite_lib.c b/lustre/llite/llite_lib.c
index a0119fc..6c13a68 100644 (file)
--- a/lustre/llite/llite_lib.c
+++ b/lustre/llite/llite_lib.c
@@ -594,7 +594,7 @@ retry_connect:
 
        /* indicate OST features supported by this client */
        data->ocd_connect_flags = OBD_CONNECT_GRANT | OBD_CONNECT_VERSION |
-                                 OBD_CONNECT_REQPORTAL | OBD_CONNECT_BRW_SIZE |
+                                 OBD_CONNECT_BRW_SIZE |
                                  OBD_CONNECT_CANCELSET | OBD_CONNECT_FID |
                                  OBD_CONNECT_SRVLOCK |
                                  OBD_CONNECT_AT | OBD_CONNECT_OSS_CAPA |

diff --git a/lustre/lod/lod_lov.c b/lustre/lod/lod_lov.c
index 0c83bae..193b82c 100644 (file)
--- a/lustre/lod/lod_lov.c
+++ b/lustre/lod/lod_lov.c
@@ -166,7 +166,6 @@ int lod_add_device(const struct lu_env *env, struct lod_device *lod,
                                           OBD_CONNECT_LRU_RESIZE |
 #endif
                                           OBD_CONNECT_MDS |
-                                          OBD_CONNECT_REQPORTAL |
                                           OBD_CONNECT_SKIP_ORPHAN |
                                           OBD_CONNECT_FID |
                                           OBD_CONNECT_LVB_TYPE |

diff --git a/lustre/obdclass/lprocfs_status.c b/lustre/obdclass/lprocfs_status.c
index 259c1fc..8a02421 100644 (file)
--- a/lustre/obdclass/lprocfs_status.c
+++ b/lustre/obdclass/lprocfs_status.c
@@ -630,7 +630,7 @@ static const char *const obd_connect_names[] = {
        "write_grant",                  /* 0x04 */
        "server_lock",                  /* 0x10 */
        "version",                      /* 0x20 */
-       "request_portal",               /* 0x40 */
+       "mgs_nidlist",                  /* 0x40 */
        "acl",                          /* 0x80 */
        "xattr",                        /* 0x100 */
        "create_on_write",              /* 0x200 */

diff --git a/lustre/obdecho/echo_client.c b/lustre/obdecho/echo_client.c
index 0378c9b..801ab94 100644 (file)
--- a/lustre/obdecho/echo_client.c
+++ b/lustre/obdecho/echo_client.c
@@ -2464,7 +2464,7 @@ static int echo_client_setup(const struct lu_env *env,
                return -ENOMEM;
        }
 
-       ocd->ocd_connect_flags = OBD_CONNECT_VERSION | OBD_CONNECT_REQPORTAL |
+       ocd->ocd_connect_flags = OBD_CONNECT_VERSION |
                                 OBD_CONNECT_BRW_SIZE |
                                 OBD_CONNECT_GRANT | OBD_CONNECT_FULL20 |
                                 OBD_CONNECT_64BITHASH | OBD_CONNECT_LVB_TYPE |

diff --git a/lustre/osc/osc_request.c b/lustre/osc/osc_request.c
index 5f20e30..ab34b8c 100644 (file)
--- a/lustre/osc/osc_request.c
+++ b/lustre/osc/osc_request.c
@@ -3895,11 +3895,6 @@ static int osc_import_event(struct obd_device *obd, struct obd_import *imp,
                if (ocd->ocd_connect_flags & OBD_CONNECT_GRANT)
                        osc_init_grant(&obd->u.cli, ocd);
 
-               /* See bug 7198 */
-               if (ocd->ocd_connect_flags & OBD_CONNECT_REQPORTAL)
-                       imp->imp_client->cli_request_portal =
-                               OST_REQUEST_PORTAL;
-
                rc = obd_notify_observer(obd, obd, OBD_NOTIFY_OCD);
                break;
        }

diff --git a/lustre/ptlrpc/layout.c b/lustre/ptlrpc/layout.c
index 6924ea4..1458430 100644 (file)
--- a/lustre/ptlrpc/layout.c
+++ b/lustre/ptlrpc/layout.c
@@ -52,6 +52,12 @@ static const struct req_msg_field *mgs_target_info_only[] = {
        &RMF_MGS_TARGET_INFO
 };
 
+static const struct req_msg_field *mgs_target_info_nidlist[] = {
+       &RMF_PTLRPC_BODY,
+       &RMF_MGS_TARGET_INFO,
+       &RMF_MGS_TARGET_NIDLIST,
+};
+
 #if LUSTRE_VERSION_CODE < OBD_OCD_VERSION(2, 18, 53, 0)
 static const struct req_msg_field *mgs_set_info[] = {
        &RMF_PTLRPC_BODY,
@@ -799,6 +805,7 @@ static struct req_format *req_formats[] = {
        &RQF_OBD_IDX_READ,
        &RQF_SEC_CTX,
        &RQF_MGS_TARGET_REG,
+       &RQF_MGS_TARGET_REG_NIDLIST,
 #if LUSTRE_VERSION_CODE < OBD_OCD_VERSION(2, 18, 53, 0)
        &RQF_MGS_SET_INFO,
 #endif
@@ -964,6 +971,11 @@ struct req_msg_field RMF_MGS_TARGET_INFO =
                    lustre_swab_mgs_target_info, NULL);
 EXPORT_SYMBOL(RMF_MGS_TARGET_INFO);
 
+struct req_msg_field RMF_MGS_TARGET_NIDLIST =
+       DEFINE_MSGF("mgs_target_nidlist", 0, sizeof(struct mgs_target_nidlist),
+                   lustre_swab_mgs_target_nidlist, NULL);
+EXPORT_SYMBOL(RMF_MGS_TARGET_NIDLIST);
+
 #if LUSTRE_VERSION_CODE < OBD_OCD_VERSION(2, 18, 53, 0)
 struct req_msg_field RMF_MGS_SEND_PARAM =
        DEFINE_MSGF("mgs_send_param", 0,
@@ -1410,6 +1422,11 @@ struct req_format RQF_MGS_TARGET_REG =
                        mgs_target_info_only);
 EXPORT_SYMBOL(RQF_MGS_TARGET_REG);
 
+struct req_format RQF_MGS_TARGET_REG_NIDLIST =
+       DEFINE_REQ_FMT0("MGS_TARGET_REG_NIDLIST", mgs_target_info_nidlist,
+                       mgs_target_info_only);
+EXPORT_SYMBOL(RQF_MGS_TARGET_REG_NIDLIST);
+
 #if LUSTRE_VERSION_CODE < OBD_OCD_VERSION(2, 18, 53, 0)
 struct req_format RQF_MGS_SET_INFO =
        DEFINE_REQ_FMT0("MGS_SET_INFO", mgs_set_info,

diff --git a/lustre/ptlrpc/pack_generic.c b/lustre/ptlrpc/pack_generic.c
index 91e95ff..3df5e8e 100644 (file)
--- a/lustre/ptlrpc/pack_generic.c
+++ b/lustre/ptlrpc/pack_generic.c
@@ -2089,6 +2089,13 @@ void lustre_swab_mgs_target_info(struct mgs_target_info *mti)
                __swab64s(&mti->mti_nids[i]);
 }
 
+void lustre_swab_mgs_target_nidlist(struct mgs_target_nidlist *mtn)
+{
+       __swab32s(&mtn->mtn_flags);
+       __swab32s(&mtn->mtn_nids);
+       BUILD_BUG_ON(MTN_NIDSTR_SIZE != 64);
+}
+
 void lustre_swab_mgs_nidtbl_entry_header(struct mgs_nidtbl_entry *entry)
 {
        __swab64s(&entry->mne_version);

diff --git a/lustre/ptlrpc/wiretest.c b/lustre/ptlrpc/wiretest.c
index f1d2855..07da401 100644 (file)
--- a/lustre/ptlrpc/wiretest.c
+++ b/lustre/ptlrpc/wiretest.c
@@ -1272,8 +1272,8 @@ void lustre_assert_wire_constants(void)
                 OBD_CONNECT_SRVLOCK);
        LASSERTF(OBD_CONNECT_VERSION == 0x20ULL, "found 0x%.16llxULL\n",
                 OBD_CONNECT_VERSION);
-       LASSERTF(OBD_CONNECT_REQPORTAL == 0x40ULL, "found 0x%.16llxULL\n",
-                OBD_CONNECT_REQPORTAL);
+       LASSERTF(OBD_CONNECT_MGS_NIDLIST == 0x40ULL, "found 0x%.16llxULL\n",
+                OBD_CONNECT_MGS_NIDLIST);
        LASSERTF(OBD_CONNECT_ACL == 0x80ULL, "found 0x%.16llxULL\n",
                 OBD_CONNECT_ACL);
        LASSERTF(OBD_CONNECT_XATTR == 0x100ULL, "found 0x%.16llxULL\n",
@@ -4836,6 +4836,25 @@ void lustre_assert_wire_constants(void)
        LASSERTF((int)sizeof(((struct mgs_target_info *)0)->mti_nidlist[0]) == 64, "found %lld\n",
                 (long long)(int)sizeof(((struct mgs_target_info *)0)->mti_nidlist[0]));
 
+       /* Checks for struct mgs_target_nidlist */
+       LASSERTF((int)sizeof(struct mgs_target_nidlist) == 8, "found %lld\n",
+                (long long)(int)sizeof(struct mgs_target_nidlist));
+       LASSERTF((int)offsetof(struct mgs_target_nidlist, mtn_flags) == 0, "found %lld\n",
+                (long long)(int)offsetof(struct mgs_target_nidlist, mtn_flags));
+       LASSERTF((int)sizeof(((struct mgs_target_nidlist *)0)->mtn_flags) == 4, "found %lld\n",
+                (long long)(int)sizeof(((struct mgs_target_nidlist *)0)->mtn_flags));
+       LASSERTF((int)offsetof(struct mgs_target_nidlist, mtn_nids) == 4, "found %lld\n",
+                (long long)(int)offsetof(struct mgs_target_nidlist, mtn_nids));
+       LASSERTF((int)sizeof(((struct mgs_target_nidlist *)0)->mtn_nids) == 4, "found %lld\n",
+                (long long)(int)sizeof(((struct mgs_target_nidlist *)0)->mtn_nids));
+       LASSERTF((int)offsetof(struct mgs_target_nidlist, mtn_inline_list[0]) == 8, "found %lld\n",
+                (long long)(int)offsetof(struct mgs_target_nidlist, mtn_inline_list[0]));
+       LASSERTF((int)sizeof(((struct mgs_target_nidlist *)0)->mtn_inline_list[0]) == 64, "found %lld\n",
+                (long long)(int)sizeof(((struct mgs_target_nidlist *)0)->mtn_inline_list[0]));
+       BUILD_BUG_ON(NIDLIST_APPEND != 0x00000001);
+       BUILD_BUG_ON(NIDLIST_IN_BULK != 0x00000002);
+       BUILD_BUG_ON(NIDLIST_COMPRESSED != 0x00000004);
+
        /* Checks for struct mgs_nidtbl_entry */
        LASSERTF((int)sizeof(struct mgs_nidtbl_entry) == 24, "found %lld\n",
                 (long long)(int)sizeof(struct mgs_nidtbl_entry));

diff --git a/lustre/utils/wirecheck.c b/lustre/utils/wirecheck.c
index 8a1f670..00e3a30 100644 (file)
--- a/lustre/utils/wirecheck.c
+++ b/lustre/utils/wirecheck.c
@@ -614,7 +614,7 @@ check_obd_connect_data(void)
        CHECK_DEFINE_64X(OBD_CONNECT_GRANT);
        CHECK_DEFINE_64X(OBD_CONNECT_SRVLOCK);
        CHECK_DEFINE_64X(OBD_CONNECT_VERSION);
-       CHECK_DEFINE_64X(OBD_CONNECT_REQPORTAL);
+       CHECK_DEFINE_64X(OBD_CONNECT_MGS_NIDLIST);
        CHECK_DEFINE_64X(OBD_CONNECT_ACL);
        CHECK_DEFINE_64X(OBD_CONNECT_XATTR);
        CHECK_DEFINE_64X(OBD_CONNECT_LARGE_ACL);
@@ -2252,6 +2252,20 @@ check_mgs_target_info(void)
 }
 
 static void
+check_mgs_target_nidlist(void)
+{
+       BLANK_LINE();
+       CHECK_STRUCT(mgs_target_nidlist);
+       CHECK_MEMBER(mgs_target_nidlist, mtn_flags);
+       CHECK_MEMBER(mgs_target_nidlist, mtn_nids);
+       CHECK_MEMBER(mgs_target_nidlist, mtn_inline_list[0]);
+
+       CHECK_CVALUE_X(NIDLIST_APPEND);
+       CHECK_CVALUE_X(NIDLIST_IN_BULK);
+       CHECK_CVALUE_X(NIDLIST_COMPRESSED);
+}
+
+static void
 check_mgs_nidtbl_entry(void)
 {
        BLANK_LINE();
@@ -3788,6 +3802,7 @@ main(int argc, char **argv)
        check_quota_body();
 #endif /* !HAVE_NATIVE_LINUX_CLIENT */
        check_mgs_target_info();
+       check_mgs_target_nidlist();
        check_mgs_nidtbl_entry();
        check_mgs_config_body();
        check_mgs_config_res();

diff --git a/lustre/utils/wiretest.c b/lustre/utils/wiretest.c
index 6019ecf..07679c9 100644 (file)
--- a/lustre/utils/wiretest.c
+++ b/lustre/utils/wiretest.c
@@ -1315,8 +1315,8 @@ void lustre_assert_wire_constants(void)
                 OBD_CONNECT_SRVLOCK);
        LASSERTF(OBD_CONNECT_VERSION == 0x20ULL, "found 0x%.16llxULL\n",
                 OBD_CONNECT_VERSION);
-       LASSERTF(OBD_CONNECT_REQPORTAL == 0x40ULL, "found 0x%.16llxULL\n",
-                OBD_CONNECT_REQPORTAL);
+       LASSERTF(OBD_CONNECT_MGS_NIDLIST == 0x40ULL, "found 0x%.16llxULL\n",
+                OBD_CONNECT_MGS_NIDLIST);
        LASSERTF(OBD_CONNECT_ACL == 0x80ULL, "found 0x%.16llxULL\n",
                 OBD_CONNECT_ACL);
        LASSERTF(OBD_CONNECT_XATTR == 0x100ULL, "found 0x%.16llxULL\n",
@@ -4881,6 +4881,25 @@ void lustre_assert_wire_constants(void)
                 (long long)(int)sizeof(((struct mgs_target_info *)0)->mti_nidlist[0]));
 #endif /* HAVE_SERVER_SUPPORT */
 
+       /* Checks for struct mgs_target_nidlist */
+       LASSERTF((int)sizeof(struct mgs_target_nidlist) == 8, "found %lld\n",
+                (long long)(int)sizeof(struct mgs_target_nidlist));
+       LASSERTF((int)offsetof(struct mgs_target_nidlist, mtn_flags) == 0, "found %lld\n",
+                (long long)(int)offsetof(struct mgs_target_nidlist, mtn_flags));
+       LASSERTF((int)sizeof(((struct mgs_target_nidlist *)0)->mtn_flags) == 4, "found %lld\n",
+                (long long)(int)sizeof(((struct mgs_target_nidlist *)0)->mtn_flags));
+       LASSERTF((int)offsetof(struct mgs_target_nidlist, mtn_nids) == 4, "found %lld\n",
+                (long long)(int)offsetof(struct mgs_target_nidlist, mtn_nids));
+       LASSERTF((int)sizeof(((struct mgs_target_nidlist *)0)->mtn_nids) == 4, "found %lld\n",
+                (long long)(int)sizeof(((struct mgs_target_nidlist *)0)->mtn_nids));
+       LASSERTF((int)offsetof(struct mgs_target_nidlist, mtn_inline_list[0]) == 8, "found %lld\n",
+                (long long)(int)offsetof(struct mgs_target_nidlist, mtn_inline_list[0]));
+       LASSERTF((int)sizeof(((struct mgs_target_nidlist *)0)->mtn_inline_list[0]) == 64, "found %lld\n",
+                (long long)(int)sizeof(((struct mgs_target_nidlist *)0)->mtn_inline_list[0]));
+       BUILD_BUG_ON(NIDLIST_APPEND != 0x00000001);
+       BUILD_BUG_ON(NIDLIST_IN_BULK != 0x00000002);
+       BUILD_BUG_ON(NIDLIST_COMPRESSED != 0x00000004);
+
        /* Checks for struct mgs_nidtbl_entry */
        LASSERTF((int)sizeof(struct mgs_nidtbl_entry) == 24, "found %lld\n",
                 (long long)(int)sizeof(struct mgs_nidtbl_entry));