LU-10391 socklnd: prepare for new KSOCK_MSG type

Various places in socklnd assume there are only two message type:
KSOCK_MSG_NOOP and KSOCK_MSG_LNET.  We will soon add another type to
support a new lnet_hdr type with large addresses.
So do some cleanup first:

- get rid of ksock_lnet_msg - it doesn't add anything to lnet_hdr
- separate out 'struct ksock_hdr'.  We often want the size of this
  header, and instead request the offset of a field in ksock_msg.
- introduce switch statements in a couple of places to handle the
  different types of ksock_msg.

Test-Parameters: trivial
Test-Parameters: serverversion=2.12 serverdistro=el7.9 testlist=runtests
Test-Parameters: clientversion=2.12 testlist=runtests
Signed-off-by: Mr NeilBrown <neilb@suse.de>
Change-Id: Ibe484f76757c4100b8532cef659c3cc369b658ba
Reviewed-on: https://review.whamcloud.com/43601
Reviewed-by: Chris Horn <chris.horn@hpe.com>
Reviewed-by: Serguei Smirnov <ssmirnov@whamcloud.com>
Tested-by: jenkins <devops@whamcloud.com>
Tested-by: Maloo <maloo@whamcloud.com>
Reviewed-by: James Simmons <jsimmons@infradead.org>
Reviewed-by: Oleg Drokin <green@whamcloud.com>

diff --git a/lnet/include/lnet/socklnd.h b/lnet/include/lnet/socklnd.h
index fed0d6e..519ccbf 100644 (file)
--- a/lnet/include/lnet/socklnd.h
+++ b/lnet/include/lnet/socklnd.h
@@ -48,28 +48,29 @@ struct ksock_hello_msg {
        __u32                   kshm_ips[0];    /* IP addrs */
 } __packed;
 
-struct ksock_lnet_msg {
-       struct lnet_hdr         ksnm_hdr;       /* lnet hdr */
-
-       /*
-        * ksnm_payload is removed because of winnt compiler's limitation:
-        * zero-sized array can only be placed at the tail of [nested]
-        * structure definitions. lnet payload will be stored just after
-        * the body of structure struct ksock_lnet_msg
-        */
+struct ksock_msg_hdr {
+       __u32                   ksh_type;       /* type of socklnd message */
+       __u32                   ksh_csum;       /* checksum if != 0 */
+       __u64                   ksh_zc_cookies[2]; /* Zero-Copy request/ACK
+                                                   * cookie
+                                                   */
 } __packed;
 
+#define KSOCK_MSG_NOOP         0xc0            /* empty */
+#define KSOCK_MSG_LNET         0xc1            /* lnet msg */
+
 struct ksock_msg {
-       __u32                   ksm_type;       /* type of socklnd message */
-       __u32                   ksm_csum;       /* checksum if != 0 */
-       __u64                   ksm_zc_cookies[2]; /* Zero-Copy request/ACK cookie */
+       struct ksock_msg_hdr    ksm_kh;
        union {
-               struct ksock_lnet_msg lnetmsg;  /* lnet message, it's empty if it's NOOP */
+               /* case ksm_kh.ksh_type == KSOCK_MSG_NOOP */
+               /* - nothing */
+               /* case ksm_kh.ksh_type == KSOCK_MSG_LNET */
+               struct lnet_hdr lnetmsg;
        } __packed ksm_u;
 } __packed;
-
-#define KSOCK_MSG_NOOP         0xc0            /* ksm_u empty */
-#define KSOCK_MSG_LNET         0xc1            /* lnet msg */
+#define ksm_type ksm_kh.ksh_type
+#define ksm_csum ksm_kh.ksh_csum
+#define ksm_zc_cookies ksm_kh.ksh_zc_cookies
 
 /* We need to know this number to parse hello msg from ksocklnd in
  * other LND (usocklnd, for example) */

diff --git a/lnet/klnds/socklnd/socklnd_cb.c b/lnet/klnds/socklnd/socklnd_cb.c
index 558a24d..b098210 100644 (file)
--- a/lnet/klnds/socklnd/socklnd_cb.c
+++ b/lnet/klnds/socklnd/socklnd_cb.c
@@ -1080,14 +1080,15 @@ ksocknal_new_packet(struct ksock_conn *conn, int nob_to_skip)
                switch (conn->ksnc_proto->pro_version) {
                case  KSOCK_PROTO_V2:
                case  KSOCK_PROTO_V3:
-                        conn->ksnc_rx_state = SOCKNAL_RX_KSM_HEADER;
+                       conn->ksnc_rx_state = SOCKNAL_RX_KSM_HEADER;
                        conn->ksnc_rx_iov = (struct kvec *)&conn->ksnc_rx_iov_space;
-                        conn->ksnc_rx_iov[0].iov_base = (char *)&conn->ksnc_msg;
+                       conn->ksnc_rx_iov[0].iov_base = (char *)&conn->ksnc_msg;
 
-                       conn->ksnc_rx_nob_wanted = offsetof(struct ksock_msg, ksm_u);
-                       conn->ksnc_rx_nob_left = offsetof(struct ksock_msg, ksm_u);
-                       conn->ksnc_rx_iov[0].iov_len  = offsetof(struct ksock_msg, ksm_u);
-                        break;
+                       conn->ksnc_rx_nob_wanted = sizeof(struct ksock_msg_hdr);
+                       conn->ksnc_rx_nob_left = sizeof(struct ksock_msg_hdr);
+                       conn->ksnc_rx_iov[0].iov_len =
+                               sizeof(struct ksock_msg_hdr);
+                       break;
 
                case KSOCK_PROTO_V1:
                        /* Receiving bare struct lnet_hdr */
@@ -1196,16 +1197,6 @@ ksocknal_process_receive(struct ksock_conn *conn,
                        __swab64s(&conn->ksnc_msg.ksm_zc_cookies[1]);
                }
 
-               if (conn->ksnc_msg.ksm_type != KSOCK_MSG_NOOP &&
-                   conn->ksnc_msg.ksm_type != KSOCK_MSG_LNET) {
-                       CERROR("%s: Unknown message type: %x\n",
-                              libcfs_idstr(&conn->ksnc_peer->ksnp_id),
-                              conn->ksnc_msg.ksm_type);
-                       ksocknal_new_packet(conn, 0);
-                       ksocknal_close_conn_and_siblings(conn, -EPROTO);
-                       return (-EPROTO);
-               }
-
                if (conn->ksnc_msg.ksm_type == KSOCK_MSG_NOOP &&
                    conn->ksnc_msg.ksm_csum != 0 &&     /* has checksum */
                    conn->ksnc_msg.ksm_csum != conn->ksnc_rx_csum) {
@@ -1240,25 +1231,36 @@ ksocknal_process_receive(struct ksock_conn *conn,
                        }
                }
 
-               if (conn->ksnc_msg.ksm_type == KSOCK_MSG_NOOP) {
+               switch (conn->ksnc_msg.ksm_type) {
+               case KSOCK_MSG_NOOP:
                        ksocknal_new_packet(conn, 0);
                        return 0;       /* NOOP is done and just return */
-               }
 
-               conn->ksnc_rx_state = SOCKNAL_RX_LNET_HEADER;
-               conn->ksnc_rx_nob_wanted = sizeof(struct ksock_lnet_msg);
-               conn->ksnc_rx_nob_left = sizeof(struct ksock_lnet_msg);
+               case KSOCK_MSG_LNET:
 
-               conn->ksnc_rx_iov = (struct kvec *)&conn->ksnc_rx_iov_space;
-               conn->ksnc_rx_iov[0].iov_base =
-                       (char *)&conn->ksnc_msg.ksm_u.lnetmsg;
-               conn->ksnc_rx_iov[0].iov_len  = sizeof(struct ksock_lnet_msg);
+                       conn->ksnc_rx_state = SOCKNAL_RX_LNET_HEADER;
+                       conn->ksnc_rx_nob_wanted = sizeof(struct lnet_hdr);
+                       conn->ksnc_rx_nob_left = sizeof(struct lnet_hdr);
 
-               conn->ksnc_rx_niov = 1;
-               conn->ksnc_rx_kiov = NULL;
-               conn->ksnc_rx_nkiov = 0;
+                       conn->ksnc_rx_iov = conn->ksnc_rx_iov_space.iov;
+                       conn->ksnc_rx_iov[0].iov_base =
+                               (void *)&conn->ksnc_msg.ksm_u.lnetmsg;
+                       conn->ksnc_rx_iov[0].iov_len = sizeof(struct lnet_hdr);
+
+                       conn->ksnc_rx_niov = 1;
+                       conn->ksnc_rx_kiov = NULL;
+                       conn->ksnc_rx_nkiov = 0;
 
-               goto again;     /* read lnet header now */
+                       goto again;     /* read lnet header now */
+
+               default:
+                       CERROR("%s: Unknown message type: %x\n",
+                              libcfs_idstr(&conn->ksnc_peer->ksnp_id),
+                              conn->ksnc_msg.ksm_type);
+                       ksocknal_new_packet(conn, 0);
+                       ksocknal_close_conn_and_siblings(conn, -EPROTO);
+                       return -EPROTO;
+               }
 
        case SOCKNAL_RX_LNET_HEADER:
                /* unpack message header */
@@ -1266,7 +1268,7 @@ ksocknal_process_receive(struct ksock_conn *conn,
 
                if ((conn->ksnc_peer->ksnp_id.pid & LNET_PID_USERFLAG) != 0) {
                        /* Userspace peer_ni */
-                       lhdr = &conn->ksnc_msg.ksm_u.lnetmsg.ksnm_hdr;
+                       lhdr = &conn->ksnc_msg.ksm_u.lnetmsg;
                        id = &conn->ksnc_peer->ksnp_id;
 
                        /* Substitute process ID assigned at connection time */
@@ -1278,7 +1280,7 @@ ksocknal_process_receive(struct ksock_conn *conn,
                ksocknal_conn_addref(conn);     /* ++ref while parsing */
 
                rc = lnet_parse(conn->ksnc_peer->ksnp_ni,
-                               &conn->ksnc_msg.ksm_u.lnetmsg.ksnm_hdr,
+                               &conn->ksnc_msg.ksm_u.lnetmsg,
                                lnet_nid_to_nid4(&conn->ksnc_peer->ksnp_id.nid),
                                conn, 0);
                if (rc < 0) {
@@ -1315,7 +1317,7 @@ ksocknal_process_receive(struct ksock_conn *conn,
                if (rc == 0 && conn->ksnc_msg.ksm_zc_cookies[0] != 0) {
                        LASSERT(conn->ksnc_proto != &ksocknal_protocol_v1x);
 
-                       lhdr = &conn->ksnc_msg.ksm_u.lnetmsg.ksnm_hdr;
+                       lhdr = &conn->ksnc_msg.ksm_u.lnetmsg;
                        id = &conn->ksnc_peer->ksnp_id;
 
                        rc = conn->ksnc_proto->pro_handle_zcreq(

diff --git a/lnet/klnds/socklnd/socklnd_proto.c b/lnet/klnds/socklnd/socklnd_proto.c
index d931ae8..954f9c0 100644 (file)
--- a/lnet/klnds/socklnd/socklnd_proto.c
+++ b/lnet/klnds/socklnd/socklnd_proto.c
@@ -281,14 +281,15 @@ ksocknal_match_tx(struct ksock_conn *conn, struct ksock_tx *tx, int nonblk)
                 return SOCKNAL_MATCH_YES;
 #endif
 
-        if (tx == NULL || tx->tx_lnetmsg == NULL) {
-                /* noop packet */
-               nob = offsetof(struct ksock_msg, ksm_u);
-        } else {
-                nob = tx->tx_lnetmsg->msg_len +
-                      ((conn->ksnc_proto == &ksocknal_protocol_v1x) ?
-                      sizeof(struct lnet_hdr) : sizeof(struct ksock_msg));
-        }
+       if (tx == NULL || tx->tx_lnetmsg == NULL) {
+               /* noop packet */
+               nob = sizeof(struct ksock_msg_hdr);
+       } else {
+               nob = tx->tx_lnetmsg->msg_len +
+                       ((conn->ksnc_proto == &ksocknal_protocol_v1x) ?
+                        0 : sizeof(struct ksock_msg_hdr)) +
+                       sizeof(struct lnet_hdr);
+       }
 
         /* default checking for typed connection */
         switch (conn->ksnc_type) {
@@ -320,10 +321,11 @@ ksocknal_match_tx_v3(struct ksock_conn *conn, struct ksock_tx *tx, int nonblk)
 {
         int nob;
 
-        if (tx == NULL || tx->tx_lnetmsg == NULL)
-               nob = offsetof(struct ksock_msg, ksm_u);
-        else
-               nob = tx->tx_lnetmsg->msg_len + sizeof(struct ksock_msg);
+       if (tx == NULL || tx->tx_lnetmsg == NULL)
+               nob = sizeof(struct ksock_msg_hdr);
+       else
+               nob = sizeof(struct ksock_msg_hdr) + sizeof(struct lnet_hdr) +
+                       tx->tx_lnetmsg->msg_len;
 
         switch (conn->ksnc_type) {
         default:
@@ -718,30 +720,41 @@ ksocknal_pack_msg_v1(struct ksock_tx *tx)
 static void
 ksocknal_pack_msg_v2(struct ksock_tx *tx)
 {
-       tx->tx_hdr.iov_base = (void *)&tx->tx_msg;
+       int hdr_size;
 
-        if (tx->tx_lnetmsg != NULL) {
-                LASSERT(tx->tx_msg.ksm_type != KSOCK_MSG_NOOP);
-
-                tx->tx_msg.ksm_u.lnetmsg.ksnm_hdr = tx->tx_lnetmsg->msg_hdr;
-               tx->tx_hdr.iov_len = sizeof(struct ksock_msg);
-               tx->tx_resid = tx->tx_nob = sizeof(struct ksock_msg) + tx->tx_lnetmsg->msg_len;
-        } else {
-                LASSERT(tx->tx_msg.ksm_type == KSOCK_MSG_NOOP);
+       tx->tx_hdr.iov_base = (void *)&tx->tx_msg;
 
-               tx->tx_hdr.iov_len = offsetof(struct ksock_msg,
-                                             ksm_u.lnetmsg.ksnm_hdr);
-               tx->tx_resid = tx->tx_nob = offsetof(struct ksock_msg,  ksm_u.lnetmsg.ksnm_hdr);
-        }
-        /* Don't checksum before start sending, because packet can be piggybacked with ACK */
+       switch (tx->tx_msg.ksm_type) {
+       case KSOCK_MSG_LNET:
+               LASSERT(tx->tx_lnetmsg != NULL);
+               hdr_size = (sizeof(struct ksock_msg_hdr) +
+                               sizeof(struct lnet_hdr));
+
+               tx->tx_msg.ksm_u.lnetmsg = tx->tx_lnetmsg->msg_hdr;
+               tx->tx_hdr.iov_len = hdr_size;
+               tx->tx_resid = tx->tx_nob = hdr_size + tx->tx_lnetmsg->msg_len;
+               break;
+       case KSOCK_MSG_NOOP:
+               LASSERT(tx->tx_lnetmsg == NULL);
+               hdr_size = sizeof(struct ksock_msg_hdr);
+
+               tx->tx_hdr.iov_len = hdr_size;
+               tx->tx_resid = tx->tx_nob = hdr_size;
+               break;
+       default:
+               LASSERT(0);
+       }
+       /* Don't checksum before start sending, because packet can be
+        * piggybacked with ACK
+        */
 }
 
 static void
 ksocknal_unpack_msg_v1(struct ksock_msg *msg)
 {
-        msg->ksm_csum           = 0;
-        msg->ksm_type           = KSOCK_MSG_LNET;
-        msg->ksm_zc_cookies[0]  = msg->ksm_zc_cookies[1]  = 0;
+       msg->ksm_csum           = 0;
+       msg->ksm_type           = KSOCK_MSG_LNET;
+       msg->ksm_zc_cookies[0]  = msg->ksm_zc_cookies[1]  = 0;
 }
 
 static void