From 6940303aad5375bb2c83160f87c0cacf9129b5aa Mon Sep 17 00:00:00 2001 From: Mr NeilBrown Date: Mon, 11 May 2020 11:06:11 +1000 Subject: [PATCH] LU-10391 socklnd: prepare for new KSOCK_MSG type Various places in socklnd assume there are only two message type: KSOCK_MSG_NOOP and KSOCK_MSG_LNET. We will soon add another type to support a new lnet_hdr type with large addresses. So do some cleanup first: - get rid of ksock_lnet_msg - it doesn't add anything to lnet_hdr - separate out 'struct ksock_hdr'. We often want the size of this header, and instead request the offset of a field in ksock_msg. - introduce switch statements in a couple of places to handle the different types of ksock_msg. Test-Parameters: trivial Test-Parameters: serverversion=2.12 serverdistro=el7.9 testlist=runtests Test-Parameters: clientversion=2.12 testlist=runtests Signed-off-by: Mr NeilBrown Change-Id: Ibe484f76757c4100b8532cef659c3cc369b658ba Reviewed-on: https://review.whamcloud.com/43601 Reviewed-by: Chris Horn Reviewed-by: Serguei Smirnov Tested-by: jenkins Tested-by: Maloo Reviewed-by: James Simmons Reviewed-by: Oleg Drokin --- lnet/include/lnet/socklnd.h | 33 +++++++++--------- lnet/klnds/socklnd/socklnd_cb.c | 66 ++++++++++++++++++----------------- lnet/klnds/socklnd/socklnd_proto.c | 71 ++++++++++++++++++++++---------------- 3 files changed, 93 insertions(+), 77 deletions(-) diff --git a/lnet/include/lnet/socklnd.h b/lnet/include/lnet/socklnd.h index fed0d6e..519ccbf 100644 --- a/lnet/include/lnet/socklnd.h +++ b/lnet/include/lnet/socklnd.h @@ -48,28 +48,29 @@ struct ksock_hello_msg { __u32 kshm_ips[0]; /* IP addrs */ } __packed; -struct ksock_lnet_msg { - struct lnet_hdr ksnm_hdr; /* lnet hdr */ - - /* - * ksnm_payload is removed because of winnt compiler's limitation: - * zero-sized array can only be placed at the tail of [nested] - * structure definitions. lnet payload will be stored just after - * the body of structure struct ksock_lnet_msg - */ +struct ksock_msg_hdr { + __u32 ksh_type; /* type of socklnd message */ + __u32 ksh_csum; /* checksum if != 0 */ + __u64 ksh_zc_cookies[2]; /* Zero-Copy request/ACK + * cookie + */ } __packed; +#define KSOCK_MSG_NOOP 0xc0 /* empty */ +#define KSOCK_MSG_LNET 0xc1 /* lnet msg */ + struct ksock_msg { - __u32 ksm_type; /* type of socklnd message */ - __u32 ksm_csum; /* checksum if != 0 */ - __u64 ksm_zc_cookies[2]; /* Zero-Copy request/ACK cookie */ + struct ksock_msg_hdr ksm_kh; union { - struct ksock_lnet_msg lnetmsg; /* lnet message, it's empty if it's NOOP */ + /* case ksm_kh.ksh_type == KSOCK_MSG_NOOP */ + /* - nothing */ + /* case ksm_kh.ksh_type == KSOCK_MSG_LNET */ + struct lnet_hdr lnetmsg; } __packed ksm_u; } __packed; - -#define KSOCK_MSG_NOOP 0xc0 /* ksm_u empty */ -#define KSOCK_MSG_LNET 0xc1 /* lnet msg */ +#define ksm_type ksm_kh.ksh_type +#define ksm_csum ksm_kh.ksh_csum +#define ksm_zc_cookies ksm_kh.ksh_zc_cookies /* We need to know this number to parse hello msg from ksocklnd in * other LND (usocklnd, for example) */ diff --git a/lnet/klnds/socklnd/socklnd_cb.c b/lnet/klnds/socklnd/socklnd_cb.c index 558a24d..b098210 100644 --- a/lnet/klnds/socklnd/socklnd_cb.c +++ b/lnet/klnds/socklnd/socklnd_cb.c @@ -1080,14 +1080,15 @@ ksocknal_new_packet(struct ksock_conn *conn, int nob_to_skip) switch (conn->ksnc_proto->pro_version) { case KSOCK_PROTO_V2: case KSOCK_PROTO_V3: - conn->ksnc_rx_state = SOCKNAL_RX_KSM_HEADER; + conn->ksnc_rx_state = SOCKNAL_RX_KSM_HEADER; conn->ksnc_rx_iov = (struct kvec *)&conn->ksnc_rx_iov_space; - conn->ksnc_rx_iov[0].iov_base = (char *)&conn->ksnc_msg; + conn->ksnc_rx_iov[0].iov_base = (char *)&conn->ksnc_msg; - conn->ksnc_rx_nob_wanted = offsetof(struct ksock_msg, ksm_u); - conn->ksnc_rx_nob_left = offsetof(struct ksock_msg, ksm_u); - conn->ksnc_rx_iov[0].iov_len = offsetof(struct ksock_msg, ksm_u); - break; + conn->ksnc_rx_nob_wanted = sizeof(struct ksock_msg_hdr); + conn->ksnc_rx_nob_left = sizeof(struct ksock_msg_hdr); + conn->ksnc_rx_iov[0].iov_len = + sizeof(struct ksock_msg_hdr); + break; case KSOCK_PROTO_V1: /* Receiving bare struct lnet_hdr */ @@ -1196,16 +1197,6 @@ ksocknal_process_receive(struct ksock_conn *conn, __swab64s(&conn->ksnc_msg.ksm_zc_cookies[1]); } - if (conn->ksnc_msg.ksm_type != KSOCK_MSG_NOOP && - conn->ksnc_msg.ksm_type != KSOCK_MSG_LNET) { - CERROR("%s: Unknown message type: %x\n", - libcfs_idstr(&conn->ksnc_peer->ksnp_id), - conn->ksnc_msg.ksm_type); - ksocknal_new_packet(conn, 0); - ksocknal_close_conn_and_siblings(conn, -EPROTO); - return (-EPROTO); - } - if (conn->ksnc_msg.ksm_type == KSOCK_MSG_NOOP && conn->ksnc_msg.ksm_csum != 0 && /* has checksum */ conn->ksnc_msg.ksm_csum != conn->ksnc_rx_csum) { @@ -1240,25 +1231,36 @@ ksocknal_process_receive(struct ksock_conn *conn, } } - if (conn->ksnc_msg.ksm_type == KSOCK_MSG_NOOP) { + switch (conn->ksnc_msg.ksm_type) { + case KSOCK_MSG_NOOP: ksocknal_new_packet(conn, 0); return 0; /* NOOP is done and just return */ - } - conn->ksnc_rx_state = SOCKNAL_RX_LNET_HEADER; - conn->ksnc_rx_nob_wanted = sizeof(struct ksock_lnet_msg); - conn->ksnc_rx_nob_left = sizeof(struct ksock_lnet_msg); + case KSOCK_MSG_LNET: - conn->ksnc_rx_iov = (struct kvec *)&conn->ksnc_rx_iov_space; - conn->ksnc_rx_iov[0].iov_base = - (char *)&conn->ksnc_msg.ksm_u.lnetmsg; - conn->ksnc_rx_iov[0].iov_len = sizeof(struct ksock_lnet_msg); + conn->ksnc_rx_state = SOCKNAL_RX_LNET_HEADER; + conn->ksnc_rx_nob_wanted = sizeof(struct lnet_hdr); + conn->ksnc_rx_nob_left = sizeof(struct lnet_hdr); - conn->ksnc_rx_niov = 1; - conn->ksnc_rx_kiov = NULL; - conn->ksnc_rx_nkiov = 0; + conn->ksnc_rx_iov = conn->ksnc_rx_iov_space.iov; + conn->ksnc_rx_iov[0].iov_base = + (void *)&conn->ksnc_msg.ksm_u.lnetmsg; + conn->ksnc_rx_iov[0].iov_len = sizeof(struct lnet_hdr); + + conn->ksnc_rx_niov = 1; + conn->ksnc_rx_kiov = NULL; + conn->ksnc_rx_nkiov = 0; - goto again; /* read lnet header now */ + goto again; /* read lnet header now */ + + default: + CERROR("%s: Unknown message type: %x\n", + libcfs_idstr(&conn->ksnc_peer->ksnp_id), + conn->ksnc_msg.ksm_type); + ksocknal_new_packet(conn, 0); + ksocknal_close_conn_and_siblings(conn, -EPROTO); + return -EPROTO; + } case SOCKNAL_RX_LNET_HEADER: /* unpack message header */ @@ -1266,7 +1268,7 @@ ksocknal_process_receive(struct ksock_conn *conn, if ((conn->ksnc_peer->ksnp_id.pid & LNET_PID_USERFLAG) != 0) { /* Userspace peer_ni */ - lhdr = &conn->ksnc_msg.ksm_u.lnetmsg.ksnm_hdr; + lhdr = &conn->ksnc_msg.ksm_u.lnetmsg; id = &conn->ksnc_peer->ksnp_id; /* Substitute process ID assigned at connection time */ @@ -1278,7 +1280,7 @@ ksocknal_process_receive(struct ksock_conn *conn, ksocknal_conn_addref(conn); /* ++ref while parsing */ rc = lnet_parse(conn->ksnc_peer->ksnp_ni, - &conn->ksnc_msg.ksm_u.lnetmsg.ksnm_hdr, + &conn->ksnc_msg.ksm_u.lnetmsg, lnet_nid_to_nid4(&conn->ksnc_peer->ksnp_id.nid), conn, 0); if (rc < 0) { @@ -1315,7 +1317,7 @@ ksocknal_process_receive(struct ksock_conn *conn, if (rc == 0 && conn->ksnc_msg.ksm_zc_cookies[0] != 0) { LASSERT(conn->ksnc_proto != &ksocknal_protocol_v1x); - lhdr = &conn->ksnc_msg.ksm_u.lnetmsg.ksnm_hdr; + lhdr = &conn->ksnc_msg.ksm_u.lnetmsg; id = &conn->ksnc_peer->ksnp_id; rc = conn->ksnc_proto->pro_handle_zcreq( diff --git a/lnet/klnds/socklnd/socklnd_proto.c b/lnet/klnds/socklnd/socklnd_proto.c index d931ae8..954f9c0 100644 --- a/lnet/klnds/socklnd/socklnd_proto.c +++ b/lnet/klnds/socklnd/socklnd_proto.c @@ -281,14 +281,15 @@ ksocknal_match_tx(struct ksock_conn *conn, struct ksock_tx *tx, int nonblk) return SOCKNAL_MATCH_YES; #endif - if (tx == NULL || tx->tx_lnetmsg == NULL) { - /* noop packet */ - nob = offsetof(struct ksock_msg, ksm_u); - } else { - nob = tx->tx_lnetmsg->msg_len + - ((conn->ksnc_proto == &ksocknal_protocol_v1x) ? - sizeof(struct lnet_hdr) : sizeof(struct ksock_msg)); - } + if (tx == NULL || tx->tx_lnetmsg == NULL) { + /* noop packet */ + nob = sizeof(struct ksock_msg_hdr); + } else { + nob = tx->tx_lnetmsg->msg_len + + ((conn->ksnc_proto == &ksocknal_protocol_v1x) ? + 0 : sizeof(struct ksock_msg_hdr)) + + sizeof(struct lnet_hdr); + } /* default checking for typed connection */ switch (conn->ksnc_type) { @@ -320,10 +321,11 @@ ksocknal_match_tx_v3(struct ksock_conn *conn, struct ksock_tx *tx, int nonblk) { int nob; - if (tx == NULL || tx->tx_lnetmsg == NULL) - nob = offsetof(struct ksock_msg, ksm_u); - else - nob = tx->tx_lnetmsg->msg_len + sizeof(struct ksock_msg); + if (tx == NULL || tx->tx_lnetmsg == NULL) + nob = sizeof(struct ksock_msg_hdr); + else + nob = sizeof(struct ksock_msg_hdr) + sizeof(struct lnet_hdr) + + tx->tx_lnetmsg->msg_len; switch (conn->ksnc_type) { default: @@ -718,30 +720,41 @@ ksocknal_pack_msg_v1(struct ksock_tx *tx) static void ksocknal_pack_msg_v2(struct ksock_tx *tx) { - tx->tx_hdr.iov_base = (void *)&tx->tx_msg; + int hdr_size; - if (tx->tx_lnetmsg != NULL) { - LASSERT(tx->tx_msg.ksm_type != KSOCK_MSG_NOOP); - - tx->tx_msg.ksm_u.lnetmsg.ksnm_hdr = tx->tx_lnetmsg->msg_hdr; - tx->tx_hdr.iov_len = sizeof(struct ksock_msg); - tx->tx_resid = tx->tx_nob = sizeof(struct ksock_msg) + tx->tx_lnetmsg->msg_len; - } else { - LASSERT(tx->tx_msg.ksm_type == KSOCK_MSG_NOOP); + tx->tx_hdr.iov_base = (void *)&tx->tx_msg; - tx->tx_hdr.iov_len = offsetof(struct ksock_msg, - ksm_u.lnetmsg.ksnm_hdr); - tx->tx_resid = tx->tx_nob = offsetof(struct ksock_msg, ksm_u.lnetmsg.ksnm_hdr); - } - /* Don't checksum before start sending, because packet can be piggybacked with ACK */ + switch (tx->tx_msg.ksm_type) { + case KSOCK_MSG_LNET: + LASSERT(tx->tx_lnetmsg != NULL); + hdr_size = (sizeof(struct ksock_msg_hdr) + + sizeof(struct lnet_hdr)); + + tx->tx_msg.ksm_u.lnetmsg = tx->tx_lnetmsg->msg_hdr; + tx->tx_hdr.iov_len = hdr_size; + tx->tx_resid = tx->tx_nob = hdr_size + tx->tx_lnetmsg->msg_len; + break; + case KSOCK_MSG_NOOP: + LASSERT(tx->tx_lnetmsg == NULL); + hdr_size = sizeof(struct ksock_msg_hdr); + + tx->tx_hdr.iov_len = hdr_size; + tx->tx_resid = tx->tx_nob = hdr_size; + break; + default: + LASSERT(0); + } + /* Don't checksum before start sending, because packet can be + * piggybacked with ACK + */ } static void ksocknal_unpack_msg_v1(struct ksock_msg *msg) { - msg->ksm_csum = 0; - msg->ksm_type = KSOCK_MSG_LNET; - msg->ksm_zc_cookies[0] = msg->ksm_zc_cookies[1] = 0; + msg->ksm_csum = 0; + msg->ksm_type = KSOCK_MSG_LNET; + msg->ksm_zc_cookies[0] = msg->ksm_zc_cookies[1] = 0; } static void -- 1.8.3.1