From 59fbfa5f3fa925f676636f1b78c986d05cd295bc Mon Sep 17 00:00:00 2001 From: Alex Zhuravlev Date: Thu, 27 Oct 2016 21:59:03 +0300 Subject: [PATCH] LU-8573 lnet: Revert LU-7650 patches These patches are causing LU-8573 Revert "LU-7650 o2iblnd: Put back work queue check previously removed" This reverts commit bde1da1ec098450f40887587b0a46c9eb86a4f6c. Revert "LU-7650 o2iblnd: handle mixed page size configurations." This reverts commit 399a5ac1fc73343c69e0fd737032adf5329df1b2. Change-Id: I4517fab64ac5b1023e615874f58b1bb4902e8c43 Signed-off-by: Alex Zhuravlev Reviewed-on: http://review.whamcloud.com/23439 Tested-by: Jenkins Reviewed-by: Oleg Drokin Tested-by: Oleg Drokin --- lnet/klnds/o2iblnd/o2iblnd.c | 17 +++++++-------- lnet/klnds/o2iblnd/o2iblnd.h | 13 ++++++------ lnet/klnds/o2iblnd/o2iblnd_cb.c | 47 ++++++++++++++++++----------------------- 3 files changed, 35 insertions(+), 42 deletions(-) diff --git a/lnet/klnds/o2iblnd/o2iblnd.c b/lnet/klnds/o2iblnd/o2iblnd.c index b1b3239..c00ae83 100644 --- a/lnet/klnds/o2iblnd/o2iblnd.c +++ b/lnet/klnds/o2iblnd/o2iblnd.c @@ -131,7 +131,6 @@ static int kiblnd_unpack_rd(kib_msg_t *msg, int flip) { kib_rdma_desc_t *rd; - int msg_size; int nob; int n; int i; @@ -148,7 +147,14 @@ kiblnd_unpack_rd(kib_msg_t *msg, int flip) __swab32s(&rd->rd_nfrags); } - n = rd->rd_nfrags; + n = rd->rd_nfrags; + + if (n <= 0 || n > IBLND_MAX_RDMA_FRAGS) { + CERROR("Bad nfrags: %d, should be 0 < n <= %d\n", + n, IBLND_MAX_RDMA_FRAGS); + return 1; + } + nob = offsetof (kib_msg_t, ibm_u) + kiblnd_rd_msg_size(rd, msg->ibm_type, n); @@ -158,13 +164,6 @@ kiblnd_unpack_rd(kib_msg_t *msg, int flip) return 1; } - msg_size = kiblnd_rd_size(rd); - if (msg_size <= 0 || msg_size > LNET_MAX_PAYLOAD) { - CERROR("Bad msg_size: %d, should be 0 < n <= %d\n", - msg_size, LNET_MAX_PAYLOAD); - return 1; - } - if (!flip) return 0; diff --git a/lnet/klnds/o2iblnd/o2iblnd.h b/lnet/klnds/o2iblnd/o2iblnd.h index 37896b3..7d0d411 100644 --- a/lnet/klnds/o2iblnd/o2iblnd.h +++ b/lnet/klnds/o2iblnd/o2iblnd.h @@ -135,9 +135,8 @@ extern kib_tunables_t kiblnd_tunables; #define IBLND_OOB_CAPABLE(v) ((v) != IBLND_MSG_VERSION_1) #define IBLND_OOB_MSGS(v) (IBLND_OOB_CAPABLE(v) ? 2 : 0) -#define IBLND_FRAG_SHIFT (PAGE_SHIFT - 12) /* frag size on wire is in 4K units */ -#define IBLND_MSG_SIZE (4 << 10) /* max size of queued messages (inc hdr) */ -#define IBLND_MAX_RDMA_FRAGS (LNET_MAX_PAYLOAD >> 12)/* max # of fragments supported in 4K size */ +#define IBLND_MSG_SIZE (4<<10) /* max size of queued messages (inc hdr) */ +#define IBLND_MAX_RDMA_FRAGS LNET_MAX_IOV /* max # of fragments supported */ /************************/ /* derived constants... */ @@ -157,8 +156,8 @@ extern kib_tunables_t kiblnd_tunables; /* WRs and CQEs (per connection) */ #define IBLND_RECV_WRS(c) IBLND_RX_MSGS(c) #define IBLND_SEND_WRS(c) \ - (((c->ibc_max_frags + 1) << IBLND_FRAG_SHIFT) * \ - kiblnd_concurrent_sends(c->ibc_version, c->ibc_peer->ibp_ni)) + ((c->ibc_max_frags + 1) * kiblnd_concurrent_sends(c->ibc_version, \ + c->ibc_peer->ibp_ni)) #define IBLND_CQ_ENTRIES(c) (IBLND_RECV_WRS(c) + IBLND_SEND_WRS(c)) struct kib_hca_dev; @@ -785,14 +784,14 @@ kiblnd_cfg_rdma_frags(struct lnet_ni *ni) tunables = &ni->ni_lnd_tunables->lt_tun_u.lt_o2ib; mod = tunables->lnd_map_on_demand; - return mod != 0 ? mod : IBLND_MAX_RDMA_FRAGS >> IBLND_FRAG_SHIFT; + return mod != 0 ? mod : IBLND_MAX_RDMA_FRAGS; } static inline int kiblnd_rdma_frags(int version, struct lnet_ni *ni) { return version == IBLND_MSG_VERSION_1 ? - (IBLND_MAX_RDMA_FRAGS >> IBLND_FRAG_SHIFT) : + IBLND_MAX_RDMA_FRAGS : kiblnd_cfg_rdma_frags(ni); } diff --git a/lnet/klnds/o2iblnd/o2iblnd_cb.c b/lnet/klnds/o2iblnd/o2iblnd_cb.c index d0823c85..b1277f4 100644 --- a/lnet/klnds/o2iblnd/o2iblnd_cb.c +++ b/lnet/klnds/o2iblnd/o2iblnd_cb.c @@ -763,6 +763,7 @@ __must_hold(&conn->ibc_lock) LASSERT(tx->tx_queued); /* We rely on this for QP sizing */ LASSERT(tx->tx_nwrq > 0); + LASSERT(tx->tx_nwrq <= 1 + conn->ibc_max_frags); LASSERT(credit == 0 || credit == 1); LASSERT(conn->ibc_outstanding_credits >= 0); @@ -1067,14 +1068,6 @@ kiblnd_init_rdma(kib_conn_t *conn, kib_tx_t *tx, int type, LASSERT (type == IBLND_MSG_GET_DONE || type == IBLND_MSG_PUT_DONE); - if (kiblnd_rd_size(srcrd) > conn->ibc_max_frags << PAGE_SHIFT) { - CERROR("RDMA is too large for peer %s (%d), src size: %d dst size: %d\n", - libcfs_nid2str(conn->ibc_peer->ibp_nid), - conn->ibc_max_frags << PAGE_SHIFT, - kiblnd_rd_size(srcrd), kiblnd_rd_size(dstrd)); - GOTO(too_big, rc = -EMSGSIZE); - } - srcidx = dstidx = 0; while (resid > 0) { @@ -1090,11 +1083,11 @@ kiblnd_init_rdma(kib_conn_t *conn, kib_tx_t *tx, int type, break; } - if (tx->tx_nwrq >= IBLND_MAX_RDMA_FRAGS) { + if (tx->tx_nwrq >= conn->ibc_max_frags) { CERROR("RDMA has too many fragments for peer %s (%d), " "src idx/frags: %d/%d dst idx/frags: %d/%d\n", libcfs_nid2str(conn->ibc_peer->ibp_nid), - IBLND_MAX_RDMA_FRAGS, + conn->ibc_max_frags, srcidx, srcrd->rd_nfrags, dstidx, dstrd->rd_nfrags); rc = -EMSGSIZE; @@ -1135,7 +1128,7 @@ kiblnd_init_rdma(kib_conn_t *conn, kib_tx_t *tx, int type, wrq++; sge++; } -too_big: + if (rc < 0) /* no RDMA if completing with failure */ tx->tx_nwrq = 0; @@ -2231,7 +2224,6 @@ kiblnd_passive_connect(struct rdma_cm_id *cmid, void *priv, int priv_nob) kib_rej_t rej; int version = IBLND_MSG_VERSION; unsigned long flags; - int max_frags; int rc; struct sockaddr_in *peer_addr; LASSERT (!in_interrupt()); @@ -2335,23 +2327,26 @@ kiblnd_passive_connect(struct rdma_cm_id *cmid, void *priv, int priv_nob) goto failed; } - max_frags = reqmsg->ibm_u.connparams.ibcp_max_frags >> IBLND_FRAG_SHIFT; - if (max_frags > kiblnd_rdma_frags(version, ni)) { + if (reqmsg->ibm_u.connparams.ibcp_max_frags > + kiblnd_rdma_frags(version, ni)) { CWARN("Can't accept conn from %s (version %x): " - "max message size %d is too large (%d wanted)\n", - libcfs_nid2str(nid), version, max_frags, + "max_frags %d too large (%d wanted)\n", + libcfs_nid2str(nid), version, + reqmsg->ibm_u.connparams.ibcp_max_frags, kiblnd_rdma_frags(version, ni)); if (version >= IBLND_MSG_VERSION) rej.ibr_why = IBLND_REJECT_RDMA_FRAGS; goto failed; - } else if ((max_frags < kiblnd_rdma_frags(version, ni)) && + } else if (reqmsg->ibm_u.connparams.ibcp_max_frags < + kiblnd_rdma_frags(version, ni) && net->ibn_fmr_ps == NULL) { CWARN("Can't accept conn from %s (version %x): " - "max message size %d incompatible without FMR pool " + "max_frags %d incompatible without FMR pool " "(%d wanted)\n", - libcfs_nid2str(nid), version, max_frags, + libcfs_nid2str(nid), version, + reqmsg->ibm_u.connparams.ibcp_max_frags, kiblnd_rdma_frags(version, ni)); if (version == IBLND_MSG_VERSION) @@ -2377,7 +2372,7 @@ kiblnd_passive_connect(struct rdma_cm_id *cmid, void *priv, int priv_nob) } /* We have validated the peer's parameters so use those */ - peer->ibp_max_frags = max_frags; + peer->ibp_max_frags = reqmsg->ibm_u.connparams.ibcp_max_frags; peer->ibp_queue_depth = reqmsg->ibm_u.connparams.ibcp_queue_depth; write_lock_irqsave(g_lock, flags); @@ -2492,7 +2487,7 @@ kiblnd_passive_connect(struct rdma_cm_id *cmid, void *priv, int priv_nob) kiblnd_init_msg(ackmsg, IBLND_MSG_CONNACK, sizeof(ackmsg->ibm_u.connparams)); ackmsg->ibm_u.connparams.ibcp_queue_depth = conn->ibc_queue_depth; - ackmsg->ibm_u.connparams.ibcp_max_frags = conn->ibc_max_frags << IBLND_FRAG_SHIFT; + ackmsg->ibm_u.connparams.ibcp_max_frags = conn->ibc_max_frags; ackmsg->ibm_u.connparams.ibcp_max_msg_size = IBLND_MSG_SIZE; kiblnd_pack_msg(ni, ackmsg, version, 0, nid, reqmsg->ibm_srcstamp); @@ -2555,7 +2550,7 @@ kiblnd_check_reconnect(kib_conn_t *conn, int version, if (cp) { msg_size = cp->ibcp_max_msg_size; - frag_num = cp->ibcp_max_frags << IBLND_FRAG_SHIFT; + frag_num = cp->ibcp_max_frags; queue_dep = cp->ibcp_queue_depth; } @@ -2816,11 +2811,11 @@ kiblnd_check_connreply (kib_conn_t *conn, void *priv, int priv_nob) goto failed; } - if ((msg->ibm_u.connparams.ibcp_max_frags >> IBLND_FRAG_SHIFT) > + if (msg->ibm_u.connparams.ibcp_max_frags > conn->ibc_max_frags) { CERROR("%s has incompatible max_frags %d (<=%d wanted)\n", libcfs_nid2str(peer->ibp_nid), - msg->ibm_u.connparams.ibcp_max_frags >> IBLND_FRAG_SHIFT, + msg->ibm_u.connparams.ibcp_max_frags, conn->ibc_max_frags); rc = -EPROTO; goto failed; @@ -2855,7 +2850,7 @@ kiblnd_check_connreply (kib_conn_t *conn, void *priv, int priv_nob) conn->ibc_credits = msg->ibm_u.connparams.ibcp_queue_depth; conn->ibc_reserved_credits = msg->ibm_u.connparams.ibcp_queue_depth; conn->ibc_queue_depth = msg->ibm_u.connparams.ibcp_queue_depth; - conn->ibc_max_frags = msg->ibm_u.connparams.ibcp_max_frags >> IBLND_FRAG_SHIFT; + conn->ibc_max_frags = msg->ibm_u.connparams.ibcp_max_frags; LASSERT(conn->ibc_credits + conn->ibc_reserved_credits + IBLND_OOB_MSGS(ver) <= IBLND_RX_MSGS(conn)); @@ -2910,7 +2905,7 @@ kiblnd_active_connect (struct rdma_cm_id *cmid) memset(msg, 0, sizeof(*msg)); kiblnd_init_msg(msg, IBLND_MSG_CONNREQ, sizeof(msg->ibm_u.connparams)); msg->ibm_u.connparams.ibcp_queue_depth = conn->ibc_queue_depth; - msg->ibm_u.connparams.ibcp_max_frags = conn->ibc_max_frags << IBLND_FRAG_SHIFT; + msg->ibm_u.connparams.ibcp_max_frags = conn->ibc_max_frags; msg->ibm_u.connparams.ibcp_max_msg_size = IBLND_MSG_SIZE; kiblnd_pack_msg(peer->ibp_ni, msg, version, -- 1.8.3.1