summary |
shortlog |
log |
commit | commitdiff |
tree
raw |
patch |
inline | side by side (from parent 1:
acadae4)
These patches are causing LU-8573
Revert "LU-7650 o2iblnd: Put back work queue check previously removed"
This reverts commit
bde1da1ec098450f40887587b0a46c9eb86a4f6c.
Revert "LU-7650 o2iblnd: handle mixed page size configurations."
This reverts commit
399a5ac1fc73343c69e0fd737032adf5329df1b2.
Change-Id: I4517fab64ac5b1023e615874f58b1bb4902e8c43
Signed-off-by: Alex Zhuravlev <alexey.zhuravlev@intel.com>
Reviewed-on: http://review.whamcloud.com/23439
Tested-by: Jenkins
Reviewed-by: Oleg Drokin <oleg.drokin@intel.com>
Tested-by: Oleg Drokin <oleg.drokin@intel.com>
kiblnd_unpack_rd(kib_msg_t *msg, int flip)
{
kib_rdma_desc_t *rd;
kiblnd_unpack_rd(kib_msg_t *msg, int flip)
{
kib_rdma_desc_t *rd;
__swab32s(&rd->rd_nfrags);
}
__swab32s(&rd->rd_nfrags);
}
+ n = rd->rd_nfrags;
+
+ if (n <= 0 || n > IBLND_MAX_RDMA_FRAGS) {
+ CERROR("Bad nfrags: %d, should be 0 < n <= %d\n",
+ n, IBLND_MAX_RDMA_FRAGS);
+ return 1;
+ }
+
nob = offsetof (kib_msg_t, ibm_u) +
kiblnd_rd_msg_size(rd, msg->ibm_type, n);
nob = offsetof (kib_msg_t, ibm_u) +
kiblnd_rd_msg_size(rd, msg->ibm_type, n);
- msg_size = kiblnd_rd_size(rd);
- if (msg_size <= 0 || msg_size > LNET_MAX_PAYLOAD) {
- CERROR("Bad msg_size: %d, should be 0 < n <= %d\n",
- msg_size, LNET_MAX_PAYLOAD);
- return 1;
- }
-
#define IBLND_OOB_CAPABLE(v) ((v) != IBLND_MSG_VERSION_1)
#define IBLND_OOB_MSGS(v) (IBLND_OOB_CAPABLE(v) ? 2 : 0)
#define IBLND_OOB_CAPABLE(v) ((v) != IBLND_MSG_VERSION_1)
#define IBLND_OOB_MSGS(v) (IBLND_OOB_CAPABLE(v) ? 2 : 0)
-#define IBLND_FRAG_SHIFT (PAGE_SHIFT - 12) /* frag size on wire is in 4K units */
-#define IBLND_MSG_SIZE (4 << 10) /* max size of queued messages (inc hdr) */
-#define IBLND_MAX_RDMA_FRAGS (LNET_MAX_PAYLOAD >> 12)/* max # of fragments supported in 4K size */
+#define IBLND_MSG_SIZE (4<<10) /* max size of queued messages (inc hdr) */
+#define IBLND_MAX_RDMA_FRAGS LNET_MAX_IOV /* max # of fragments supported */
/************************/
/* derived constants... */
/************************/
/* derived constants... */
/* WRs and CQEs (per connection) */
#define IBLND_RECV_WRS(c) IBLND_RX_MSGS(c)
#define IBLND_SEND_WRS(c) \
/* WRs and CQEs (per connection) */
#define IBLND_RECV_WRS(c) IBLND_RX_MSGS(c)
#define IBLND_SEND_WRS(c) \
- (((c->ibc_max_frags + 1) << IBLND_FRAG_SHIFT) * \
- kiblnd_concurrent_sends(c->ibc_version, c->ibc_peer->ibp_ni))
+ ((c->ibc_max_frags + 1) * kiblnd_concurrent_sends(c->ibc_version, \
+ c->ibc_peer->ibp_ni))
#define IBLND_CQ_ENTRIES(c) (IBLND_RECV_WRS(c) + IBLND_SEND_WRS(c))
struct kib_hca_dev;
#define IBLND_CQ_ENTRIES(c) (IBLND_RECV_WRS(c) + IBLND_SEND_WRS(c))
struct kib_hca_dev;
tunables = &ni->ni_lnd_tunables->lt_tun_u.lt_o2ib;
mod = tunables->lnd_map_on_demand;
tunables = &ni->ni_lnd_tunables->lt_tun_u.lt_o2ib;
mod = tunables->lnd_map_on_demand;
- return mod != 0 ? mod : IBLND_MAX_RDMA_FRAGS >> IBLND_FRAG_SHIFT;
+ return mod != 0 ? mod : IBLND_MAX_RDMA_FRAGS;
}
static inline int
kiblnd_rdma_frags(int version, struct lnet_ni *ni)
{
return version == IBLND_MSG_VERSION_1 ?
}
static inline int
kiblnd_rdma_frags(int version, struct lnet_ni *ni)
{
return version == IBLND_MSG_VERSION_1 ?
- (IBLND_MAX_RDMA_FRAGS >> IBLND_FRAG_SHIFT) :
kiblnd_cfg_rdma_frags(ni);
}
kiblnd_cfg_rdma_frags(ni);
}
LASSERT(tx->tx_queued);
/* We rely on this for QP sizing */
LASSERT(tx->tx_nwrq > 0);
LASSERT(tx->tx_queued);
/* We rely on this for QP sizing */
LASSERT(tx->tx_nwrq > 0);
+ LASSERT(tx->tx_nwrq <= 1 + conn->ibc_max_frags);
LASSERT(credit == 0 || credit == 1);
LASSERT(conn->ibc_outstanding_credits >= 0);
LASSERT(credit == 0 || credit == 1);
LASSERT(conn->ibc_outstanding_credits >= 0);
LASSERT (type == IBLND_MSG_GET_DONE ||
type == IBLND_MSG_PUT_DONE);
LASSERT (type == IBLND_MSG_GET_DONE ||
type == IBLND_MSG_PUT_DONE);
- if (kiblnd_rd_size(srcrd) > conn->ibc_max_frags << PAGE_SHIFT) {
- CERROR("RDMA is too large for peer %s (%d), src size: %d dst size: %d\n",
- libcfs_nid2str(conn->ibc_peer->ibp_nid),
- conn->ibc_max_frags << PAGE_SHIFT,
- kiblnd_rd_size(srcrd), kiblnd_rd_size(dstrd));
- GOTO(too_big, rc = -EMSGSIZE);
- }
-
srcidx = dstidx = 0;
while (resid > 0) {
srcidx = dstidx = 0;
while (resid > 0) {
- if (tx->tx_nwrq >= IBLND_MAX_RDMA_FRAGS) {
+ if (tx->tx_nwrq >= conn->ibc_max_frags) {
CERROR("RDMA has too many fragments for peer %s (%d), "
"src idx/frags: %d/%d dst idx/frags: %d/%d\n",
libcfs_nid2str(conn->ibc_peer->ibp_nid),
CERROR("RDMA has too many fragments for peer %s (%d), "
"src idx/frags: %d/%d dst idx/frags: %d/%d\n",
libcfs_nid2str(conn->ibc_peer->ibp_nid),
srcidx, srcrd->rd_nfrags,
dstidx, dstrd->rd_nfrags);
rc = -EMSGSIZE;
srcidx, srcrd->rd_nfrags,
dstidx, dstrd->rd_nfrags);
rc = -EMSGSIZE;
if (rc < 0) /* no RDMA if completing with failure */
tx->tx_nwrq = 0;
if (rc < 0) /* no RDMA if completing with failure */
tx->tx_nwrq = 0;
kib_rej_t rej;
int version = IBLND_MSG_VERSION;
unsigned long flags;
kib_rej_t rej;
int version = IBLND_MSG_VERSION;
unsigned long flags;
int rc;
struct sockaddr_in *peer_addr;
LASSERT (!in_interrupt());
int rc;
struct sockaddr_in *peer_addr;
LASSERT (!in_interrupt());
- max_frags = reqmsg->ibm_u.connparams.ibcp_max_frags >> IBLND_FRAG_SHIFT;
- if (max_frags > kiblnd_rdma_frags(version, ni)) {
+ if (reqmsg->ibm_u.connparams.ibcp_max_frags >
+ kiblnd_rdma_frags(version, ni)) {
CWARN("Can't accept conn from %s (version %x): "
CWARN("Can't accept conn from %s (version %x): "
- "max message size %d is too large (%d wanted)\n",
- libcfs_nid2str(nid), version, max_frags,
+ "max_frags %d too large (%d wanted)\n",
+ libcfs_nid2str(nid), version,
+ reqmsg->ibm_u.connparams.ibcp_max_frags,
kiblnd_rdma_frags(version, ni));
if (version >= IBLND_MSG_VERSION)
rej.ibr_why = IBLND_REJECT_RDMA_FRAGS;
goto failed;
kiblnd_rdma_frags(version, ni));
if (version >= IBLND_MSG_VERSION)
rej.ibr_why = IBLND_REJECT_RDMA_FRAGS;
goto failed;
- } else if ((max_frags < kiblnd_rdma_frags(version, ni)) &&
+ } else if (reqmsg->ibm_u.connparams.ibcp_max_frags <
+ kiblnd_rdma_frags(version, ni) &&
net->ibn_fmr_ps == NULL) {
CWARN("Can't accept conn from %s (version %x): "
net->ibn_fmr_ps == NULL) {
CWARN("Can't accept conn from %s (version %x): "
- "max message size %d incompatible without FMR pool "
+ "max_frags %d incompatible without FMR pool "
- libcfs_nid2str(nid), version, max_frags,
+ libcfs_nid2str(nid), version,
+ reqmsg->ibm_u.connparams.ibcp_max_frags,
kiblnd_rdma_frags(version, ni));
if (version == IBLND_MSG_VERSION)
kiblnd_rdma_frags(version, ni));
if (version == IBLND_MSG_VERSION)
}
/* We have validated the peer's parameters so use those */
}
/* We have validated the peer's parameters so use those */
- peer->ibp_max_frags = max_frags;
+ peer->ibp_max_frags = reqmsg->ibm_u.connparams.ibcp_max_frags;
peer->ibp_queue_depth = reqmsg->ibm_u.connparams.ibcp_queue_depth;
write_lock_irqsave(g_lock, flags);
peer->ibp_queue_depth = reqmsg->ibm_u.connparams.ibcp_queue_depth;
write_lock_irqsave(g_lock, flags);
kiblnd_init_msg(ackmsg, IBLND_MSG_CONNACK,
sizeof(ackmsg->ibm_u.connparams));
ackmsg->ibm_u.connparams.ibcp_queue_depth = conn->ibc_queue_depth;
kiblnd_init_msg(ackmsg, IBLND_MSG_CONNACK,
sizeof(ackmsg->ibm_u.connparams));
ackmsg->ibm_u.connparams.ibcp_queue_depth = conn->ibc_queue_depth;
- ackmsg->ibm_u.connparams.ibcp_max_frags = conn->ibc_max_frags << IBLND_FRAG_SHIFT;
+ ackmsg->ibm_u.connparams.ibcp_max_frags = conn->ibc_max_frags;
ackmsg->ibm_u.connparams.ibcp_max_msg_size = IBLND_MSG_SIZE;
kiblnd_pack_msg(ni, ackmsg, version, 0, nid, reqmsg->ibm_srcstamp);
ackmsg->ibm_u.connparams.ibcp_max_msg_size = IBLND_MSG_SIZE;
kiblnd_pack_msg(ni, ackmsg, version, 0, nid, reqmsg->ibm_srcstamp);
if (cp) {
msg_size = cp->ibcp_max_msg_size;
if (cp) {
msg_size = cp->ibcp_max_msg_size;
- frag_num = cp->ibcp_max_frags << IBLND_FRAG_SHIFT;
+ frag_num = cp->ibcp_max_frags;
queue_dep = cp->ibcp_queue_depth;
}
queue_dep = cp->ibcp_queue_depth;
}
- if ((msg->ibm_u.connparams.ibcp_max_frags >> IBLND_FRAG_SHIFT) >
+ if (msg->ibm_u.connparams.ibcp_max_frags >
conn->ibc_max_frags) {
CERROR("%s has incompatible max_frags %d (<=%d wanted)\n",
libcfs_nid2str(peer->ibp_nid),
conn->ibc_max_frags) {
CERROR("%s has incompatible max_frags %d (<=%d wanted)\n",
libcfs_nid2str(peer->ibp_nid),
- msg->ibm_u.connparams.ibcp_max_frags >> IBLND_FRAG_SHIFT,
+ msg->ibm_u.connparams.ibcp_max_frags,
conn->ibc_max_frags);
rc = -EPROTO;
goto failed;
conn->ibc_max_frags);
rc = -EPROTO;
goto failed;
conn->ibc_credits = msg->ibm_u.connparams.ibcp_queue_depth;
conn->ibc_reserved_credits = msg->ibm_u.connparams.ibcp_queue_depth;
conn->ibc_queue_depth = msg->ibm_u.connparams.ibcp_queue_depth;
conn->ibc_credits = msg->ibm_u.connparams.ibcp_queue_depth;
conn->ibc_reserved_credits = msg->ibm_u.connparams.ibcp_queue_depth;
conn->ibc_queue_depth = msg->ibm_u.connparams.ibcp_queue_depth;
- conn->ibc_max_frags = msg->ibm_u.connparams.ibcp_max_frags >> IBLND_FRAG_SHIFT;
+ conn->ibc_max_frags = msg->ibm_u.connparams.ibcp_max_frags;
LASSERT(conn->ibc_credits + conn->ibc_reserved_credits +
IBLND_OOB_MSGS(ver) <= IBLND_RX_MSGS(conn));
LASSERT(conn->ibc_credits + conn->ibc_reserved_credits +
IBLND_OOB_MSGS(ver) <= IBLND_RX_MSGS(conn));
memset(msg, 0, sizeof(*msg));
kiblnd_init_msg(msg, IBLND_MSG_CONNREQ, sizeof(msg->ibm_u.connparams));
msg->ibm_u.connparams.ibcp_queue_depth = conn->ibc_queue_depth;
memset(msg, 0, sizeof(*msg));
kiblnd_init_msg(msg, IBLND_MSG_CONNREQ, sizeof(msg->ibm_u.connparams));
msg->ibm_u.connparams.ibcp_queue_depth = conn->ibc_queue_depth;
- msg->ibm_u.connparams.ibcp_max_frags = conn->ibc_max_frags << IBLND_FRAG_SHIFT;
+ msg->ibm_u.connparams.ibcp_max_frags = conn->ibc_max_frags;
msg->ibm_u.connparams.ibcp_max_msg_size = IBLND_MSG_SIZE;
kiblnd_pack_msg(peer->ibp_ni, msg, version,
msg->ibm_u.connparams.ibcp_max_msg_size = IBLND_MSG_SIZE;
kiblnd_pack_msg(peer->ibp_ni, msg, version,