mxlnd - MX 1.2.1 or later,
ptllnd - Portals 3.3 / UNICOS/lc 1.5.x, 2.0.x
-Severity :
-Bugzilla :
-Description:
-Details :
+Severity :
+Bugzilla :
+Description:
+Details :
+
+Severity : major
+Bugzilla : 13621, 15983
+Description: Protocol V2 of o2iblnd
+Details : o2iblnd V2 has several new features:
+ . map-on-demand: map-on-demand is disabled by default, it can
+ be enabled by using modparam "map_on_demand=@value@", @value@
+ should >= 0 and < 256, 0 will disable map-on-demand, any other
+ valid value will enable map-on-demand.
+ Oi2blnd will create FMR or physical MR for RDMA if fragments of
+ RD > @value@.
+ Enable map-on-demand will take less memory for new connection,
+ but a little more CPU for RDMA.
+ . iWARP : to support iWARP, please enable map-on-demand, 32 and 64
+ are recommanded value. iWARP will probably fail for value >=128.
+ . OOB NOOP message: to resolve deadlock on router.
+ . tunable peer_credits_hiw: (high water to return credits),
+ default value of peer_credits_hiw equals to (peer_credits -1),
+ user can change it between peer_credits/2 and (peer_credits - 1).
+ Lower value is recommended for high latency network.
+ . tunable message queue size: it always equals to peer_credits,
+ higher value is recommended for high latency network.
+ . It's compatible with earlier version of o2iblnd
Severity : normal
Bugzilla : 16034
#include "o2iblnd.h"
-lnd_t the_kiblnd = {
+lnd_t the_o2iblnd = {
.lnd_type = O2IBLND,
.lnd_startup = kiblnd_startup,
.lnd_shutdown = kiblnd_shutdown,
return (sum == 0) ? 1 : sum;
}
-void
-kiblnd_init_msg (kib_msg_t *msg, int type, int body_nob)
+static char *
+kiblnd_msgtype2str(int type)
+{
+ switch (type) {
+ case IBLND_MSG_CONNREQ:
+ return "CONNREQ";
+
+ case IBLND_MSG_CONNACK:
+ return "CONNACK";
+
+ case IBLND_MSG_NOOP:
+ return "NOOP";
+
+ case IBLND_MSG_IMMEDIATE:
+ return "IMMEDIATE";
+
+ case IBLND_MSG_PUT_REQ:
+ return "PUT_REQ";
+
+ case IBLND_MSG_PUT_NAK:
+ return "PUT_NAK";
+
+ case IBLND_MSG_PUT_ACK:
+ return "PUT_ACK";
+
+ case IBLND_MSG_PUT_DONE:
+ return "PUT_DONE";
+
+ case IBLND_MSG_GET_REQ:
+ return "GET_REQ";
+
+ case IBLND_MSG_GET_DONE:
+ return "GET_DONE";
+
+ default:
+ return "???";
+ }
+}
+
+static int
+kiblnd_msgtype2size(int type)
+{
+ const int hdr_size = offsetof(kib_msg_t, ibm_u);
+
+ switch (type) {
+ case IBLND_MSG_CONNREQ:
+ case IBLND_MSG_CONNACK:
+ return hdr_size + sizeof(kib_connparams_t);
+
+ case IBLND_MSG_NOOP:
+ return hdr_size;
+
+ case IBLND_MSG_IMMEDIATE:
+ return offsetof(kib_msg_t, ibm_u.immediate.ibim_payload[0]);
+
+ case IBLND_MSG_PUT_REQ:
+ return hdr_size + sizeof(kib_putreq_msg_t);
+
+ case IBLND_MSG_PUT_ACK:
+ return hdr_size + sizeof(kib_putack_msg_t);
+
+ case IBLND_MSG_GET_REQ:
+ return hdr_size + sizeof(kib_get_msg_t);
+
+ case IBLND_MSG_PUT_NAK:
+ case IBLND_MSG_PUT_DONE:
+ case IBLND_MSG_GET_DONE:
+ return hdr_size + sizeof(kib_completion_msg_t);
+ default:
+ return -1;
+ }
+}
+
+static int
+kiblnd_unpack_rd(kib_msg_t *msg, int flip)
{
- msg->ibm_type = type;
- msg->ibm_nob = offsetof(kib_msg_t, ibm_u) + body_nob;
+ kib_rdma_desc_t *rd;
+ int nob;
+ int n;
+ int i;
+
+ LASSERT (msg->ibm_type == IBLND_MSG_GET_REQ ||
+ msg->ibm_type == IBLND_MSG_PUT_ACK);
+
+ rd = msg->ibm_type == IBLND_MSG_GET_REQ ?
+ &msg->ibm_u.get.ibgm_rd :
+ &msg->ibm_u.putack.ibpam_rd;
+
+ if (flip) {
+ __swab32s(&rd->rd_key);
+ __swab32s(&rd->rd_nfrags);
+ }
+
+ n = rd->rd_nfrags;
+
+ if (n <= 0 || n > IBLND_MAX_RDMA_FRAGS) {
+ CERROR("Bad nfrags: %d, should be 0 < n <= %d\n",
+ n, IBLND_MAX_RDMA_FRAGS);
+ return 1;
+ }
+
+ nob = offsetof (kib_msg_t, ibm_u) +
+ kiblnd_rd_msg_size(rd, msg->ibm_type, n);
+
+ if (msg->ibm_nob < nob) {
+ CERROR("Short %s: %d(%d)\n",
+ kiblnd_msgtype2str(msg->ibm_type), msg->ibm_nob, nob);
+ return 1;
+ }
+
+ if (!flip)
+ return 0;
+
+ for (i = 0; i < n; i++) {
+ __swab32s(&rd->rd_frags[i].rf_nob);
+ __swab64s(&rd->rd_frags[i].rf_addr);
+ }
+
+ return 0;
}
void
-kiblnd_pack_msg (lnet_ni_t *ni, kib_msg_t *msg,
+kiblnd_pack_msg (lnet_ni_t *ni, kib_msg_t *msg, int version,
int credits, lnet_nid_t dstnid, __u64 dststamp)
{
kib_net_t *net = ni->ni_data;
/* CAVEAT EMPTOR! all message fields not set here should have been
* initialised previously. */
msg->ibm_magic = IBLND_MSG_MAGIC;
- msg->ibm_version = IBLND_MSG_VERSION;
+ msg->ibm_version = version;
/* ibm_type */
msg->ibm_credits = credits;
/* ibm_nob */
{
const int hdr_size = offsetof(kib_msg_t, ibm_u);
__u32 msg_cksum;
- int flip;
+ __u16 version;
int msg_nob;
-#if !IBLND_MAP_ON_DEMAND
- int i;
- int n;
-#endif
+ int flip;
+
/* 6 bytes are enough to have received magic + version */
if (nob < 6) {
CERROR("Short message: %d\n", nob);
return -EPROTO;
}
- if (msg->ibm_version !=
- (flip ? __swab16(IBLND_MSG_VERSION) : IBLND_MSG_VERSION)) {
- CERROR("Bad version: %d\n", msg->ibm_version);
+ version = flip ? __swab16(msg->ibm_version) : msg->ibm_version;
+ if (version != IBLND_MSG_VERSION &&
+ version != IBLND_MSG_VERSION_1) {
+ CERROR("Bad version: %x\n", version);
return -EPROTO;
}
CERROR("Bad checksum\n");
return -EPROTO;
}
+
msg->ibm_cksum = msg_cksum;
if (flip) {
/* leave magic unflipped as a clue to peer endianness */
- __swab16s(&msg->ibm_version);
+ msg->ibm_version = version;
CLASSERT (sizeof(msg->ibm_type) == 1);
CLASSERT (sizeof(msg->ibm_credits) == 1);
- msg->ibm_nob = msg_nob;
+ msg->ibm_nob = msg_nob;
__swab64s(&msg->ibm_srcnid);
__swab64s(&msg->ibm_srcstamp);
__swab64s(&msg->ibm_dstnid);
return -EPROTO;
}
+ if (msg_nob < kiblnd_msgtype2size(msg->ibm_type)) {
+ CERROR("Short %s: %d(%d)\n", kiblnd_msgtype2str(msg->ibm_type),
+ msg_nob, kiblnd_msgtype2size(msg->ibm_type));
+ return -EPROTO;
+ }
+
switch (msg->ibm_type) {
default:
CERROR("Unknown message type %x\n", msg->ibm_type);
return -EPROTO;
case IBLND_MSG_NOOP:
- break;
-
case IBLND_MSG_IMMEDIATE:
- if (msg_nob < offsetof(kib_msg_t, ibm_u.immediate.ibim_payload[0])) {
- CERROR("Short IMMEDIATE: %d(%d)\n", msg_nob,
- (int)offsetof(kib_msg_t, ibm_u.immediate.ibim_payload[0]));
- return -EPROTO;
- }
- break;
-
case IBLND_MSG_PUT_REQ:
- if (msg_nob < hdr_size + sizeof(msg->ibm_u.putreq)) {
- CERROR("Short PUT_REQ: %d(%d)\n", msg_nob,
- (int)(hdr_size + sizeof(msg->ibm_u.putreq)));
- return -EPROTO;
- }
break;
case IBLND_MSG_PUT_ACK:
- if (msg_nob < hdr_size + sizeof(msg->ibm_u.putack)) {
- CERROR("Short PUT_ACK: %d(%d)\n", msg_nob,
- (int)(hdr_size + sizeof(msg->ibm_u.putack)));
- return -EPROTO;
- }
-#if IBLND_MAP_ON_DEMAND
- if (flip) {
- __swab64s(&msg->ibm_u.putack.ibpam_rd.rd_addr);
- __swab32s(&msg->ibm_u.putack.ibpam_rd.rd_nob);
- __swab32s(&msg->ibm_u.putack.ibpam_rd.rd_key);
- }
-#else
- if (flip) {
- __swab32s(&msg->ibm_u.putack.ibpam_rd.rd_key);
- __swab32s(&msg->ibm_u.putack.ibpam_rd.rd_nfrags);
- }
-
- n = msg->ibm_u.putack.ibpam_rd.rd_nfrags;
- if (n <= 0 || n > IBLND_MAX_RDMA_FRAGS) {
- CERROR("Bad PUT_ACK nfrags: %d, should be 0 < n <= %d\n",
- n, IBLND_MAX_RDMA_FRAGS);
- return -EPROTO;
- }
-
- if (msg_nob < offsetof(kib_msg_t, ibm_u.putack.ibpam_rd.rd_frags[n])) {
- CERROR("Short PUT_ACK: %d(%d)\n", msg_nob,
- (int)offsetof(kib_msg_t, ibm_u.putack.ibpam_rd.rd_frags[n]));
- return -EPROTO;
- }
-
- if (flip) {
- for (i = 0; i < n; i++) {
- __swab32s(&msg->ibm_u.putack.ibpam_rd.rd_frags[i].rf_nob);
- __swab64s(&msg->ibm_u.putack.ibpam_rd.rd_frags[i].rf_addr);
- }
- }
-#endif
- break;
-
case IBLND_MSG_GET_REQ:
- if (msg_nob < hdr_size + sizeof(msg->ibm_u.get)) {
- CERROR("Short GET_REQ: %d(%d)\n", msg_nob,
- (int)(hdr_size + sizeof(msg->ibm_u.get)));
- return -EPROTO;
- }
-#if IBLND_MAP_ON_DEMAND
- if (flip) {
- __swab64s(&msg->ibm_u.get.ibgm_rd.rd_addr);
- __swab32s(&msg->ibm_u.get.ibgm_rd.rd_nob);
- __swab32s(&msg->ibm_u.get.ibgm_rd.rd_key);
- }
-#else
- if (flip) {
- __swab32s(&msg->ibm_u.get.ibgm_rd.rd_key);
- __swab32s(&msg->ibm_u.get.ibgm_rd.rd_nfrags);
- }
-
- n = msg->ibm_u.get.ibgm_rd.rd_nfrags;
- if (n <= 0 || n > IBLND_MAX_RDMA_FRAGS) {
- CERROR("Bad GET_REQ nfrags: %d, should be 0 < n <= %d\n",
- n, IBLND_MAX_RDMA_FRAGS);
+ if (kiblnd_unpack_rd(msg, flip))
return -EPROTO;
- }
-
- if (msg_nob < offsetof(kib_msg_t, ibm_u.get.ibgm_rd.rd_frags[n])) {
- CERROR("Short GET_REQ: %d(%d)\n", msg_nob,
- (int)offsetof(kib_msg_t, ibm_u.get.ibgm_rd.rd_frags[n]));
- return -EPROTO;
- }
-
- if (flip)
- for (i = 0; i < msg->ibm_u.get.ibgm_rd.rd_nfrags; i++) {
- __swab32s(&msg->ibm_u.get.ibgm_rd.rd_frags[i].rf_nob);
- __swab64s(&msg->ibm_u.get.ibgm_rd.rd_frags[i].rf_addr);
- }
-#endif
break;
case IBLND_MSG_PUT_NAK:
case IBLND_MSG_PUT_DONE:
case IBLND_MSG_GET_DONE:
- if (msg_nob < hdr_size + sizeof(msg->ibm_u.completion)) {
- CERROR("Short RDMA completion: %d(%d)\n", msg_nob,
- (int)(hdr_size + sizeof(msg->ibm_u.completion)));
- return -EPROTO;
- }
if (flip)
__swab32s(&msg->ibm_u.completion.ibcm_status);
break;
case IBLND_MSG_CONNREQ:
case IBLND_MSG_CONNACK:
- if (msg_nob < hdr_size + sizeof(msg->ibm_u.connparams)) {
- CERROR("Short connreq/ack: %d(%d)\n", msg_nob,
- (int)(hdr_size + sizeof(msg->ibm_u.connparams)));
- return -EPROTO;
- }
if (flip) {
__swab16s(&msg->ibm_u.connparams.ibcp_queue_depth);
__swab16s(&msg->ibm_u.connparams.ibcp_max_frags);
atomic_dec(&net->ibn_npeers);
}
-void
-kiblnd_destroy_dev (kib_dev_t *dev)
-{
- LASSERT (dev->ibd_nnets == 0);
-
- if (!list_empty(&dev->ibd_list)) /* on kib_devs? */
- list_del_init(&dev->ibd_list);
-
- if (dev->ibd_mr != NULL)
- ib_dereg_mr(dev->ibd_mr);
-
- if (dev->ibd_pd != NULL)
- ib_dealloc_pd(dev->ibd_pd);
-
- if (dev->ibd_cmid != NULL)
- rdma_destroy_id(dev->ibd_cmid);
-
- LIBCFS_FREE(dev, sizeof(*dev));
-}
-
kib_peer_t *
kiblnd_find_peer_locked (lnet_nid_t nid)
{
if (peer->ibp_nid != nid)
continue;
- CDEBUG(D_NET, "got peer [%p] -> %s (%d)\n",
+ CDEBUG(D_NET, "got peer [%p] -> %s (%d) version: %x\n",
peer, libcfs_nid2str(nid),
- atomic_read(&peer->ibp_refcount));
+ atomic_read(&peer->ibp_refcount),
+ peer->ibp_version);
return peer;
}
return NULL;
spin_lock(&conn->ibc_lock);
- CDEBUG(D_CONSOLE, "conn[%d] %p -> %s: \n",
+ CDEBUG(D_CONSOLE, "conn[%d] %p [version %x] -> %s: \n",
atomic_read(&conn->ibc_refcount), conn,
- libcfs_nid2str(conn->ibc_peer->ibp_nid));
- CDEBUG(D_CONSOLE, " state %d nposted %d cred %d o_cred %d r_cred %d\n",
- conn->ibc_state, conn->ibc_nsends_posted, conn->ibc_credits,
+ conn->ibc_version, libcfs_nid2str(conn->ibc_peer->ibp_nid));
+ CDEBUG(D_CONSOLE, " state %d nposted %d/%d cred %d o_cred %d r_cred %d\n",
+ conn->ibc_state, conn->ibc_noops_posted,
+ conn->ibc_nsends_posted, conn->ibc_credits,
conn->ibc_outstanding_credits, conn->ibc_reserved_credits);
CDEBUG(D_CONSOLE, " comms_err %d\n", conn->ibc_comms_error);
list_for_each(tmp, &conn->ibc_early_rxs)
kiblnd_debug_rx(list_entry(tmp, kib_rx_t, rx_list));
- CDEBUG(D_CONSOLE, " tx_noops:\n");
- list_for_each(tmp, &conn->ibc_tx_noops)
- kiblnd_debug_tx(list_entry(tmp, kib_tx_t, tx_list));
-
CDEBUG(D_CONSOLE, " tx_queue_nocred:\n");
list_for_each(tmp, &conn->ibc_tx_queue_nocred)
kiblnd_debug_tx(list_entry(tmp, kib_tx_t, tx_list));
kiblnd_debug_tx(list_entry(tmp, kib_tx_t, tx_list));
CDEBUG(D_CONSOLE, " rxs:\n");
- for (i = 0; i < IBLND_RX_MSGS; i++)
+ for (i = 0; i < IBLND_RX_MSGS(conn->ibc_version); i++)
kiblnd_debug_rx(&conn->ibc_rxs[i]);
spin_unlock(&conn->ibc_lock);
}
+int
+kiblnd_translate_mtu(int value)
+{
+ switch (value) {
+ default:
+ return -1;
+ case 0:
+ return 0;
+ case 256:
+ return IB_MTU_256;
+ case 512:
+ return IB_MTU_512;
+ case 1024:
+ return IB_MTU_1024;
+ case 2048:
+ return IB_MTU_2048;
+ case 4096:
+ return IB_MTU_4096;
+ }
+}
+
+static void
+kiblnd_setup_mtu(struct rdma_cm_id *cmid)
+{
+ unsigned long flags;
+ int mtu;
+
+ /* XXX There is no path record for iWARP, set by netdev->change_mtu? */
+ if (cmid->route.path_rec == NULL)
+ return;
+
+ write_lock_irqsave(&kiblnd_data.kib_global_lock, flags);
+
+ mtu = kiblnd_translate_mtu(*kiblnd_tunables.kib_ib_mtu);
+ LASSERT (mtu >= 0);
+ if (mtu != 0)
+ cmid->route.path_rec->mtu = mtu;
+
+ write_unlock_irqrestore(&kiblnd_data.kib_global_lock, flags);
+}
+
kib_conn_t *
-kiblnd_create_conn (kib_peer_t *peer, struct rdma_cm_id *cmid, int state)
+kiblnd_create_conn(kib_peer_t *peer, struct rdma_cm_id *cmid,
+ int state, int version)
{
/* CAVEAT EMPTOR:
* If the new conn is created successfully it takes over the caller's
* she must dispose of 'cmid'. (Actually I'd block forever if I tried
* to destroy 'cmid' here since I'm called from the CM which still has
* its ref on 'cmid'). */
- kib_conn_t *conn;
kib_net_t *net = peer->ibp_ni->ni_data;
- int i;
- int page_offset;
- int ipage;
- int rc;
- struct ib_cq *cq;
struct ib_qp_init_attr *init_qp_attr;
+ kib_conn_t *conn;
+ struct ib_cq *cq;
unsigned long flags;
+ int rc;
+ int i;
LASSERT (net != NULL);
LASSERT (!in_interrupt());
memset(conn, 0, sizeof(*conn)); /* zero flags, NULL pointers etc... */
conn->ibc_state = IBLND_CONN_INIT;
+ conn->ibc_version = version;
conn->ibc_peer = peer; /* I take the caller's ref */
cmid->context = conn; /* for future CM callbacks */
conn->ibc_cmid = cmid;
INIT_LIST_HEAD(&conn->ibc_early_rxs);
- INIT_LIST_HEAD(&conn->ibc_tx_noops);
INIT_LIST_HEAD(&conn->ibc_tx_queue);
INIT_LIST_HEAD(&conn->ibc_tx_queue_rsrvd);
INIT_LIST_HEAD(&conn->ibc_tx_queue_nocred);
}
memset(conn->ibc_connvars, 0, sizeof(*conn->ibc_connvars));
- LIBCFS_ALLOC(conn->ibc_rxs, IBLND_RX_MSGS * sizeof(kib_rx_t));
+ LIBCFS_ALLOC(conn->ibc_rxs, IBLND_RX_MSGS(version) * sizeof(kib_rx_t));
if (conn->ibc_rxs == NULL) {
CERROR("Cannot allocate RX buffers\n");
goto failed_2;
}
- memset(conn->ibc_rxs, 0, IBLND_RX_MSGS * sizeof(kib_rx_t));
+ memset(conn->ibc_rxs, 0, IBLND_RX_MSGS(version) * sizeof(kib_rx_t));
- rc = kiblnd_alloc_pages(&conn->ibc_rx_pages, IBLND_RX_MSG_PAGES);
+ rc = kiblnd_alloc_pages(&conn->ibc_rx_pages,
+ IBLND_RX_MSG_PAGES(version));
if (rc != 0)
goto failed_2;
- for (i = ipage = page_offset = 0; i < IBLND_RX_MSGS; i++) {
- struct page *page = conn->ibc_rx_pages->ibp_pages[ipage];
- kib_rx_t *rx = &conn->ibc_rxs[i];
-
- rx->rx_conn = conn;
- rx->rx_msg = (kib_msg_t *)(((char *)page_address(page)) +
- page_offset);
- rx->rx_msgaddr = kiblnd_dma_map_single(cmid->device,
- rx->rx_msg, IBLND_MSG_SIZE,
- DMA_FROM_DEVICE);
- KIBLND_UNMAP_ADDR_SET(rx, rx_msgunmap, rx->rx_msgaddr);
-
- CDEBUG(D_NET,"rx %d: %p "LPX64"("LPX64")\n",
- i, rx->rx_msg, rx->rx_msgaddr,
- lnet_page2phys(page) + page_offset);
-
- page_offset += IBLND_MSG_SIZE;
- LASSERT (page_offset <= PAGE_SIZE);
-
- if (page_offset == PAGE_SIZE) {
- page_offset = 0;
- ipage++;
- LASSERT (ipage <= IBLND_RX_MSG_PAGES);
- }
- }
+ kiblnd_map_rx_descs(conn);
#ifdef HAVE_OFED_IB_COMP_VECTOR
cq = ib_create_cq(cmid->device,
kiblnd_cq_completion, kiblnd_cq_event, conn,
- IBLND_CQ_ENTRIES(), 0);
+ IBLND_CQ_ENTRIES(version), 0);
#else
cq = ib_create_cq(cmid->device,
kiblnd_cq_completion, kiblnd_cq_event, conn,
- IBLND_CQ_ENTRIES());
+ IBLND_CQ_ENTRIES(version));
#endif
- if (!IS_ERR(cq)) {
- conn->ibc_cq = cq;
- } else {
- CERROR("Can't create CQ: %ld\n", PTR_ERR(cq));
+ if (IS_ERR(cq)) {
+ CERROR("Can't create CQ: %ld, cqe: %d\n",
+ PTR_ERR(cq), IBLND_CQ_ENTRIES(version));
goto failed_2;
}
+ conn->ibc_cq = cq;
+
rc = ib_req_notify_cq(cq, IB_CQ_NEXT_COMP);
if (rc != 0) {
CERROR("Can't request completion notificiation: %d\n", rc);
goto failed_2;
}
+ kiblnd_setup_mtu(cmid);
+
memset(init_qp_attr, 0, sizeof(*init_qp_attr));
init_qp_attr->event_handler = kiblnd_qp_event;
init_qp_attr->qp_context = conn;
- init_qp_attr->cap.max_send_wr = IBLND_SEND_WRS;
- init_qp_attr->cap.max_recv_wr = IBLND_RECV_WRS;
+ init_qp_attr->cap.max_send_wr = IBLND_SEND_WRS(version);
+ init_qp_attr->cap.max_recv_wr = IBLND_RECV_WRS(version);
init_qp_attr->cap.max_send_sge = 1;
init_qp_attr->cap.max_recv_sge = 1;
init_qp_attr->sq_sig_type = IB_SIGNAL_REQ_WR;
init_qp_attr->send_cq = cq;
init_qp_attr->recv_cq = cq;
- write_lock_irqsave(&kiblnd_data.kib_global_lock, flags);
- switch (*kiblnd_tunables.kib_ib_mtu) {
- case 0: /* don't force path MTU */
- break;
- case 256:
- cmid->route.path_rec->mtu = IB_MTU_256;
- break;
- case 512:
- cmid->route.path_rec->mtu = IB_MTU_512;
- break;
- case 1024:
- cmid->route.path_rec->mtu = IB_MTU_1024;
- break;
- case 2048:
- cmid->route.path_rec->mtu = IB_MTU_2048;
- break;
- case 4096:
- cmid->route.path_rec->mtu = IB_MTU_4096;
- break;
- default:
- LBUG();
- break;
- }
- write_unlock_irqrestore(&kiblnd_data.kib_global_lock, flags);
-
rc = rdma_create_qp(cmid, net->ibn_dev->ibd_pd, init_qp_attr);
if (rc != 0) {
- CERROR("Can't create QP: %d\n", rc);
+ CERROR("Can't create QP: %d, send_wr: %d, recv_wr: %d\n",
+ rc, init_qp_attr->cap.max_send_wr,
+ init_qp_attr->cap.max_recv_wr);
goto failed_2;
}
LIBCFS_FREE(init_qp_attr, sizeof(*init_qp_attr));
/* 1 ref for caller and each rxmsg */
- atomic_set(&conn->ibc_refcount, 1 + IBLND_RX_MSGS);
- conn->ibc_nrx = IBLND_RX_MSGS;
+ atomic_set(&conn->ibc_refcount, 1 + IBLND_RX_MSGS(version));
+ conn->ibc_nrx = IBLND_RX_MSGS(version);
/* post receives */
- for (i = 0; i < IBLND_RX_MSGS; i++) {
+ for (i = 0; i < IBLND_RX_MSGS(version); i++) {
rc = kiblnd_post_rx(&conn->ibc_rxs[i],
IBLND_POSTRX_NO_CREDIT);
if (rc != 0) {
/* correct # of posted buffers
* NB locking needed now I'm racing with completion */
spin_lock_irqsave(&kiblnd_data.kib_sched_lock, flags);
- conn->ibc_nrx -= IBLND_RX_MSGS - i;
+ conn->ibc_nrx -= IBLND_RX_MSGS(version) - i;
spin_unlock_irqrestore(&kiblnd_data.kib_sched_lock,
flags);
+ /* cmid will be destroyed by CM(ofed) after cm_callback
+ * returned, so we can't refer it anymore
+ * (by kiblnd_connd()->kiblnd_destroy_conn) */
+ rdma_destroy_qp(conn->ibc_cmid);
+ conn->ibc_cmid = NULL;
+
/* Drop my own and unused rxbuffer refcounts */
- while (i++ <= IBLND_RX_MSGS)
+ while (i++ <= IBLND_RX_MSGS(version))
kiblnd_conn_decref(conn);
return NULL;
}
}
-
+
/* Init successful! */
LASSERT (state == IBLND_CONN_ACTIVE_CONNECT ||
state == IBLND_CONN_PASSIVE_WAIT);
struct rdma_cm_id *cmid = conn->ibc_cmid;
kib_peer_t *peer = conn->ibc_peer;
int rc;
- int i;
LASSERT (!in_interrupt());
LASSERT (atomic_read(&conn->ibc_refcount) == 0);
LASSERT (list_empty(&conn->ibc_early_rxs));
- LASSERT (list_empty(&conn->ibc_tx_noops));
LASSERT (list_empty(&conn->ibc_tx_queue));
LASSERT (list_empty(&conn->ibc_tx_queue_rsrvd));
LASSERT (list_empty(&conn->ibc_tx_queue_nocred));
LASSERT (list_empty(&conn->ibc_active_txs));
+ LASSERT (conn->ibc_noops_posted == 0);
LASSERT (conn->ibc_nsends_posted == 0);
switch (conn->ibc_state) {
break;
}
- if (cmid->qp != NULL)
+ /* conn->ibc_cmid might be destroyed by CM already */
+ if (cmid != NULL && cmid->qp != NULL)
rdma_destroy_qp(cmid);
if (conn->ibc_cq != NULL) {
CWARN("Error destroying CQ: %d\n", rc);
}
- if (conn->ibc_rx_pages != NULL) {
- LASSERT (conn->ibc_rxs != NULL);
-
- for (i = 0; i < IBLND_RX_MSGS; i++) {
- kib_rx_t *rx = &conn->ibc_rxs[i];
-
- LASSERT (rx->rx_nob >= 0); /* not posted */
-
- kiblnd_dma_unmap_single(cmid->device,
- KIBLND_UNMAP_ADDR(rx, rx_msgunmap,
- rx->rx_msgaddr),
- IBLND_MSG_SIZE, DMA_FROM_DEVICE);
- }
-
- kiblnd_free_pages(conn->ibc_rx_pages);
- }
+ if (conn->ibc_rx_pages != NULL)
+ kiblnd_unmap_rx_descs(conn);
if (conn->ibc_rxs != NULL) {
LIBCFS_FREE(conn->ibc_rxs,
- IBLND_RX_MSGS * sizeof(kib_rx_t));
+ IBLND_RX_MSGS(conn->ibc_version) * sizeof(kib_rx_t));
}
if (conn->ibc_connvars != NULL)
list_for_each_safe (ctmp, cnxt, &peer->ibp_conns) {
conn = list_entry(ctmp, kib_conn_t, ibc_list);
- count++;
+ CDEBUG(D_NET, "Closing conn -> %s, "
+ "version: %x, reason: %d\n",
+ libcfs_nid2str(peer->ibp_nid),
+ conn->ibc_version, why);
+
kiblnd_close_conn_locked(conn, why);
+ count++;
}
return count;
}
int
-kiblnd_close_stale_conns_locked (kib_peer_t *peer, __u64 incarnation)
+kiblnd_close_stale_conns_locked (kib_peer_t *peer,
+ int version, __u64 incarnation)
{
kib_conn_t *conn;
struct list_head *ctmp;
list_for_each_safe (ctmp, cnxt, &peer->ibp_conns) {
conn = list_entry(ctmp, kib_conn_t, ibc_list);
- if (conn->ibc_incarnation == incarnation)
+ if (conn->ibc_version == version &&
+ conn->ibc_incarnation == incarnation)
continue;
- CDEBUG(D_NET, "Closing stale conn -> %s incarnation:"LPX64"("LPX64")\n",
+ CDEBUG(D_NET, "Closing stale conn -> %s version: %x, "
+ "incarnation:"LPX64"(%x, "LPX64")\n",
libcfs_nid2str(peer->ibp_nid),
- conn->ibc_incarnation, incarnation);
+ conn->ibc_version, conn->ibc_incarnation,
+ version, incarnation);
- count++;
kiblnd_close_conn_locked(conn, -ESTALE);
+ count++;
}
return count;
int npages = p->ibp_npages;
int i;
+ LASSERT (p->ibp_device == NULL);
+
for (i = 0; i < npages; i++)
if (p->ibp_pages[i] != NULL)
__free_page(p->ibp_pages[i]);
}
void
-kiblnd_free_tx_descs (lnet_ni_t *ni)
+kiblnd_unmap_rx_descs(kib_conn_t *conn)
{
- int i;
- kib_net_t *net = ni->ni_data;
+ kib_rx_t *rx;
+ int i;
- LASSERT (net != NULL);
+ LASSERT (conn->ibc_rxs != NULL);
+ LASSERT (conn->ibc_rx_pages->ibp_device != NULL);
- if (net->ibn_tx_descs != NULL) {
- for (i = 0; i < IBLND_TX_MSGS(); i++) {
- kib_tx_t *tx = &net->ibn_tx_descs[i];
+ for (i = 0; i < IBLND_RX_MSGS(conn->ibc_version); i++) {
+ rx = &conn->ibc_rxs[i];
-#if IBLND_MAP_ON_DEMAND
- if (tx->tx_pages != NULL)
- LIBCFS_FREE(tx->tx_pages, LNET_MAX_IOV *
- sizeof(*tx->tx_pages));
-#else
- if (tx->tx_wrq != NULL)
- LIBCFS_FREE(tx->tx_wrq,
- (1 + IBLND_MAX_RDMA_FRAGS) *
- sizeof(*tx->tx_wrq));
-
- if (tx->tx_sge != NULL)
- LIBCFS_FREE(tx->tx_sge,
- (1 + IBLND_MAX_RDMA_FRAGS) *
- sizeof(*tx->tx_sge));
-
- if (tx->tx_rd != NULL)
- LIBCFS_FREE(tx->tx_rd,
- offsetof(kib_rdma_desc_t,
- rd_frags[IBLND_MAX_RDMA_FRAGS]));
-
- if (tx->tx_frags != NULL)
- LIBCFS_FREE(tx->tx_frags,
- IBLND_MAX_RDMA_FRAGS *
- sizeof(*tx->tx_frags));
-#endif
- }
+ LASSERT (rx->rx_nob >= 0); /* not posted */
- LIBCFS_FREE(net->ibn_tx_descs,
- IBLND_TX_MSGS() * sizeof(kib_tx_t));
+ kiblnd_dma_unmap_single(conn->ibc_rx_pages->ibp_device,
+ KIBLND_UNMAP_ADDR(rx, rx_msgunmap,
+ rx->rx_msgaddr),
+ IBLND_MSG_SIZE, DMA_FROM_DEVICE);
}
- if (net->ibn_tx_pages != NULL)
- kiblnd_free_pages(net->ibn_tx_pages);
+ conn->ibc_rx_pages->ibp_device = NULL;
+
+ kiblnd_free_pages(conn->ibc_rx_pages);
+
+ conn->ibc_rx_pages = NULL;
}
-int
-kiblnd_alloc_tx_descs (lnet_ni_t *ni)
+void
+kiblnd_map_rx_descs(kib_conn_t *conn)
{
- int i;
- int rc;
- kib_net_t *net = ni->ni_data;
-
- LASSERT (net != NULL);
+ kib_rx_t *rx;
+ struct page *pg;
+ int pg_off;
+ int ipg;
+ int i;
- rc = kiblnd_alloc_pages(&net->ibn_tx_pages, IBLND_TX_MSG_PAGES());
+ for (pg_off = ipg = i = 0;
+ i < IBLND_RX_MSGS(conn->ibc_version); i++) {
+ pg = conn->ibc_rx_pages->ibp_pages[ipg];
+ rx = &conn->ibc_rxs[i];
- if (rc != 0) {
- CERROR("Can't allocate tx pages\n");
- return rc;
- }
+ rx->rx_conn = conn;
+ rx->rx_msg = (kib_msg_t *)(((char *)page_address(pg)) + pg_off);
- LIBCFS_ALLOC (net->ibn_tx_descs,
- IBLND_TX_MSGS() * sizeof(kib_tx_t));
- if (net->ibn_tx_descs == NULL) {
- CERROR("Can't allocate %d tx descriptors\n", IBLND_TX_MSGS());
- return -ENOMEM;
- }
+ rx->rx_msgaddr = kiblnd_dma_map_single(conn->ibc_cmid->device,
+ rx->rx_msg, IBLND_MSG_SIZE,
+ DMA_FROM_DEVICE);
+ LASSERT (!kiblnd_dma_mapping_error(conn->ibc_cmid->device,
+ rx->rx_msgaddr));
+ KIBLND_UNMAP_ADDR_SET(rx, rx_msgunmap, rx->rx_msgaddr);
- memset(net->ibn_tx_descs, 0,
- IBLND_TX_MSGS() * sizeof(kib_tx_t));
+ CDEBUG(D_NET,"rx %d: %p "LPX64"("LPX64")\n",
+ i, rx->rx_msg, rx->rx_msgaddr,
+ lnet_page2phys(pg) + pg_off);
- for (i = 0; i < IBLND_TX_MSGS(); i++) {
- kib_tx_t *tx = &net->ibn_tx_descs[i];
+ pg_off += IBLND_MSG_SIZE;
+ LASSERT (pg_off <= PAGE_SIZE);
-#if IBLND_MAP_ON_DEMAND
- LIBCFS_ALLOC(tx->tx_pages, LNET_MAX_IOV *
- sizeof(*tx->tx_pages));
- if (tx->tx_pages == NULL) {
- CERROR("Can't allocate phys page vector[%d]\n",
- LNET_MAX_IOV);
- return -ENOMEM;
+ if (pg_off == PAGE_SIZE) {
+ pg_off = 0;
+ ipg++;
+ LASSERT (ipg <= IBLND_RX_MSG_PAGES(conn->ibc_version));
}
-#else
- LIBCFS_ALLOC(tx->tx_wrq,
- (1 + IBLND_MAX_RDMA_FRAGS) *
- sizeof(*tx->tx_wrq));
- if (tx->tx_wrq == NULL)
- return -ENOMEM;
-
- LIBCFS_ALLOC(tx->tx_sge,
- (1 + IBLND_MAX_RDMA_FRAGS) *
- sizeof(*tx->tx_sge));
- if (tx->tx_sge == NULL)
- return -ENOMEM;
-
- LIBCFS_ALLOC(tx->tx_rd,
- offsetof(kib_rdma_desc_t,
- rd_frags[IBLND_MAX_RDMA_FRAGS]));
- if (tx->tx_rd == NULL)
- return -ENOMEM;
-
- LIBCFS_ALLOC(tx->tx_frags,
- IBLND_MAX_RDMA_FRAGS *
- sizeof(*tx->tx_frags));
- if (tx->tx_frags == NULL)
- return -ENOMEM;
-#endif
}
- return 0;
+ conn->ibc_rx_pages->ibp_device = conn->ibc_cmid->device;
}
void
-kiblnd_unmap_tx_descs (lnet_ni_t *ni)
+kiblnd_unmap_tx_descs(lnet_ni_t *ni)
{
int i;
kib_tx_t *tx;
kib_net_t *net = ni->ni_data;
- LASSERT (net != NULL);
+ LASSERT (net->ibn_tx_pages != NULL);
+ LASSERT (net->ibn_tx_pages->ibp_device != NULL);
for (i = 0; i < IBLND_TX_MSGS(); i++) {
tx = &net->ibn_tx_descs[i];
- kiblnd_dma_unmap_single(net->ibn_dev->ibd_cmid->device,
+ kiblnd_dma_unmap_single(net->ibn_tx_pages->ibp_device,
KIBLND_UNMAP_ADDR(tx, tx_msgunmap,
tx->tx_msgaddr),
IBLND_MSG_SIZE, DMA_TO_DEVICE);
}
+
+ net->ibn_tx_pages->ibp_device = NULL;
}
void
kiblnd_map_tx_descs (lnet_ni_t *ni)
{
- int ipage = 0;
- int page_offset = 0;
- int i;
+ kib_net_t *net = ni->ni_data;
struct page *page;
kib_tx_t *tx;
- kib_net_t *net = ni->ni_data;
+ int page_offset;
+ int ipage;
+ int i;
LASSERT (net != NULL);
/* No fancy arithmetic when we do the buffer calculations */
CLASSERT (PAGE_SIZE % IBLND_MSG_SIZE == 0);
- for (i = 0; i < IBLND_TX_MSGS(); i++) {
+
+ for (ipage = page_offset = i = 0; i < IBLND_TX_MSGS(); i++) {
page = net->ibn_tx_pages->ibp_pages[ipage];
tx = &net->ibn_tx_descs[i];
tx->tx_msgaddr = kiblnd_dma_map_single(
net->ibn_dev->ibd_cmid->device,
tx->tx_msg, IBLND_MSG_SIZE, DMA_TO_DEVICE);
+ LASSERT (!kiblnd_dma_mapping_error(net->ibn_dev->ibd_cmid->device,
+ tx->tx_msgaddr));
KIBLND_UNMAP_ADDR_SET(tx, tx_msgunmap, tx->tx_msgaddr);
list_add(&tx->tx_list, &net->ibn_idle_txs);
LASSERT (ipage <= IBLND_TX_MSG_PAGES());
}
}
+
+ net->ibn_tx_pages->ibp_device = net->ibn_dev->ibd_cmid->device;
+}
+
+void
+kiblnd_free_tx_descs (lnet_ni_t *ni)
+{
+ int i;
+ kib_net_t *net = ni->ni_data;
+
+ LASSERT (net != NULL);
+
+ if (net->ibn_tx_pages != NULL)
+ kiblnd_free_pages(net->ibn_tx_pages);
+
+ if (net->ibn_tx_descs == NULL)
+ return;
+
+ for (i = 0; i < IBLND_TX_MSGS(); i++) {
+ kib_tx_t *tx = &net->ibn_tx_descs[i];
+
+ if (tx->tx_pages != NULL)
+ LIBCFS_FREE(tx->tx_pages,
+ LNET_MAX_IOV *
+ sizeof(*tx->tx_pages));
+
+ if (tx->tx_ipb != NULL)
+ LIBCFS_FREE(tx->tx_ipb,
+ IBLND_MAX_RDMA_FRAGS *
+ sizeof(*tx->tx_ipb));
+
+ if (tx->tx_frags != NULL)
+ LIBCFS_FREE(tx->tx_frags,
+ IBLND_MAX_RDMA_FRAGS *
+ sizeof(*tx->tx_frags));
+
+ if (tx->tx_wrq != NULL)
+ LIBCFS_FREE(tx->tx_wrq,
+ (1 + IBLND_MAX_RDMA_FRAGS) *
+ sizeof(*tx->tx_wrq));
+
+ if (tx->tx_sge != NULL)
+ LIBCFS_FREE(tx->tx_sge,
+ (1 + IBLND_MAX_RDMA_FRAGS) *
+ sizeof(*tx->tx_sge));
+
+ if (tx->tx_rd != NULL)
+ LIBCFS_FREE(tx->tx_rd,
+ offsetof(kib_rdma_desc_t,
+ rd_frags[IBLND_MAX_RDMA_FRAGS]));
+ }
+
+ LIBCFS_FREE(net->ibn_tx_descs,
+ IBLND_TX_MSGS() * sizeof(kib_tx_t));
+}
+
+int
+kiblnd_alloc_tx_descs (lnet_ni_t *ni)
+{
+ int i;
+ int rc;
+ kib_net_t *net = ni->ni_data;
+
+ LASSERT (net != NULL);
+
+ rc = kiblnd_alloc_pages(&net->ibn_tx_pages, IBLND_TX_MSG_PAGES());
+
+ if (rc != 0) {
+ CERROR("Can't allocate tx pages\n");
+ return rc;
+ }
+
+ LIBCFS_ALLOC (net->ibn_tx_descs,
+ IBLND_TX_MSGS() * sizeof(kib_tx_t));
+ if (net->ibn_tx_descs == NULL) {
+ CERROR("Can't allocate %d tx descriptors\n", IBLND_TX_MSGS());
+ return -ENOMEM;
+ }
+
+ memset(net->ibn_tx_descs, 0,
+ IBLND_TX_MSGS() * sizeof(kib_tx_t));
+
+ for (i = 0; i < IBLND_TX_MSGS(); i++) {
+ kib_tx_t *tx = &net->ibn_tx_descs[i];
+
+ if (net->ibn_fmrpool != NULL) {
+ LIBCFS_ALLOC(tx->tx_pages, LNET_MAX_IOV *
+ sizeof(*tx->tx_pages));
+ if (tx->tx_pages == NULL)
+ return -ENOMEM;
+ }
+
+ if (net->ibn_pmrpool != NULL) {
+ LIBCFS_ALLOC(tx->tx_ipb,
+ IBLND_MAX_RDMA_FRAGS *
+ sizeof(*tx->tx_ipb));
+ if (tx->tx_ipb == NULL)
+ return -ENOMEM;
+ }
+
+ LIBCFS_ALLOC(tx->tx_frags,
+ IBLND_MAX_RDMA_FRAGS *
+ sizeof(*tx->tx_frags));
+ if (tx->tx_frags == NULL)
+ return -ENOMEM;
+
+ LIBCFS_ALLOC(tx->tx_wrq,
+ (1 + IBLND_MAX_RDMA_FRAGS) *
+ sizeof(*tx->tx_wrq));
+ if (tx->tx_wrq == NULL)
+ return -ENOMEM;
+
+ LIBCFS_ALLOC(tx->tx_sge,
+ (1 + IBLND_MAX_RDMA_FRAGS) *
+ sizeof(*tx->tx_sge));
+ if (tx->tx_sge == NULL)
+ return -ENOMEM;
+
+ LIBCFS_ALLOC(tx->tx_rd,
+ offsetof(kib_rdma_desc_t,
+ rd_frags[IBLND_MAX_RDMA_FRAGS]));
+ if (tx->tx_rd == NULL)
+ return -ENOMEM;
+ }
+
+ return 0;
+}
+
+struct ib_mr *
+kiblnd_find_dma_mr(kib_net_t *net, __u64 addr, __u64 size)
+{
+ __u64 index;
+
+ LASSERT (net->ibn_dev->ibd_mrs[0] != NULL);
+
+ if (net->ibn_dev->ibd_nmrs == 1)
+ return net->ibn_dev->ibd_mrs[0];
+
+ index = addr >> net->ibn_dev->ibd_mr_shift;
+
+ if (index < net->ibn_dev->ibd_nmrs &&
+ index == ((addr + size - 1) >> net->ibn_dev->ibd_mr_shift))
+ return net->ibn_dev->ibd_mrs[index];
+
+ return NULL;
+}
+
+struct ib_mr *
+kiblnd_find_rd_dma_mr(kib_net_t *net, kib_rdma_desc_t *rd)
+{
+ struct ib_mr *prev_mr;
+ struct ib_mr *mr;
+ int i;
+
+ LASSERT (net->ibn_dev->ibd_mrs[0] != NULL);
+
+ if (*kiblnd_tunables.kib_map_on_demand > 0 &&
+ *kiblnd_tunables.kib_map_on_demand < rd->rd_nfrags)
+ return NULL;
+
+ if (net->ibn_dev->ibd_nmrs == 1)
+ return net->ibn_dev->ibd_mrs[0];
+
+ for (i = 0, mr = prev_mr = NULL;
+ i < rd->rd_nfrags; i++) {
+ mr = kiblnd_find_dma_mr(net,
+ rd->rd_frags[i].rf_addr,
+ rd->rd_frags[i].rf_nob);
+ if (prev_mr == NULL)
+ prev_mr = mr;
+
+ if (mr == NULL || prev_mr != mr) {
+ /* Can't covered by one single MR */
+ mr = NULL;
+ break;
+ }
+ }
+
+ return mr;
+}
+
+void
+kiblnd_dev_cleanup(kib_dev_t *ibdev)
+{
+ int i;
+
+ if (ibdev->ibd_mrs == NULL)
+ return;
+
+ for (i = 0; i < ibdev->ibd_nmrs; i++) {
+ if (ibdev->ibd_mrs[i] == NULL)
+ break;
+
+ ib_dereg_mr(ibdev->ibd_mrs[i]);
+ }
+
+ LIBCFS_FREE(ibdev->ibd_mrs, sizeof(*ibdev->ibd_mrs) * ibdev->ibd_nmrs);
+ ibdev->ibd_mrs = NULL;
+}
+
+int
+kiblnd_ib_create_fmr_pool(kib_dev_t *ibdev, struct ib_fmr_pool **fmrpp)
+{
+ /* FMR pool for RDMA */
+ struct ib_fmr_pool *fmrpool;
+ struct ib_fmr_pool_param param = {
+ .max_pages_per_fmr = LNET_MAX_PAYLOAD/PAGE_SIZE,
+ .page_shift = PAGE_SHIFT,
+ .access = (IB_ACCESS_LOCAL_WRITE |
+ IB_ACCESS_REMOTE_WRITE),
+ .pool_size = *kiblnd_tunables.kib_fmr_pool_size,
+ .dirty_watermark = *kiblnd_tunables.kib_fmr_flush_trigger,
+ .flush_function = NULL,
+ .flush_arg = NULL,
+ .cache = *kiblnd_tunables.kib_fmr_cache};
+
+ if (*kiblnd_tunables.kib_fmr_pool_size <
+ *kiblnd_tunables.kib_ntx) {
+ CERROR("Can't set fmr pool size (%d) < ntx(%d)\n",
+ *kiblnd_tunables.kib_fmr_pool_size,
+ *kiblnd_tunables.kib_ntx);
+ return -EINVAL;
+ }
+
+ fmrpool = ib_create_fmr_pool(ibdev->ibd_pd, ¶m);
+ if (IS_ERR(fmrpool))
+ return PTR_ERR(fmrpool);
+
+ *fmrpp = fmrpool;
+
+ return 0;
+}
+
+void
+kiblnd_phys_mr_unmap(kib_net_t *net, kib_phys_mr_t *pmr)
+{
+ kib_phys_mr_pool_t *pool = net->ibn_pmrpool;
+ struct ib_mr *mr;
+
+ spin_lock(&pool->ibmp_lock);
+
+ mr = pmr->ibpm_mr;
+
+ list_add(&pmr->ibpm_link, &pool->ibmp_free_list);
+ pool->ibmp_allocated --;
+
+ spin_unlock(&pool->ibmp_lock);
+
+ if (mr != NULL)
+ ib_dereg_mr(mr);
+}
+
+kib_phys_mr_t *
+kiblnd_phys_mr_map(kib_net_t *net, kib_rdma_desc_t *rd,
+ struct ib_phys_buf *ipb, __u64 *iova)
+{
+ kib_phys_mr_pool_t *pool = net->ibn_pmrpool;
+ kib_phys_mr_t *pmr;
+ int i;
+
+ LASSERT (ipb != NULL);
+
+ spin_lock(&pool->ibmp_lock);
+ if (list_empty(&pool->ibmp_free_list)) {
+ spin_unlock(&pool->ibmp_lock);
+ CERROR("pre-allocated MRs is not enough\n");
+
+ return NULL;
+ }
+
+ pmr = list_entry(pool->ibmp_free_list.next,
+ kib_phys_mr_t, ibpm_link);
+ list_del_init(&pmr->ibpm_link);
+ pool->ibmp_allocated ++;
+
+ spin_unlock(&pool->ibmp_lock);
+
+ for (i = 0; i < rd->rd_nfrags; i ++) {
+ ipb[i].addr = rd->rd_frags[i].rf_addr;
+ ipb[i].size = rd->rd_frags[i].rf_nob;
+ }
+
+ pmr->ibpm_mr = ib_reg_phys_mr(net->ibn_dev->ibd_pd, ipb,
+ rd->rd_nfrags,
+ IB_ACCESS_LOCAL_WRITE |
+ IB_ACCESS_REMOTE_WRITE,
+ iova);
+ if (!IS_ERR(pmr->ibpm_mr)) {
+ pmr->ibpm_iova = *iova;
+ return pmr;
+ }
+
+ CERROR("Failed ib_reg_phys_mr: %ld\n", PTR_ERR(pmr->ibpm_mr));
+ pmr->ibpm_mr = NULL;
+
+ spin_lock(&pool->ibmp_lock);
+
+ list_add(&pmr->ibpm_link, &pool->ibmp_free_list);
+ pool->ibmp_allocated --;
+
+ spin_unlock(&pool->ibmp_lock);
+
+ return NULL;
+}
+
+void
+kiblnd_destroy_pmr_pool(kib_phys_mr_pool_t *pool)
+{
+ kib_phys_mr_t *pmr;
+
+ LASSERT (pool->ibmp_allocated == 0);
+
+ while (!list_empty(&pool->ibmp_free_list)) {
+ pmr = list_entry(pool->ibmp_free_list.next,
+ kib_phys_mr_t, ibpm_link);
+
+ LASSERT (pmr->ibpm_mr == NULL);
+
+ list_del(&pmr->ibpm_link);
+
+ LIBCFS_FREE(pmr, sizeof(kib_phys_mr_t));
+ }
+
+ LIBCFS_FREE(pool, sizeof(kib_phys_mr_pool_t));
+}
+
+int
+kiblnd_create_pmr_pool(kib_dev_t *ibdev, kib_phys_mr_pool_t **poolpp)
+{
+ kib_phys_mr_pool_t *pool;
+ kib_phys_mr_t *pmr;
+ int i;
+
+ if (*kiblnd_tunables.kib_pmr_pool_size <
+ *kiblnd_tunables.kib_ntx) {
+ CERROR("Can't set pmr pool size (%d) < ntx(%d)\n",
+ *kiblnd_tunables.kib_pmr_pool_size,
+ *kiblnd_tunables.kib_ntx);
+ return -EINVAL;
+ }
+
+ LIBCFS_ALLOC(pool, sizeof(kib_phys_mr_pool_t));
+ if (pool == NULL)
+ return -ENOMEM;
+
+ spin_lock_init(&pool->ibmp_lock);
+
+ pool->ibmp_allocated = 0;
+ CFS_INIT_LIST_HEAD(&pool->ibmp_free_list);
+
+ for (i = 0; i < *kiblnd_tunables.kib_pmr_pool_size; i++) {
+ LIBCFS_ALLOC(pmr, sizeof(kib_phys_mr_t));
+
+ if (pmr == NULL) {
+ kiblnd_destroy_pmr_pool(pool);
+ return -ENOMEM;
+ }
+
+ memset(pmr, 0, sizeof(kib_phys_mr_t));
+
+ list_add(&pmr->ibpm_link, &pool->ibmp_free_list);
+ }
+
+ *poolpp = pool;
+
+ return 0;
+}
+
+static int
+kiblnd_dev_get_attr(kib_dev_t *ibdev)
+{
+ struct ib_device_attr *attr;
+ int rc;
+
+ /* XXX here should be HCA's page shift/size/mask in the future? */
+ ibdev->ibd_page_shift = PAGE_SHIFT;
+ ibdev->ibd_page_size = 1 << PAGE_SHIFT;
+ ibdev->ibd_page_mask = ~((__u64)ibdev->ibd_page_size - 1);
+
+ LIBCFS_ALLOC(attr, sizeof(*attr));
+ if (attr == NULL) {
+ CERROR("Out of memory\n");
+ return -ENOMEM;
+ }
+
+ rc = ib_query_device(ibdev->ibd_cmid->device, attr);
+ if (rc == 0)
+ ibdev->ibd_mr_size = attr->max_mr_size;
+
+ LIBCFS_FREE(attr, sizeof(*attr));
+
+ if (rc != 0) {
+ CERROR("Failed to query IB device: %d\n", rc);
+ return rc;
+ }
+
+#if 1
+ /* XXX We can't trust this value returned by Chelsio driver, it's wrong
+ * and we have reported the bug, remove these in the future when Chelsio
+ * bug got fixed. */
+ if (rdma_node_get_transport(ibdev->ibd_cmid->device->node_type) ==
+ RDMA_TRANSPORT_IWARP)
+ ibdev->ibd_mr_size = (1ULL << 32) - 1;
+#endif
+
+ if (ibdev->ibd_mr_size == ~0ULL) {
+ ibdev->ibd_mr_shift = 64;
+ return 0;
+ }
+
+ for (ibdev->ibd_mr_shift = 0;
+ ibdev->ibd_mr_shift < 64; ibdev->ibd_mr_shift ++) {
+ if (ibdev->ibd_mr_size == (1ULL << ibdev->ibd_mr_shift) ||
+ ibdev->ibd_mr_size == (1ULL << ibdev->ibd_mr_shift) - 1)
+ return 0;
+ }
+
+ CERROR("Invalid mr size: "LPX64"\n", ibdev->ibd_mr_size);
+ return -EINVAL;
+}
+
+int
+kiblnd_dev_setup(kib_dev_t *ibdev)
+{
+ struct ib_mr *mr;
+ int i;
+ int rc;
+ __u64 mm_size;
+ __u64 mr_size;
+ int acflags = IB_ACCESS_LOCAL_WRITE |
+ IB_ACCESS_REMOTE_WRITE;
+
+ rc = kiblnd_dev_get_attr(ibdev);
+ if (rc != 0)
+ return rc;
+
+ if (ibdev->ibd_mr_shift == 64) {
+ LIBCFS_ALLOC(ibdev->ibd_mrs, 1 * sizeof(*ibdev->ibd_mrs));
+ if (ibdev->ibd_mrs == NULL) {
+ CERROR("Failed to allocate MRs table\n");
+ return -ENOMEM;
+ }
+
+ ibdev->ibd_mrs[0] = NULL;
+ ibdev->ibd_nmrs = 1;
+
+ mr = ib_get_dma_mr(ibdev->ibd_pd, acflags);
+ if (IS_ERR(mr)) {
+ CERROR("Failed ib_get_dma_mr : %ld\n", PTR_ERR(mr));
+ kiblnd_dev_cleanup(ibdev);
+ return PTR_ERR(mr);
+ }
+
+ ibdev->ibd_mrs[0] = mr;
+
+ goto out;
+ }
+
+ mr_size = (1ULL << ibdev->ibd_mr_shift);
+ mm_size = (unsigned long)high_memory - PAGE_OFFSET;
+
+ ibdev->ibd_nmrs = (int)((mm_size + mr_size - 1) >> ibdev->ibd_mr_shift);
+
+ if (ibdev->ibd_mr_shift < 32 || ibdev->ibd_nmrs > 1024) {
+ /* it's 4T..., assume we will re-code at that time */
+ CERROR("Can't support memory size: x"LPX64
+ " with MR size: x"LPX64"\n", mm_size, mr_size);
+ return -EINVAL;
+ }
+
+ /* create an array of MRs to cover all memory */
+ LIBCFS_ALLOC(ibdev->ibd_mrs, sizeof(*ibdev->ibd_mrs) * ibdev->ibd_nmrs);
+ if (ibdev->ibd_mrs == NULL) {
+ CERROR("Failed to allocate MRs' table\n");
+ return -ENOMEM;
+ }
+
+ memset(ibdev->ibd_mrs, 0, sizeof(*ibdev->ibd_mrs) * ibdev->ibd_nmrs);
+
+ for (i = 0; i < ibdev->ibd_nmrs; i++) {
+ struct ib_phys_buf ipb;
+ __u64 iova;
+
+ ipb.size = ibdev->ibd_mr_size;
+ ipb.addr = i * mr_size;
+ iova = ipb.addr;
+
+ mr = ib_reg_phys_mr(ibdev->ibd_pd, &ipb, 1, acflags, &iova);
+ if (IS_ERR(mr)) {
+ CERROR("Failed ib_reg_phys_mr addr "LPX64
+ " size "LPX64" : %ld\n",
+ ipb.addr, ipb.size, PTR_ERR(mr));
+ kiblnd_dev_cleanup(ibdev);
+ return PTR_ERR(mr);
+ }
+
+ LASSERT (iova == ipb.addr);
+
+ ibdev->ibd_mrs[i] = mr;
+ }
+
+out:
+ CDEBUG(D_CONSOLE, "Register global MR array, MR size: "
+ LPX64", array size: %d\n",
+ ibdev->ibd_mr_size, ibdev->ibd_nmrs);
+
+ list_add_tail(&ibdev->ibd_list,
+ &kiblnd_data.kib_devs);
+ return 0;
+}
+
+void
+kiblnd_destroy_dev (kib_dev_t *dev)
+{
+ LASSERT (dev->ibd_nnets == 0);
+
+ if (!list_empty(&dev->ibd_list)) /* on kib_devs? */
+ list_del_init(&dev->ibd_list);
+
+ kiblnd_dev_cleanup(dev);
+
+ if (dev->ibd_pd != NULL)
+ ib_dealloc_pd(dev->ibd_pd);
+
+ if (dev->ibd_cmid != NULL)
+ rdma_destroy_id(dev->ibd_cmid);
+
+ LIBCFS_FREE(dev, sizeof(*dev));
}
void
case IBLND_INIT_NOTHING:
LASSERT (atomic_read(&net->ibn_nconns) == 0);
-#if IBLND_MAP_ON_DEMAND
if (net->ibn_fmrpool != NULL)
ib_destroy_fmr_pool(net->ibn_fmrpool);
-#endif
+ if (net->ibn_pmrpool != NULL)
+ kiblnd_destroy_pmr_pool(net->ibn_pmrpool);
+
if (net->ibn_dev != NULL &&
net->ibn_dev->ibd_nnets == 0)
kiblnd_destroy_dev(net->ibn_dev);
kiblnd_startup (lnet_ni_t *ni)
{
char *ifname;
+ kib_dev_t *ibdev = NULL;
kib_net_t *net;
- kib_dev_t *ibdev;
struct list_head *tmp;
struct timeval tv;
int rc;
- LASSERT (ni->ni_lnd == &the_kiblnd);
+ LASSERT (ni->ni_lnd == &the_o2iblnd);
if (kiblnd_data.kib_init == IBLND_INIT_NOTHING) {
rc = kiblnd_base_startup();
spin_lock_init(&net->ibn_tx_lock);
INIT_LIST_HEAD(&net->ibn_idle_txs);
- rc = kiblnd_alloc_tx_descs(ni);
- if (rc != 0) {
- CERROR("Can't allocate tx descs\n");
- goto failed;
- }
-
if (ni->ni_interfaces[0] != NULL) {
/* Use the IPoIB interface specified in 'networks=' */
goto failed;
}
- ibdev = NULL;
list_for_each (tmp, &kiblnd_data.kib_devs) {
ibdev = list_entry(tmp, kib_dev_t, ibd_list);
int up;
struct rdma_cm_id *id;
struct ib_pd *pd;
- struct ib_mr *mr;
struct sockaddr_in addr;
rc = libcfs_ipif_query(ifname, &up, &ip, &netmask);
memset(ibdev, 0, sizeof(*ibdev));
- INIT_LIST_HEAD(&ibdev->ibd_list); /* not yet in kib_devs */
+ CFS_INIT_LIST_HEAD(&ibdev->ibd_list); /* not yet in kib_devs */
ibdev->ibd_ifip = ip;
strcpy(&ibdev->ibd_ifname[0], ifname);
id = rdma_create_id(kiblnd_cm_callback, ibdev, RDMA_PS_TCP);
- if (!IS_ERR(id)) {
- ibdev->ibd_cmid = id;
- } else {
+ if (IS_ERR(id)) {
CERROR("Can't create listen ID: %ld\n", PTR_ERR(id));
goto failed;
}
+ ibdev->ibd_cmid = id;
+
memset(&addr, 0, sizeof(addr));
addr.sin_family = AF_INET;
addr.sin_port = htons(*kiblnd_tunables.kib_service);
LASSERT (id->device != NULL);
pd = ib_alloc_pd(id->device);
- if (!IS_ERR(pd)) {
- ibdev->ibd_pd = pd;
- } else {
+ if (IS_ERR(pd)) {
CERROR("Can't allocate PD: %ld\n", PTR_ERR(pd));
goto failed;
}
-#if IBLND_MAP_ON_DEMAND
- /* MR for sends and receives */
- mr = ib_get_dma_mr(pd, IB_ACCESS_LOCAL_WRITE);
-#else
- /* MR for sends, recieves _and_ RDMA...........v */
- mr = ib_get_dma_mr(pd, IB_ACCESS_LOCAL_WRITE |
- IB_ACCESS_REMOTE_WRITE);
-#endif
- if (!IS_ERR(mr)) {
- ibdev->ibd_mr = mr;
- } else {
- CERROR("Can't get MR: %ld\n", PTR_ERR(mr));
- goto failed;
- }
+ ibdev->ibd_pd = pd;
- rc = rdma_listen(id, 0);
+ rc = rdma_listen(id, 256);
if (rc != 0) {
CERROR("Can't start listener: %d\n", rc);
goto failed;
}
- list_add_tail(&ibdev->ibd_list,
- &kiblnd_data.kib_devs);
+ rc = kiblnd_dev_setup(ibdev);
+ if (rc != 0) {
+ CERROR("Can't setup device: %d\n", rc);
+ goto failed;
+ }
}
ni->ni_nid = LNET_MKNID(LNET_NIDNET(ni->ni_nid), ibdev->ibd_ifip);
net->ibn_dev = ibdev;
-#if IBLND_MAP_ON_DEMAND
- /* FMR pool for RDMA */
- {
- struct ib_fmr_pool *fmrpool;
- struct ib_fmr_pool_param param = {
- .max_pages_per_fmr = LNET_MAX_PAYLOAD/PAGE_SIZE,
- .page_shift = PAGE_SHIFT,
- .access = (IB_ACCESS_LOCAL_WRITE |
- IB_ACCESS_REMOTE_WRITE),
- .pool_size = *kiblnd_tunables.kib_fmr_pool_size,
- .dirty_watermark = *kiblnd_tunables.kib_fmr_flush_trigger,
- .flush_function = NULL,
- .flush_arg = NULL,
- .cache = *kiblnd_tunables.kib_fmr_cache};
-
- if (*kiblnd_tunables.kib_fmr_pool_size <
- *kiblnd_tunables.kib_ntx) {
- CERROR("Can't set fmr pool size (%d) < ntx(%d)\n",
- *kiblnd_tunables.kib_fmr_pool_size,
- *kiblnd_tunables.kib_ntx);
- goto failed;
+ if (*kiblnd_tunables.kib_map_on_demand > 0 ||
+ ibdev->ibd_nmrs > 1) { /* premapping can fail if ibd_nmr > 1,
+ * so we always create FMR/PMR pool and
+ * map-on-demand if premapping failed */
+ /* Map on demand */
+ rc = kiblnd_ib_create_fmr_pool(ibdev, &net->ibn_fmrpool);
+ if (rc == -ENOSYS) {
+ CDEBUG(D_CONSOLE, "No FMR, creating physical mapping\n");
+
+ rc = kiblnd_create_pmr_pool(ibdev, &net->ibn_pmrpool);
}
- fmrpool = ib_create_fmr_pool(ibdev->ibd_pd, ¶m);
- if (!IS_ERR(fmrpool)) {
- net->ibn_fmrpool = fmrpool;
- } else {
- CERROR("Can't create FMR pool: %ld\n",
- PTR_ERR(fmrpool));
+ if (rc != 0) {
+ CERROR("Can't create FMR or physical mapping pool: %d, "
+ "please disable map_on_demand and retry\n", rc);
goto failed;
}
+
+ }
+
+ rc = kiblnd_alloc_tx_descs(ni);
+ if (rc != 0) {
+ CERROR("Can't allocate tx descs\n");
+ goto failed;
}
-#endif
kiblnd_map_tx_descs(ni);
return 0;
failed:
+ if (net->ibn_dev == NULL && ibdev != NULL)
+ kiblnd_destroy_dev(ibdev);
+
kiblnd_shutdown(ni);
CDEBUG(D_NET, "kiblnd_startup failed\n");
void __exit
kiblnd_module_fini (void)
{
- lnet_unregister_lnd(&the_kiblnd);
+ lnet_unregister_lnd(&the_o2iblnd);
kiblnd_tunables_fini();
}
int rc;
CLASSERT (sizeof(kib_msg_t) <= IBLND_MSG_SIZE);
-#if !IBLND_MAP_ON_DEMAND
CLASSERT (offsetof(kib_msg_t, ibm_u.get.ibgm_rd.rd_frags[IBLND_MAX_RDMA_FRAGS])
<= IBLND_MSG_SIZE);
CLASSERT (offsetof(kib_msg_t, ibm_u.putack.ibpam_rd.rd_frags[IBLND_MAX_RDMA_FRAGS])
<= IBLND_MSG_SIZE);
-#endif
+
rc = kiblnd_tunables_init();
if (rc != 0)
return rc;
- lnet_register_lnd(&the_kiblnd);
+ lnet_register_lnd(&the_o2iblnd);
return 0;
}
MODULE_AUTHOR("Sun Microsystems, Inc. <http://www.lustre.org/>");
-MODULE_DESCRIPTION("Kernel OpenIB gen2 LND v1.00");
+MODULE_DESCRIPTION("Kernel OpenIB gen2 LND v2.00");
MODULE_LICENSE("GPL");
module_init(kiblnd_module_init);
#define IBLND_PEER_HASH_SIZE 101 /* # peer lists */
#define IBLND_RESCHED 100 /* # scheduler loops before reschedule */
-#define IBLND_MSG_QUEUE_SIZE 8 /* # messages/RDMAs in-flight */
-#define IBLND_CREDIT_HIGHWATER 7 /* when eagerly to return credits */
-#define IBLND_MSG_SIZE (4<<10) /* max size of queued messages (inc hdr) */
-
-#define IBLND_MAP_ON_DEMAND 0
-#if IBLND_MAP_ON_DEMAND
-# define IBLND_MAX_RDMA_FRAGS 1
-#else
-# define IBLND_MAX_RDMA_FRAGS LNET_MAX_IOV
-#endif
-
-/************************/
-/* derived constants... */
-
-/* TX messages (shared by all connections) */
-#define IBLND_TX_MSGS() (*kiblnd_tunables.kib_ntx)
-#define IBLND_TX_MSG_BYTES() (IBLND_TX_MSGS() * IBLND_MSG_SIZE)
-#define IBLND_TX_MSG_PAGES() ((IBLND_TX_MSG_BYTES() + PAGE_SIZE - 1)/PAGE_SIZE)
-
-/* RX messages (per connection) */
-#define IBLND_RX_MSGS (IBLND_MSG_QUEUE_SIZE * 2)
-#define IBLND_RX_MSG_BYTES (IBLND_RX_MSGS * IBLND_MSG_SIZE)
-#define IBLND_RX_MSG_PAGES ((IBLND_RX_MSG_BYTES + PAGE_SIZE - 1)/PAGE_SIZE)
-
-/* WRs and CQEs (per connection) */
-#define IBLND_RECV_WRS IBLND_RX_MSGS
-#define IBLND_SEND_WRS ((*kiblnd_tunables.kib_concurrent_sends) * \
- (1 + IBLND_MAX_RDMA_FRAGS))
-#define IBLND_CQ_ENTRIES() (IBLND_RECV_WRS + IBLND_SEND_WRS)
typedef struct
{
int *kib_ntx; /* # tx descs */
int *kib_credits; /* # concurrent sends */
int *kib_peercredits; /* # concurrent sends to 1 peer */
+ int *kib_peercredits_hiw; /* # when eagerly to return credits */
int *kib_peertimeout; /* seconds to consider peer dead */
char **kib_default_ipif; /* default IPoIB interface */
int *kib_retry_count;
int *kib_rnr_retry_count;
int *kib_concurrent_sends; /* send work queue sizing */
int *kib_ib_mtu; /* IB MTU */
-#if IBLND_MAP_ON_DEMAND
+ int *kib_map_on_demand; /* map-on-demand if RD has more fragments
+ * than this value, 0 disable map-on-demand */
+ int *kib_pmr_pool_size; /* # physical MR in pool */
int *kib_fmr_pool_size; /* # FMRs in pool */
int *kib_fmr_flush_trigger; /* When to trigger FMR flush */
int *kib_fmr_cache; /* enable FMR pool cache? */
-#endif
#if defined(CONFIG_SYSCTL) && !CFS_SYSFS_MODULE_PARM
cfs_sysctl_table_header_t *kib_sysctl; /* sysctl interface */
#endif
} kib_tunables_t;
+extern kib_tunables_t kiblnd_tunables;
+
+#define IBLND_MSG_QUEUE_SIZE_V1 8 /* V1 only : # messages/RDMAs in-flight */
+#define IBLND_CREDIT_HIGHWATER_V1 7 /* V1 only : when eagerly to return credits */
+
+#define IBLND_CREDITS_DEFAULT 8 /* default # of peer credits */
+#define IBLND_CREDITS_MAX 4096 /* Max # of peer credits */
+
+#define IBLND_MSG_QUEUE_SIZE(v) ((v) == IBLND_MSG_VERSION_1 ? \
+ IBLND_MSG_QUEUE_SIZE_V1 : \
+ *kiblnd_tunables.kib_peercredits) /* # messages/RDMAs in-flight */
+#define IBLND_CREDITS_HIGHWATER(v) ((v) == IBLND_MSG_VERSION_1 ? \
+ IBLND_CREDIT_HIGHWATER_V1 : \
+ *kiblnd_tunables.kib_peercredits_hiw) /* when eagerly to return credits */
+
+static inline int
+kiblnd_concurrent_sends_v1(void)
+{
+ if (*kiblnd_tunables.kib_concurrent_sends > IBLND_MSG_QUEUE_SIZE_V1 * 2)
+ return IBLND_MSG_QUEUE_SIZE_V1 * 2;
+
+ if (*kiblnd_tunables.kib_concurrent_sends < IBLND_MSG_QUEUE_SIZE_V1 / 2)
+ return IBLND_MSG_QUEUE_SIZE_V1 / 2;
+
+ return *kiblnd_tunables.kib_concurrent_sends;
+}
+
+#define IBLND_CONCURRENT_SENDS(v) ((v) == IBLND_MSG_VERSION_1 ? \
+ kiblnd_concurrent_sends_v1() : \
+ *kiblnd_tunables.kib_concurrent_sends)
+/* 2 OOB shall suffice for 1 keepalive and 1 returning credits */
+#define IBLND_OOB_CAPABLE(v) ((v) != IBLND_MSG_VERSION_1)
+#define IBLND_OOB_MSGS(v) (IBLND_OOB_CAPABLE(v) ? 2 : 0)
+
+#define IBLND_MSG_SIZE (4<<10) /* max size of queued messages (inc hdr) */
+#define IBLND_MAX_RDMA_FRAGS LNET_MAX_IOV /* max # of fragments supported */
+#define IBLND_CFG_RDMA_FRAGS (*kiblnd_tunables.kib_map_on_demand != 0 ? \
+ *kiblnd_tunables.kib_map_on_demand : \
+ IBLND_MAX_RDMA_FRAGS) /* max # of fragments configured by user */
+#define IBLND_RDMA_FRAGS(v) ((v) == IBLND_MSG_VERSION_1 ? \
+ IBLND_MAX_RDMA_FRAGS : IBLND_CFG_RDMA_FRAGS)
+
+/************************/
+/* derived constants... */
+
+/* TX messages (shared by all connections) */
+#define IBLND_TX_MSGS() (*kiblnd_tunables.kib_ntx)
+#define IBLND_TX_MSG_BYTES() (IBLND_TX_MSGS() * IBLND_MSG_SIZE)
+#define IBLND_TX_MSG_PAGES() ((IBLND_TX_MSG_BYTES() + PAGE_SIZE - 1) / PAGE_SIZE)
+
+/* RX messages (per connection) */
+#define IBLND_RX_MSGS(v) (IBLND_MSG_QUEUE_SIZE(v) * 2 + IBLND_OOB_MSGS(v))
+#define IBLND_RX_MSG_BYTES(v) (IBLND_RX_MSGS(v) * IBLND_MSG_SIZE)
+#define IBLND_RX_MSG_PAGES(v) ((IBLND_RX_MSG_BYTES(v) + PAGE_SIZE - 1) / PAGE_SIZE)
+
+/* WRs and CQEs (per connection) */
+#define IBLND_RECV_WRS(v) IBLND_RX_MSGS(v)
+#define IBLND_SEND_WRS(v) ((IBLND_RDMA_FRAGS(v) + 1) * IBLND_CONCURRENT_SENDS(v))
+#define IBLND_CQ_ENTRIES(v) (IBLND_RECV_WRS(v) + IBLND_SEND_WRS(v))
+
typedef struct
{
+ struct ib_device *ibp_device; /* device for mapping */
int ibp_npages; /* # pages */
struct page *ibp_pages[0];
} kib_pages_t;
-typedef struct
+typedef struct {
+ spinlock_t ibmp_lock; /* serialize */
+ int ibmp_allocated; /* MR in use */
+ struct list_head ibmp_free_list; /* pre-allocated MR */
+} kib_phys_mr_pool_t;
+
+typedef struct {
+ struct list_head ibpm_link; /* link node */
+ struct ib_mr *ibpm_mr; /* MR */
+ __u64 ibpm_iova; /* Virtual I/O address */
+ int ibpm_refcount; /* reference count */
+} kib_phys_mr_t;
+
+typedef struct
{
struct list_head ibd_list; /* chain on kib_devs */
__u32 ibd_ifip; /* IPoIB interface IP */
struct rdma_cm_id *ibd_cmid; /* IB listener (bound to 1 device) */
struct ib_pd *ibd_pd; /* PD for the device */
- struct ib_mr *ibd_mr; /* MR for non RDMA I/O */
+ int ibd_page_shift; /* page shift of current HCA */
+ int ibd_page_size; /* page size of current HCA */
+ __u64 ibd_page_mask; /* page mask of current HCA */
+ int ibd_mr_shift; /* bits shift of max MR size */
+ __u64 ibd_mr_size; /* size of MR */
+
+ int ibd_nmrs; /* # of global MRs */
+ struct ib_mr **ibd_mrs; /* MR for non RDMA I/O */
} kib_dev_t;
typedef struct
atomic_t ibn_npeers; /* # peers extant */
atomic_t ibn_nconns; /* # connections extant */
+ __u64 ibn_tx_next_cookie; /* RDMA completion cookie */
struct kib_tx *ibn_tx_descs; /* all the tx descriptors */
kib_pages_t *ibn_tx_pages; /* premapped tx msg pages */
struct list_head ibn_idle_txs; /* idle tx descriptors */
spinlock_t ibn_tx_lock; /* serialise */
-#if IBLND_MAP_ON_DEMAND
struct ib_fmr_pool *ibn_fmrpool; /* FMR pool for RDMA I/O */
-#endif
+ kib_phys_mr_pool_t *ibn_pmrpool; /* Physical MR pool for RDMA I/O */
kib_dev_t *ibn_dev; /* underlying IB device */
} kib_net_t;
struct list_head kib_sched_conns; /* conns to check for rx completions */
spinlock_t kib_sched_lock; /* serialise */
- __u64 kib_next_tx_cookie; /* RDMA completion cookie */
struct ib_qp_attr kib_error_qpa; /* QP->ERROR */
} kib_data_t;
char ibim_payload[0]; /* piggy-backed payload */
} WIRE_ATTR kib_immediate_msg_t;
-#if IBLND_MAP_ON_DEMAND
-typedef struct
-{
- __u64 rd_addr; /* IO VMA address */
- __u32 rd_nob; /* # of bytes */
- __u32 rd_key; /* remote key */
-} WIRE_ATTR kib_rdma_desc_t;
-#else
typedef struct
{
__u32 rf_nob; /* # bytes this frag */
__u32 rd_nfrags; /* # fragments */
kib_rdma_frag_t rd_frags[0]; /* buffer frags */
} WIRE_ATTR kib_rdma_desc_t;
-#endif
-
+
typedef struct
{
lnet_hdr_t ibprm_hdr; /* portals header */
#define IBLND_MSG_MAGIC LNET_PROTO_IB_MAGIC /* unique magic */
-#define IBLND_MSG_VERSION 0x11
+#define IBLND_MSG_VERSION_1 0x11
+#define IBLND_MSG_VERSION_2 0x12
+#define IBLND_MSG_VERSION IBLND_MSG_VERSION_2
#define IBLND_MSG_CONNREQ 0xc0 /* connection request */
#define IBLND_MSG_CONNACK 0xc1 /* connection acknowledge */
__u32 ibr_magic; /* sender's magic */
__u16 ibr_version; /* sender's version */
__u8 ibr_why; /* reject reason */
+ __u8 ibr_padding; /* padding */
+ __u64 ibr_incarnation; /* incarnation of peer */
+ kib_connparams_t ibr_cp; /* connection parameters */
} WIRE_ATTR kib_rej_t;
-
/* connection rejection reasons */
#define IBLND_REJECT_CONN_RACE 1 /* You lost connection race */
#define IBLND_REJECT_NO_RESOURCES 2 /* Out of memory/conns etc */
#define IBLND_REJECT_FATAL 3 /* Anything else */
+#define IBLND_REJECT_CONN_UNCOMPAT 4 /* incompatible version peer */
+#define IBLND_REJECT_CONN_STALE 5 /* stale peer */
+
+#define IBLND_REJECT_RDMA_FRAGS 6 /* Fatal: peer's rdma frags can't match mine */
+#define IBLND_REJECT_MSG_QUEUE_SIZE 7 /* Fatal: peer's msg queue size can't match mine */
+
/***********************************************************************/
typedef struct kib_rx /* receive message */
__u64 tx_msgaddr; /* message buffer (I/O addr) */
DECLARE_PCI_UNMAP_ADDR (tx_msgunmap); /* for dma_unmap_single() */
int tx_nwrq; /* # send work items */
-#if IBLND_MAP_ON_DEMAND
- struct ib_send_wr tx_wrq[2]; /* send work items... */
- struct ib_sge tx_sge[2]; /* ...and their memory */
- kib_rdma_desc_t tx_rd[1]; /* rdma descriptor */
- __u64 *tx_pages; /* rdma phys page addrs */
- struct ib_pool_fmr *tx_fmr; /* rdma mapping (mapped if != NULL) */
-#else
struct ib_send_wr *tx_wrq; /* send work items... */
struct ib_sge *tx_sge; /* ...and their memory */
kib_rdma_desc_t *tx_rd; /* rdma descriptor */
int tx_nfrags; /* # entries in... */
struct scatterlist *tx_frags; /* dma_map_sg descriptor */
+ struct ib_phys_buf *tx_ipb; /* physical buffer (for iWARP) */
+ __u64 *tx_pages; /* rdma phys page addrs */
+ union {
+ kib_phys_mr_t *pmr; /* MR for physical buffer */
+ struct ib_pool_fmr *fmr; /* rdma mapping (mapped if != NULL) */
+ } tx_u;
int tx_dmadir; /* dma direction */
-#endif
} kib_tx_t;
typedef struct kib_connvars
struct kib_peer *ibc_peer; /* owning peer */
struct list_head ibc_list; /* stash on peer's conn list */
struct list_head ibc_sched_list; /* schedule for attention */
+ __u16 ibc_version; /* version of connection */
__u64 ibc_incarnation; /* which instance of the peer */
atomic_t ibc_refcount; /* # users */
int ibc_state; /* what's happening */
int ibc_nsends_posted; /* # uncompleted sends */
+ int ibc_noops_posted; /* # uncompleted NOOPs */
int ibc_credits; /* # credits I have */
int ibc_outstanding_credits; /* # credits to return */
int ibc_reserved_credits;/* # ACK/DONE msg credits */
int ibc_comms_error; /* set on comms error */
- int ibc_nrx:8; /* receive buffers owned */
+ int ibc_nrx:16; /* receive buffers owned */
int ibc_scheduled:1; /* scheduled for attention */
int ibc_ready:1; /* CQ callback fired */
unsigned long ibc_last_send; /* time of last send */
struct list_head ibc_early_rxs; /* rxs completed before ESTABLISHED */
- struct list_head ibc_tx_noops; /* IBLND_MSG_NOOPs */
struct list_head ibc_tx_queue; /* sends that need a credit */
struct list_head ibc_tx_queue_nocred;/* sends that don't need a credit */
struct list_head ibc_tx_queue_rsrvd; /* sends that need to reserve an ACK/DONE msg */
atomic_t ibp_refcount; /* # users */
struct list_head ibp_conns; /* all active connections */
struct list_head ibp_tx_queue; /* msgs waiting for a conn */
+ __u16 ibp_version; /* version of peer */
+ __u64 ibp_incarnation; /* incarnation of peer */
int ibp_connecting; /* current active connection attempts */
int ibp_accepting; /* current passive connection attempts */
int ibp_error; /* errno on closing this peer */
cfs_time_t ibp_last_alive; /* when (in jiffies) I was last alive */
} kib_peer_t;
-
extern kib_data_t kiblnd_data;
-extern kib_tunables_t kiblnd_tunables;
#define kiblnd_conn_addref(conn) \
do { \
{
LASSERT (conn->ibc_state >= IBLND_CONN_ESTABLISHED);
- if (conn->ibc_outstanding_credits < IBLND_CREDIT_HIGHWATER &&
+ if (conn->ibc_outstanding_credits <
+ IBLND_CREDITS_HIGHWATER(conn->ibc_version) &&
!kiblnd_send_keepalive(conn))
return 0; /* No need to send NOOP */
- if (!list_empty(&conn->ibc_tx_noops) || /* NOOP already queued */
- !list_empty(&conn->ibc_tx_queue_nocred) || /* can be piggybacked */
- conn->ibc_credits == 0) /* no credit */
- return 0;
+ if (!list_empty(&conn->ibc_tx_queue_nocred))
+ return 0; /* NOOP can be piggybacked */
- if (conn->ibc_credits == 1 && /* last credit reserved for */
- conn->ibc_outstanding_credits == 0) /* giving back credits */
- return 0;
+ if (!IBLND_OOB_CAPABLE(conn->ibc_version))
+ return list_empty(&conn->ibc_tx_queue); /* can't piggyback? */
/* No tx to piggyback NOOP onto or no credit to send a tx */
- return (list_empty(&conn->ibc_tx_queue) || conn->ibc_credits == 1);
+ return (list_empty(&conn->ibc_tx_queue) || conn->ibc_credits == 0);
}
static inline void
&kiblnd_data.kib_error_qpa, IB_QP_STATE);
}
+static inline const char *
+kiblnd_queue2str (kib_conn_t *conn, struct list_head *q)
+{
+ if (q == &conn->ibc_tx_queue)
+ return "tx_queue";
+
+ if (q == &conn->ibc_tx_queue_rsrvd)
+ return "tx_queue_rsrvd";
+
+ if (q == &conn->ibc_tx_queue_nocred)
+ return "tx_queue_nocred";
+
+ if (q == &conn->ibc_active_txs)
+ return "active_txs";
+
+ LBUG();
+ return NULL;
+}
+
/* CAVEAT EMPTOR: We rely on descriptor alignment to allow us to use the
* lowest bits of the work request id to stash the work item type. */
mb();
}
-#if IBLND_MAP_ON_DEMAND
-static inline int
-kiblnd_rd_size (kib_rdma_desc_t *rd)
+static inline void
+kiblnd_init_msg (kib_msg_t *msg, int type, int body_nob)
{
- return rd->rd_nob;
+ msg->ibm_type = type;
+ msg->ibm_nob = offsetof(kib_msg_t, ibm_u) + body_nob;
}
-#else
+
static inline int
kiblnd_rd_size (kib_rdma_desc_t *rd)
{
int i;
int size;
-
+
for (i = size = 0; i < rd->rd_nfrags; i++)
size += rd->rd_frags[i].rf_nob;
-
+
return size;
}
-#endif
+
+static inline __u64
+kiblnd_rd_frag_addr(kib_rdma_desc_t *rd, int index)
+{
+ return rd->rd_frags[index].rf_addr;
+}
+
+static inline __u32
+kiblnd_rd_frag_size(kib_rdma_desc_t *rd, int index)
+{
+ return rd->rd_frags[index].rf_nob;
+}
+
+static inline __u32
+kiblnd_rd_frag_key(kib_rdma_desc_t *rd, int index)
+{
+ return rd->rd_key;
+}
+
+static inline int
+kiblnd_rd_consume_frag(kib_rdma_desc_t *rd, int index, __u32 nob)
+{
+ if (nob < rd->rd_frags[index].rf_nob) {
+ rd->rd_frags[index].rf_addr += nob;
+ rd->rd_frags[index].rf_nob -= nob;
+ } else {
+ index ++;
+ }
+
+ return index;
+}
+
+static inline int
+kiblnd_rd_msg_size(kib_rdma_desc_t *rd, int msgtype, int n)
+{
+ LASSERT (msgtype == IBLND_MSG_GET_REQ ||
+ msgtype == IBLND_MSG_PUT_ACK);
+
+ return msgtype == IBLND_MSG_GET_REQ ?
+ offsetof(kib_get_msg_t, ibgm_rd.rd_frags[n]) :
+ offsetof(kib_putack_msg_t, ibpam_rd.rd_frags[n]);
+}
#ifdef HAVE_OFED_IB_DMA_MAP
+static inline __u64
+kiblnd_dma_mapping_error(struct ib_device *dev, u64 dma_addr)
+{
+ return ib_dma_mapping_error(dev, dma_addr);
+}
+
static inline __u64 kiblnd_dma_map_single(struct ib_device *dev,
void *msg, size_t size,
enum dma_data_direction direction)
#else
+static inline __u64
+kiblnd_dma_mapping_error(struct ib_device *dev, dma_addr_t dma_addr)
+{
+ return dma_mapping_error(dma_addr);
+}
+
static inline dma_addr_t kiblnd_dma_map_single(struct ib_device *dev,
void *msg, size_t size,
enum dma_data_direction direction)
#endif
+struct ib_mr *kiblnd_find_rd_dma_mr(kib_net_t *net,
+ kib_rdma_desc_t *rd);
+struct ib_mr *kiblnd_find_dma_mr(kib_net_t *net,
+ __u64 addr, __u64 size);
+void kiblnd_map_rx_descs(kib_conn_t *conn);
+void kiblnd_unmap_rx_descs(kib_conn_t *conn);
+void kiblnd_map_tx_descs (lnet_ni_t *ni);
+void kiblnd_unmap_tx_descs(lnet_ni_t *ni);
+int kiblnd_map_tx(lnet_ni_t *ni, kib_tx_t *tx,
+ kib_rdma_desc_t *rd, int nfrags);
+void kiblnd_unmap_tx(lnet_ni_t *ni, kib_tx_t *tx);
+kib_phys_mr_t *kiblnd_phys_mr_map(kib_net_t *net, kib_rdma_desc_t *rd,
+ struct ib_phys_buf *ipb, __u64 *iova);
+void kiblnd_phys_mr_unmap(kib_net_t *net, kib_phys_mr_t *pmr);
+
int kiblnd_startup (lnet_ni_t *ni);
void kiblnd_shutdown (lnet_ni_t *ni);
int kiblnd_ctl (lnet_ni_t *ni, unsigned int cmd, void *arg);
int kiblnd_cm_callback(struct rdma_cm_id *cmid,
struct rdma_cm_event *event);
+int kiblnd_translate_mtu(int value);
int kiblnd_create_peer (lnet_ni_t *ni, kib_peer_t **peerp, lnet_nid_t nid);
void kiblnd_destroy_peer (kib_peer_t *peer);
void kiblnd_peer_alive (kib_peer_t *peer);
kib_peer_t *kiblnd_find_peer_locked (lnet_nid_t nid);
void kiblnd_peer_connect_failed (kib_peer_t *peer, int active, int error);
-int kiblnd_close_stale_conns_locked (kib_peer_t *peer, __u64 incarnation);
+int kiblnd_close_stale_conns_locked (kib_peer_t *peer,
+ int version, __u64 incarnation);
+int kiblnd_close_peer_conns_locked (kib_peer_t *peer, int why);
void kiblnd_connreq_done(kib_conn_t *conn, int status);
kib_conn_t *kiblnd_create_conn (kib_peer_t *peer, struct rdma_cm_id *cmid,
- int state);
+ int state, int version);
void kiblnd_destroy_conn (kib_conn_t *conn);
void kiblnd_close_conn (kib_conn_t *conn, int error);
void kiblnd_close_conn_locked (kib_conn_t *conn, int error);
-int kiblnd_init_rdma (lnet_ni_t *ni, kib_tx_t *tx, int type,
+int kiblnd_init_rdma (kib_conn_t *conn, kib_tx_t *tx, int type,
int nob, kib_rdma_desc_t *dstrd, __u64 dstcookie);
void kiblnd_launch_tx (lnet_ni_t *ni, kib_tx_t *tx, lnet_nid_t nid);
void kiblnd_cq_event(struct ib_event *event, void *arg);
void kiblnd_cq_completion(struct ib_cq *cq, void *arg);
-void kiblnd_init_msg (kib_msg_t *msg, int type, int body_nob);
-void kiblnd_pack_msg (lnet_ni_t *ni, kib_msg_t *msg,
+void kiblnd_pack_msg (lnet_ni_t *ni, kib_msg_t *msg, int version,
int credits, lnet_nid_t dstnid, __u64 dststamp);
int kiblnd_unpack_msg(kib_msg_t *msg, int nob);
int kiblnd_post_rx (kib_rx_t *rx, int credit);
#include "o2iblnd.h"
-char *
-kiblnd_msgtype2str(int type)
-{
- switch (type) {
- case IBLND_MSG_CONNREQ:
- return "CONNREQ";
-
- case IBLND_MSG_CONNACK:
- return "CONNACK";
-
- case IBLND_MSG_NOOP:
- return "NOOP";
-
- case IBLND_MSG_IMMEDIATE:
- return "IMMEDIATE";
-
- case IBLND_MSG_PUT_REQ:
- return "PUT_REQ";
-
- case IBLND_MSG_PUT_NAK:
- return "PUT_NAK";
-
- case IBLND_MSG_PUT_ACK:
- return "PUT_ACK";
-
- case IBLND_MSG_PUT_DONE:
- return "PUT_DONE";
-
- case IBLND_MSG_GET_REQ:
- return "GET_REQ";
-
- case IBLND_MSG_GET_DONE:
- return "GET_DONE";
-
- default:
- return "???";
- }
-}
-
void
kiblnd_tx_done (lnet_ni_t *ni, kib_tx_t *tx)
{
LASSERT (tx->tx_sending == 0); /* mustn't be awaiting sent callback */
LASSERT (!tx->tx_waiting); /* mustn't be awaiting peer response */
-#if IBLND_MAP_ON_DEMAND
- if (tx->tx_fmr != NULL) {
- rc = ib_fmr_pool_unmap(tx->tx_fmr);
- LASSERT (rc == 0);
+ kiblnd_unmap_tx(ni, tx);
- if (tx->tx_status != 0) {
- rc = ib_flush_fmr_pool(net->ibn_fmrpool);
- LASSERT (rc == 0);
- }
-
- tx->tx_fmr = NULL;
- }
-#else
- if (tx->tx_nfrags != 0) {
- kiblnd_dma_unmap_sg(net->ibn_dev->ibd_cmid->device,
- tx->tx_frags, tx->tx_nfrags, tx->tx_dmadir);
- tx->tx_nfrags = 0;
- }
-#endif
/* tx may have up to 2 lnet msgs to finalise */
lntmsg[0] = tx->tx_lntmsg[0]; tx->tx_lntmsg[0] = NULL;
lntmsg[1] = tx->tx_lntmsg[1]; tx->tx_lntmsg[1] = NULL;
kiblnd_txlist_done (lnet_ni_t *ni, struct list_head *txlist, int status)
{
kib_tx_t *tx;
-
+
while (!list_empty (txlist)) {
tx = list_entry (txlist->next, kib_tx_t, tx_list);
- list_del (&tx->tx_list);
+ list_del(&tx->tx_list);
/* complete now */
tx->tx_waiting = 0;
tx->tx_status = status;
/* Allocate a new completion cookie. It might not be needed,
* but we've got a lock right now and we're unlikely to
* wrap... */
- tx->tx_cookie = kiblnd_data.kib_next_tx_cookie++;
+ tx->tx_cookie = net->ibn_tx_next_cookie++;
spin_unlock(&net->ibn_tx_lock);
LASSERT (tx->tx_conn == NULL);
LASSERT (tx->tx_lntmsg[0] == NULL);
LASSERT (tx->tx_lntmsg[1] == NULL);
-#if IBLND_MAP_ON_DEMAND
- LASSERT (tx->tx_fmr == NULL);
-#else
+ LASSERT (tx->tx_u.fmr == NULL);
LASSERT (tx->tx_nfrags == 0);
-#endif
return tx;
}
{
kib_conn_t *conn = rx->rx_conn;
kib_net_t *net = conn->ibc_peer->ibp_ni->ni_data;
- struct ib_recv_wr *bad_wrq;
+ struct ib_recv_wr *bad_wrq = NULL;
+ struct ib_mr *mr;
int rc;
LASSERT (net != NULL);
credit == IBLND_POSTRX_PEER_CREDIT ||
credit == IBLND_POSTRX_RSRVD_CREDIT);
+ mr = kiblnd_find_dma_mr(net, rx->rx_msgaddr, IBLND_MSG_SIZE);
+ LASSERT (mr != NULL);
+
+ rx->rx_sge.lkey = mr->lkey;
+ rx->rx_sge.addr = rx->rx_msgaddr;
rx->rx_sge.length = IBLND_MSG_SIZE;
- rx->rx_sge.lkey = net->ibn_dev->ibd_mr->lkey;
- rx->rx_sge.addr = rx->rx_msgaddr;
rx->rx_wrq.next = NULL;
rx->rx_wrq.sg_list = &rx->rx_sge;
rx->rx_nob = -1; /* flag posted */
rc = ib_post_recv(conn->ibc_cmid->qp, &rx->rx_wrq, &bad_wrq);
+ if (rc != 0) {
+ CERROR("Can't post rx for %s: %d, bad_wrq: %p\n",
+ libcfs_nid2str(conn->ibc_peer->ibp_nid), rc, bad_wrq);
+ rx->rx_nob = 0;
+ }
if (conn->ibc_state < IBLND_CONN_ESTABLISHED) /* Initial post */
return rc;
if (rc != 0) {
- CERROR("Can't post rx for %s: %d\n",
- libcfs_nid2str(conn->ibc_peer->ibp_nid), rc);
kiblnd_close_conn(conn, rc);
kiblnd_drop_rx(rx); /* No more posts for this rx */
return rc;
/* Have I received credits that will let me send? */
spin_lock(&conn->ibc_lock);
- if (conn->ibc_credits + credits > IBLND_MSG_QUEUE_SIZE) {
+ if (conn->ibc_credits + credits >
+ IBLND_MSG_QUEUE_SIZE(conn->ibc_version)) {
rc2 = conn->ibc_credits;
spin_unlock(&conn->ibc_lock);
CERROR("Bad credits from %s: %d + %d > %d\n",
libcfs_nid2str(conn->ibc_peer->ibp_nid),
- rc2, credits, IBLND_MSG_QUEUE_SIZE);
+ rc2, credits, IBLND_MSG_QUEUE_SIZE(conn->ibc_version));
kiblnd_close_conn(conn, -EPROTO);
kiblnd_post_rx(rx, IBLND_POSTRX_NO_CREDIT);
conn->ibc_credits += credits;
- /* This ensures the credit taken by NOOP can be returned */
- if (msg->ibm_type == IBLND_MSG_NOOP)
- conn->ibc_outstanding_credits++;
-
spin_unlock(&conn->ibc_lock);
kiblnd_check_sends(conn);
}
break;
case IBLND_MSG_NOOP:
- if (credits != 0) /* credit already posted */
+ if (IBLND_OOB_CAPABLE(conn->ibc_version))
post_credit = IBLND_POSTRX_NO_CREDIT;
- else /* a keepalive NOOP */
+ else
post_credit = IBLND_POSTRX_PEER_CREDIT;
break;
tx->tx_nwrq = 0; /* overwrite PUT_REQ */
- rc2 = kiblnd_init_rdma(ni, tx, IBLND_MSG_PUT_DONE,
+ rc2 = kiblnd_init_rdma(conn, tx, IBLND_MSG_PUT_DONE,
kiblnd_rd_size(&msg->ibm_u.putack.ibpam_rd),
&msg->ibm_u.putack.ibpam_rd,
msg->ibm_u.putack.ibpam_dst_cookie);
kib_conn_t *conn = rx->rx_conn;
lnet_ni_t *ni = conn->ibc_peer->ibp_ni;
kib_net_t *net = ni->ni_data;
- unsigned long flags;
int rc;
int err = -EIO;
/* racing with connection establishment/teardown! */
if (conn->ibc_state < IBLND_CONN_ESTABLISHED) {
- write_lock_irqsave(&kiblnd_data.kib_global_lock, flags);
+ rwlock_t *g_lock = &kiblnd_data.kib_global_lock;
+ unsigned long flags;
+
+ write_lock_irqsave(g_lock, flags);
/* must check holding global lock to eliminate race */
if (conn->ibc_state < IBLND_CONN_ESTABLISHED) {
list_add_tail(&rx->rx_list, &conn->ibc_early_rxs);
- write_unlock_irqrestore(&kiblnd_data.kib_global_lock,
- flags);
+ write_unlock_irqrestore(g_lock, flags);
return;
}
- write_unlock_irqrestore(&kiblnd_data.kib_global_lock,
- flags);
+ write_unlock_irqrestore(g_lock, flags);
}
kiblnd_handle_rx(rx);
return;
return page;
}
-#if !IBLND_MAP_ON_DEMAND
+static void
+kiblnd_fmr_unmap_tx(kib_net_t *net, kib_tx_t *tx)
+{
+ int rc;
+
+ if (tx->tx_u.fmr == NULL)
+ return;
+
+ rc = ib_fmr_pool_unmap(tx->tx_u.fmr);
+ LASSERT (rc == 0);
+
+ if (tx->tx_status != 0) {
+ rc = ib_flush_fmr_pool(net->ibn_fmrpool);
+ LASSERT (rc == 0);
+ }
+
+ tx->tx_u.fmr = NULL;
+}
+
+static int
+kiblnd_fmr_map_tx(kib_net_t *net, kib_tx_t *tx, kib_rdma_desc_t *rd, int nob)
+{
+ struct ib_pool_fmr *fmr;
+ kib_dev_t *ibdev = net->ibn_dev;
+ __u64 *pages = tx->tx_pages;
+ int npages;
+ int size;
+ int i;
+
+ for (i = 0, npages = 0; i < rd->rd_nfrags; i++) {
+ for (size = 0; size < rd->rd_frags[i].rf_nob;
+ size += ibdev->ibd_page_size) {
+ pages[npages ++] = (rd->rd_frags[i].rf_addr &
+ ibdev->ibd_page_mask) + size;
+ }
+ }
+
+ fmr = ib_fmr_pool_map_phys(net->ibn_fmrpool, pages, npages, 0);
+
+ if (IS_ERR(fmr)) {
+ CERROR ("Can't map %d pages: %ld\n", npages, PTR_ERR(fmr));
+ return PTR_ERR(fmr);
+ }
+
+ /* If rd is not tx_rd, it's going to get sent to a peer, who will need
+ * the rkey */
+ rd->rd_key = (rd != tx->tx_rd) ? fmr->fmr->rkey :
+ fmr->fmr->lkey;
+ rd->rd_frags[0].rf_addr &= ~ibdev->ibd_page_mask;
+ rd->rd_frags[0].rf_nob = nob;
+ rd->rd_nfrags = 1;
+
+ tx->tx_u.fmr = fmr;
+
+ return 0;
+}
+
+static void
+kiblnd_pmr_unmap_tx(kib_net_t *net, kib_tx_t *tx)
+{
+ if (tx->tx_u.pmr == NULL)
+ return;
+
+ kiblnd_phys_mr_unmap(net, tx->tx_u.pmr);
+
+ tx->tx_u.pmr = NULL;
+}
+
+static int
+kiblnd_pmr_map_tx(kib_net_t *net, kib_tx_t *tx, kib_rdma_desc_t *rd, int nob)
+{
+ kib_phys_mr_t *pmr;
+ __u64 iova;
+
+ iova = rd->rd_frags[0].rf_addr & ~net->ibn_dev->ibd_page_mask;
+
+ pmr = kiblnd_phys_mr_map(net, rd, tx->tx_ipb, &iova);
+ if (pmr == NULL) {
+ CERROR("Failed to create MR by phybuf\n");
+ return -ENOMEM;
+ }
+
+ rd->rd_key = (rd != tx->tx_rd) ? pmr->ibpm_mr->rkey :
+ pmr->ibpm_mr->lkey;
+ rd->rd_nfrags = 1;
+ rd->rd_frags[0].rf_addr = iova;
+ rd->rd_frags[0].rf_nob = nob;
+
+ tx->tx_u.pmr = pmr;
+
+ return 0;
+}
+
+void
+kiblnd_unmap_tx(lnet_ni_t *ni, kib_tx_t *tx)
+{
+ kib_net_t *net = ni->ni_data;
+
+ LASSERT (net != NULL);
+
+ if (net->ibn_fmrpool != NULL)
+ kiblnd_fmr_unmap_tx(net, tx);
+ else if (net->ibn_pmrpool != NULL)
+ kiblnd_pmr_unmap_tx(net, tx);
+
+ if (tx->tx_nfrags != 0) {
+ kiblnd_dma_unmap_sg(net->ibn_dev->ibd_cmid->device,
+ tx->tx_frags, tx->tx_nfrags, tx->tx_dmadir);
+ tx->tx_nfrags = 0;
+ }
+}
+
int
-kiblnd_setup_rd_iov(lnet_ni_t *ni, kib_tx_t *tx, kib_rdma_desc_t *rd,
+kiblnd_map_tx(lnet_ni_t *ni, kib_tx_t *tx,
+ kib_rdma_desc_t *rd, int nfrags)
+{
+ kib_net_t *net = ni->ni_data;
+ struct ib_mr *mr = NULL;
+ __u32 nob;
+ int i;
+
+ /* If rd is not tx_rd, it's going to get sent to a peer and I'm the
+ * RDMA sink */
+ tx->tx_dmadir = (rd != tx->tx_rd) ? DMA_FROM_DEVICE : DMA_TO_DEVICE;
+ tx->tx_nfrags = nfrags;
+
+ rd->rd_nfrags =
+ kiblnd_dma_map_sg(net->ibn_dev->ibd_cmid->device,
+ tx->tx_frags, tx->tx_nfrags, tx->tx_dmadir);
+
+ for (i = 0, nob = 0; i < rd->rd_nfrags; i++) {
+ rd->rd_frags[i].rf_nob = kiblnd_sg_dma_len(
+ net->ibn_dev->ibd_cmid->device, &tx->tx_frags[i]);
+ rd->rd_frags[i].rf_addr = kiblnd_sg_dma_address(
+ net->ibn_dev->ibd_cmid->device, &tx->tx_frags[i]);
+ nob += rd->rd_frags[i].rf_nob;
+ }
+
+ /* looking for pre-mapping MR */
+ mr = kiblnd_find_rd_dma_mr(net, rd);
+ if (mr != NULL) {
+ /* found pre-mapping MR */
+ rd->rd_key = (rd != tx->tx_rd) ? mr->rkey : mr->lkey;
+ return 0;
+ }
+
+ if (net->ibn_fmrpool != NULL)
+ return kiblnd_fmr_map_tx(net, tx, rd, nob);
+
+ if (net->ibn_pmrpool != NULL);
+ return kiblnd_pmr_map_tx(net, tx, rd, nob);
+
+ return -EINVAL;
+}
+
+
+int
+kiblnd_setup_rd_iov(lnet_ni_t *ni, kib_tx_t *tx, kib_rdma_desc_t *rd,
unsigned int niov, struct iovec *iov, int offset, int nob)
-
{
+ kib_net_t *net = ni->ni_data;
+ struct page *page;
struct scatterlist *sg;
- int i;
- int fragnob;
unsigned long vaddr;
- struct page *page;
+ int fragnob;
int page_offset;
- kib_net_t *net = ni->ni_data;
LASSERT (nob > 0);
LASSERT (niov > 0);
}
nob -= fragnob;
} while (nob > 0);
-
- /* If rd is not tx_rd, it's going to get sent to a peer and I'm the
- * RDMA sink */
- tx->tx_nfrags = sg - tx->tx_frags;
- tx->tx_dmadir = (rd != tx->tx_rd) ? DMA_FROM_DEVICE : DMA_TO_DEVICE;
-
- rd->rd_nfrags = kiblnd_dma_map_sg(net->ibn_dev->ibd_cmid->device,
- tx->tx_frags, tx->tx_nfrags,
- tx->tx_dmadir);
- rd->rd_key = (rd != tx->tx_rd) ?
- net->ibn_dev->ibd_mr->rkey : net->ibn_dev->ibd_mr->lkey;
- for (i = 0; i < rd->rd_nfrags; i++) {
- rd->rd_frags[i].rf_nob = kiblnd_sg_dma_len(
- net->ibn_dev->ibd_cmid->device, &tx->tx_frags[i]);
- rd->rd_frags[i].rf_addr = kiblnd_sg_dma_address(
- net->ibn_dev->ibd_cmid->device, &tx->tx_frags[i]);
- }
-
- return 0;
+ return kiblnd_map_tx(ni, tx, rd, sg - tx->tx_frags);
}
int
-kiblnd_setup_rd_kiov (lnet_ni_t *ni, kib_tx_t *tx, kib_rdma_desc_t *rd,
+kiblnd_setup_rd_kiov (lnet_ni_t *ni, kib_tx_t *tx, kib_rdma_desc_t *rd,
int nkiov, lnet_kiov_t *kiov, int offset, int nob)
{
+ kib_net_t *net = ni->ni_data;
struct scatterlist *sg;
- int i;
int fragnob;
- kib_net_t *net = ni->ni_data;
CDEBUG(D_NET, "niov %d offset %d nob %d\n", nkiov, offset, nob);
sg_set_page(sg, kiov->kiov_page, fragnob,
kiov->kiov_offset + offset);
sg++;
-
+
offset = 0;
kiov++;
nkiov--;
nob -= fragnob;
} while (nob > 0);
- /* If rd is not tx_rd, it's going to get sent to a peer and I'm the
- * RDMA sink */
- tx->tx_nfrags = sg - tx->tx_frags;
- tx->tx_dmadir = (rd != tx->tx_rd) ? DMA_FROM_DEVICE : DMA_TO_DEVICE;
-
- rd->rd_nfrags = kiblnd_dma_map_sg(net->ibn_dev->ibd_cmid->device,
- tx->tx_frags, tx->tx_nfrags, tx->tx_dmadir);
- rd->rd_key = (rd != tx->tx_rd) ?
- net->ibn_dev->ibd_mr->rkey : net->ibn_dev->ibd_mr->lkey;
-
- for (i = 0; i < tx->tx_nfrags; i++) {
- rd->rd_frags[i].rf_nob = kiblnd_sg_dma_len(
- net->ibn_dev->ibd_cmid->device, &tx->tx_frags[i]);
- rd->rd_frags[i].rf_addr = kiblnd_sg_dma_address(
- net->ibn_dev->ibd_cmid->device, &tx->tx_frags[i]);
-#if 0
- CDEBUG(D_WARNING,"frag[%d]: "LPX64" for %d\n",
- i, rd->rd_frags[i].rf_addr, rd->rd_frags[i].rf_nob);
-#endif
- }
-
- return 0;
+ return kiblnd_map_tx(ni, tx, rd, sg - tx->tx_frags);
}
-#else
+
int
-kiblnd_map_tx (lnet_ni_t *ni, kib_tx_t *tx, kib_rdma_desc_t *rd,
- int npages, unsigned long page_offset, int nob)
+kiblnd_post_tx_locked (kib_conn_t *conn, kib_tx_t *tx, int credit)
{
- struct ib_pool_fmr *fmr;
- kib_net_t *net = ni->ni_data;
+ kib_msg_t *msg = tx->tx_msg;
+ kib_peer_t *peer = conn->ibc_peer;
+ int ver = conn->ibc_version;
+ int rc;
+ int done;
+ struct ib_send_wr *bad_wrq;
- LASSERT (net != NULL);
- LASSERT (tx->tx_fmr == NULL);
- LASSERT (page_offset < PAGE_SIZE);
- LASSERT (npages >= (1 + ((page_offset + nob - 1)>>PAGE_SHIFT)));
- LASSERT (npages <= LNET_MAX_IOV);
+ LASSERT (tx->tx_queued);
+ /* We rely on this for QP sizing */
+ LASSERT (tx->tx_nwrq > 0);
+ LASSERT (tx->tx_nwrq <= 1 + IBLND_RDMA_FRAGS(ver));
- rd->rd_addr = 0;
+ LASSERT (credit == 0 || credit == 1);
+ LASSERT (conn->ibc_outstanding_credits >= 0);
+ LASSERT (conn->ibc_outstanding_credits <= IBLND_MSG_QUEUE_SIZE(ver));
+ LASSERT (conn->ibc_credits >= 0);
+ LASSERT (conn->ibc_credits <= IBLND_MSG_QUEUE_SIZE(ver));
- fmr = ib_fmr_pool_map_phys(net->ibn_fmrpool, tx->tx_pages,
- npages, rd->rd_addr);
- if (IS_ERR(fmr)) {
- CERROR ("Can't map %d pages: %ld\n", npages, PTR_ERR(fmr));
- return PTR_ERR(fmr);
+ if (conn->ibc_nsends_posted == IBLND_CONCURRENT_SENDS(ver)) {
+ /* tx completions outstanding... */
+ CDEBUG(D_NET, "%s: posted enough\n",
+ libcfs_nid2str(peer->ibp_nid));
+ return -EAGAIN;
}
- /* If rd is not tx_rd, it's going to get sent to a peer, who will need
- * the rkey */
-
- rd->rd_key = (rd != tx->tx_rd) ? fmr->fmr->rkey : fmr->fmr->lkey;
- rd->rd_nob = nob;
-
- tx->tx_fmr = fmr;
- return 0;
-}
-
-int
-kiblnd_setup_rd_iov (lnet_ni_t *ni, kib_tx_t *tx, kib_rdma_desc_t *rd,
- unsigned int niov, struct iovec *iov, int offset, int nob)
-
-{
- int resid;
- int fragnob;
- struct page *page;
- int npages;
- unsigned long page_offset;
- unsigned long vaddr;
-
- LASSERT (nob > 0);
- LASSERT (niov > 0);
-
- while (offset >= iov->iov_len) {
- offset -= iov->iov_len;
- niov--;
- iov++;
- LASSERT (niov > 0);
+ if (credit != 0 && conn->ibc_credits == 0) { /* no credits */
+ CDEBUG(D_NET, "%s: no credits\n",
+ libcfs_nid2str(peer->ibp_nid));
+ return -EAGAIN;
}
- if (nob > iov->iov_len - offset) {
- CERROR ("Can't map multiple vaddr fragments\n");
- return (-EMSGSIZE);
+ if (credit != 0 && !IBLND_OOB_CAPABLE(ver) &&
+ conn->ibc_credits == 1 && /* last credit reserved for */
+ conn->ibc_outstanding_credits == 0) { /* giving back credits */
+ CDEBUG(D_NET, "%s: not using last credit\n",
+ libcfs_nid2str(peer->ibp_nid));
+ return -EAGAIN;
}
- vaddr = ((unsigned long)iov->iov_base) + offset;
-
- page_offset = vaddr & (PAGE_SIZE - 1);
- resid = nob;
- npages = 0;
-
- do {
- LASSERT (npages < LNET_MAX_IOV);
-
- page = kiblnd_kvaddr_to_page(vaddr);
- if (page == NULL) {
- CERROR("Can't find page for %lu\n", vaddr);
- return -EFAULT;
- }
-
- tx->tx_pages[npages++] = lnet_page2phys(page);
-
- fragnob = PAGE_SIZE - (vaddr & (PAGE_SIZE - 1));
- vaddr += fragnob;
- resid -= fragnob;
-
- } while (resid > 0);
+ /* NB don't drop ibc_lock before bumping tx_sending */
+ list_del(&tx->tx_list);
+ tx->tx_queued = 0;
+
+ if (msg->ibm_type == IBLND_MSG_NOOP &&
+ (!kiblnd_send_noop(conn) || /* redundant NOOP */
+ (IBLND_OOB_CAPABLE(ver) && /* posted enough NOOP */
+ conn->ibc_noops_posted == IBLND_OOB_MSGS(ver)))) {
+ /* OK to drop when posted enough NOOPs, since
+ * kiblnd_check_sends will queue NOOP again when
+ * posted NOOPs complete */
+ spin_unlock(&conn->ibc_lock);
+ kiblnd_tx_done(peer->ibp_ni, tx);
+ spin_lock(&conn->ibc_lock);
+ CDEBUG(D_NET, "%s(%d): redundant or enough NOOP\n",
+ libcfs_nid2str(peer->ibp_nid),
+ conn->ibc_noops_posted);
+ return 0;
+ }
- return kiblnd_map_tx(ni, tx, rd, npages, page_offset, nob);
-}
+ kiblnd_pack_msg(peer->ibp_ni, msg, ver, conn->ibc_outstanding_credits,
+ peer->ibp_nid, conn->ibc_incarnation);
-int
-kiblnd_setup_rd_kiov (lnet_ni_t *ni, kib_tx_t *tx, kib_rdma_desc_t *rd,
- int nkiov, lnet_kiov_t *kiov, int offset, int nob)
-{
- int resid;
- int npages;
- unsigned long page_offset;
+ conn->ibc_credits -= credit;
+ conn->ibc_outstanding_credits = 0;
+ conn->ibc_nsends_posted++;
+ if (msg->ibm_type == IBLND_MSG_NOOP)
+ conn->ibc_noops_posted++;
- CDEBUG(D_NET, "niov %d offset %d nob %d\n", nkiov, offset, nob);
+ /* CAVEAT EMPTOR! This tx could be the PUT_DONE of an RDMA
+ * PUT. If so, it was first queued here as a PUT_REQ, sent and
+ * stashed on ibc_active_txs, matched by an incoming PUT_ACK,
+ * and then re-queued here. It's (just) possible that
+ * tx_sending is non-zero if we've not done the tx_complete()
+ * from the first send; hence the ++ rather than = below. */
+ tx->tx_sending++;
+ list_add(&tx->tx_list, &conn->ibc_active_txs);
- LASSERT (nob > 0);
- LASSERT (nkiov > 0);
- LASSERT (nkiov <= LNET_MAX_IOV);
+ /* I'm still holding ibc_lock! */
+ if (conn->ibc_state != IBLND_CONN_ESTABLISHED)
+ rc = -ECONNABORTED;
+ else
+ rc = ib_post_send(conn->ibc_cmid->qp,
+ tx->tx_wrq, &bad_wrq);
+ conn->ibc_last_send = jiffies;
- while (offset >= kiov->kiov_len) {
- offset -= kiov->kiov_len;
- nkiov--;
- kiov++;
- LASSERT (nkiov > 0);
- }
+ if (rc == 0)
+ return 0;
- page_offset = kiov->kiov_offset + offset;
+ /* NB credits are transferred in the actual
+ * message, which can only be the last work item */
+ conn->ibc_credits += credit;
+ conn->ibc_outstanding_credits += msg->ibm_credits;
+ conn->ibc_nsends_posted--;
+ if (msg->ibm_type == IBLND_MSG_NOOP)
+ conn->ibc_noops_posted--;
- resid = offset + nob;
- npages = 0;
+ tx->tx_status = rc;
+ tx->tx_waiting = 0;
+ tx->tx_sending--;
- do {
- LASSERT (npages < LNET_MAX_IOV);
- LASSERT (nkiov > 0);
+ done = (tx->tx_sending == 0);
+ if (done)
+ list_del(&tx->tx_list);
- if ((npages > 0 && kiov->kiov_offset != 0) ||
- (resid > kiov->kiov_len &&
- (kiov->kiov_offset + kiov->kiov_len) != PAGE_SIZE)) {
- /* Can't have gaps */
- CERROR ("Can't make payload contiguous in I/O VM:"
- "page %d, offset %d, len %d \n",
- npages, kiov->kiov_offset, kiov->kiov_len);
+ spin_unlock(&conn->ibc_lock);
- return -EINVAL;
- }
+ if (conn->ibc_state == IBLND_CONN_ESTABLISHED)
+ CERROR("Error %d posting transmit to %s\n",
+ rc, libcfs_nid2str(peer->ibp_nid));
+ else
+ CDEBUG(D_NET, "Error %d posting transmit to %s\n",
+ rc, libcfs_nid2str(peer->ibp_nid));
- tx->tx_pages[npages++] = lnet_page2phys(kiov->kiov_page);
- resid -= kiov->kiov_len;
- kiov++;
- nkiov--;
- } while (resid > 0);
+ kiblnd_close_conn(conn, rc);
- return kiblnd_map_tx(ni, tx, rd, npages, page_offset, nob);
+ if (done)
+ kiblnd_tx_done(peer->ibp_ni, tx);
+ return -EIO;
}
-#endif
void
kiblnd_check_sends (kib_conn_t *conn)
{
- kib_tx_t *tx;
- lnet_ni_t *ni = conn->ibc_peer->ibp_ni;
- int rc;
- int consume_cred = 0;
- struct ib_send_wr *bad_wrq;
- int done;
+ int ver = conn->ibc_version;
+ lnet_ni_t *ni = conn->ibc_peer->ibp_ni;
+ kib_tx_t *tx;
/* Don't send anything until after the connection is established */
if (conn->ibc_state < IBLND_CONN_ESTABLISHED) {
spin_lock(&conn->ibc_lock);
- LASSERT (conn->ibc_nsends_posted <=
- *kiblnd_tunables.kib_concurrent_sends);
+ LASSERT (conn->ibc_nsends_posted <= IBLND_CONCURRENT_SENDS(ver));
+ LASSERT (!IBLND_OOB_CAPABLE(ver) ||
+ conn->ibc_noops_posted <= IBLND_OOB_MSGS(ver));
LASSERT (conn->ibc_reserved_credits >= 0);
while (conn->ibc_reserved_credits > 0 &&
kiblnd_init_tx_msg(ni, tx, IBLND_MSG_NOOP, 0);
spin_lock(&conn->ibc_lock);
-
if (tx != NULL)
kiblnd_queue_tx_locked(tx, conn);
}
for (;;) {
+ int credit;
+
if (!list_empty(&conn->ibc_tx_queue_nocred)) {
- tx = list_entry(conn->ibc_tx_queue_nocred.next,
- kib_tx_t, tx_list);
- consume_cred = 0;
- } else if (!list_empty(&conn->ibc_tx_noops)) {
- tx = list_entry(conn->ibc_tx_noops.next,
+ credit = 0;
+ tx = list_entry(conn->ibc_tx_queue_nocred.next,
kib_tx_t, tx_list);
- consume_cred = 1;
} else if (!list_empty(&conn->ibc_tx_queue)) {
+ credit = 1;
tx = list_entry(conn->ibc_tx_queue.next,
kib_tx_t, tx_list);
- consume_cred = 1;
- } else {
- /* nothing to send right now */
+ } else
break;
- }
-
- LASSERT (tx->tx_queued);
- /* We rely on this for QP sizing */
- LASSERT (tx->tx_nwrq > 0 &&
- tx->tx_nwrq <= 1 + IBLND_MAX_RDMA_FRAGS);
-
- LASSERT (conn->ibc_outstanding_credits >= 0);
- LASSERT (conn->ibc_outstanding_credits <= IBLND_MSG_QUEUE_SIZE);
- LASSERT (conn->ibc_credits >= 0);
- LASSERT (conn->ibc_credits <= IBLND_MSG_QUEUE_SIZE);
-
- if (conn->ibc_nsends_posted ==
- *kiblnd_tunables.kib_concurrent_sends) {
- /* tx completions outstanding... */
- CDEBUG(D_NET, "%s: posted enough\n",
- libcfs_nid2str(conn->ibc_peer->ibp_nid));
- break;
- }
-
- if (consume_cred) {
- if (conn->ibc_credits == 0) { /* no credits */
- CDEBUG(D_NET, "%s: no credits\n",
- libcfs_nid2str(conn->ibc_peer->ibp_nid));
- break; /* NB ibc_tx_queue_nocred checked */
- }
- /* Last credit reserved for NOOP */
- if (conn->ibc_credits == 1 &&
- tx->tx_msg->ibm_type != IBLND_MSG_NOOP) {
- CDEBUG(D_NET, "%s: not using last credit\n",
- libcfs_nid2str(conn->ibc_peer->ibp_nid));
- break; /* NB ibc_tx_noops checked */
- }
- }
-
- list_del(&tx->tx_list);
- tx->tx_queued = 0;
-
- /* NB don't drop ibc_lock before bumping tx_sending */
-
- if (tx->tx_msg->ibm_type == IBLND_MSG_NOOP &&
- !kiblnd_send_noop(conn)) {
- /* redundant NOOP */
- spin_unlock(&conn->ibc_lock);
- kiblnd_tx_done(ni, tx);
- spin_lock(&conn->ibc_lock);
- CDEBUG(D_NET, "%s: redundant noop\n",
- libcfs_nid2str(conn->ibc_peer->ibp_nid));
- continue;
- }
-
- kiblnd_pack_msg(ni, tx->tx_msg, conn->ibc_outstanding_credits,
- conn->ibc_peer->ibp_nid, conn->ibc_incarnation);
-
- conn->ibc_outstanding_credits = 0;
- conn->ibc_nsends_posted++;
- if (consume_cred)
- conn->ibc_credits--;
-
- /* CAVEAT EMPTOR! This tx could be the PUT_DONE of an RDMA
- * PUT. If so, it was first queued here as a PUT_REQ, sent and
- * stashed on ibc_active_txs, matched by an incoming PUT_ACK,
- * and then re-queued here. It's (just) possible that
- * tx_sending is non-zero if we've not done the tx_complete() from
- * the first send; hence the ++ rather than = below. */
- tx->tx_sending++;
-
- list_add (&tx->tx_list, &conn->ibc_active_txs);
-#if 0
- {
- int i;
-
- for (i = 0; i < tx->tx_nwrq - 1; i++) {
- LASSERT (tx->tx_wrq[i].opcode == IB_WR_RDMA_WRITE);
- LASSERT (tx->tx_wrq[i].next == &tx->tx_wrq[i+1]);
- LASSERT (tx->tx_wrq[i].sg_list == &tx->tx_sge[i]);
-
- CDEBUG(D_WARNING, "WORK[%d]: RDMA "LPX64
- " for %d k %x -> "LPX64" k %x\n", i,
- tx->tx_wrq[i].sg_list->addr,
- tx->tx_wrq[i].sg_list->length,
- tx->tx_wrq[i].sg_list->lkey,
- tx->tx_wrq[i].wr.rdma.remote_addr,
- tx->tx_wrq[i].wr.rdma.rkey);
- }
-
- LASSERT (tx->tx_wrq[i].opcode == IB_WR_SEND);
- LASSERT (tx->tx_wrq[i].next == NULL);
- LASSERT (tx->tx_wrq[i].sg_list == &tx->tx_sge[i]);
-
- CDEBUG(D_WARNING, "WORK[%d]: SEND "LPX64" for %d k %x\n", i,
- tx->tx_wrq[i].sg_list->addr,
- tx->tx_wrq[i].sg_list->length,
- tx->tx_wrq[i].sg_list->lkey);
- }
-#endif
- /* I'm still holding ibc_lock! */
- if (conn->ibc_state != IBLND_CONN_ESTABLISHED)
- rc = -ECONNABORTED;
- else
- rc = ib_post_send(conn->ibc_cmid->qp, tx->tx_wrq, &bad_wrq);
-
- conn->ibc_last_send = jiffies;
-
- if (rc != 0) {
- /* NB credits are transferred in the actual
- * message, which can only be the last work item */
- conn->ibc_outstanding_credits += tx->tx_msg->ibm_credits;
- if (consume_cred)
- conn->ibc_credits++;
- conn->ibc_nsends_posted--;
-
- tx->tx_status = rc;
- tx->tx_waiting = 0;
- tx->tx_sending--;
-
- done = (tx->tx_sending == 0);
- if (done)
- list_del (&tx->tx_list);
-
- spin_unlock(&conn->ibc_lock);
-
- if (conn->ibc_state == IBLND_CONN_ESTABLISHED)
- CERROR("Error %d posting transmit to %s\n",
- rc, libcfs_nid2str(conn->ibc_peer->ibp_nid));
- else
- CDEBUG(D_NET, "Error %d posting transmit to %s\n",
- rc, libcfs_nid2str(conn->ibc_peer->ibp_nid));
-
- kiblnd_close_conn(conn, rc);
-
- if (done)
- kiblnd_tx_done(ni, tx);
- return;
- }
+ if (kiblnd_post_tx_locked(conn, tx, credit) != 0)
+ break;
}
spin_unlock(&conn->ibc_lock);
tx->tx_sending--;
conn->ibc_nsends_posted--;
+ if (tx->tx_msg->ibm_type == IBLND_MSG_NOOP)
+ conn->ibc_noops_posted--;
if (failed) {
tx->tx_waiting = 0; /* don't wait for peer */
struct ib_sge *sge = &tx->tx_sge[tx->tx_nwrq];
struct ib_send_wr *wrq = &tx->tx_wrq[tx->tx_nwrq];
int nob = offsetof (kib_msg_t, ibm_u) + body_nob;
+ struct ib_mr *mr;
LASSERT (net != NULL);
LASSERT (tx->tx_nwrq >= 0);
kiblnd_init_msg(tx->tx_msg, type, body_nob);
- sge->addr = tx->tx_msgaddr;
- sge->lkey = net->ibn_dev->ibd_mr->lkey;
+ mr = kiblnd_find_dma_mr(net, tx->tx_msgaddr, nob);
+ LASSERT (mr != NULL);
+
+ sge->lkey = mr->lkey;
+ sge->addr = tx->tx_msgaddr;
sge->length = nob;
memset(wrq, 0, sizeof(*wrq));
}
int
-kiblnd_init_rdma (lnet_ni_t *ni, kib_tx_t *tx, int type,
- int nob, kib_rdma_desc_t *dstrd, __u64 dstcookie)
+kiblnd_init_rdma (kib_conn_t *conn, kib_tx_t *tx, int type,
+ int resid, kib_rdma_desc_t *dstrd, __u64 dstcookie)
{
kib_msg_t *ibmsg = tx->tx_msg;
kib_rdma_desc_t *srcrd = tx->tx_rd;
struct ib_sge *sge = &tx->tx_sge[0];
struct ib_send_wr *wrq = &tx->tx_wrq[0];
- int rc = nob;
-
-#if IBLND_MAP_ON_DEMAND
- LASSERT (!in_interrupt());
- LASSERT (tx->tx_nwrq == 0);
- LASSERT (type == IBLND_MSG_GET_DONE ||
- type == IBLND_MSG_PUT_DONE);
-
- sge->addr = srcrd->rd_addr;
- sge->lkey = srcrd->rd_key;
- sge->length = nob;
-
- wrq = &tx->tx_wrq[0];
-
- wrq->next = &tx->tx_wrq[1];
- wrq->wr_id = kiblnd_ptr2wreqid(tx, IBLND_WID_RDMA);
- wrq->sg_list = sge;
- wrq->num_sge = 1;
- wrq->opcode = IB_WR_RDMA_WRITE;
- wrq->send_flags = 0;
-
- wrq->wr.rdma.remote_addr = dstrd->rd_addr;
- wrq->wr.rdma.rkey = dstrd->rd_key;
-
- tx->tx_nwrq = 1;
-#else
- /* CAVEAT EMPTOR: this 'consumes' the frags in 'dstrd' */
- int resid = nob;
- kib_rdma_frag_t *srcfrag;
- int srcidx;
- kib_rdma_frag_t *dstfrag;
- int dstidx;
- int wrknob;
+ int rc = resid;
+ int srcidx;
+ int dstidx;
+ int wrknob;
LASSERT (!in_interrupt());
LASSERT (tx->tx_nwrq == 0);
type == IBLND_MSG_PUT_DONE);
srcidx = dstidx = 0;
- srcfrag = &srcrd->rd_frags[0];
- dstfrag = &dstrd->rd_frags[0];
while (resid > 0) {
if (srcidx >= srcrd->rd_nfrags) {
rc = -EPROTO;
break;
}
-
+
if (dstidx == dstrd->rd_nfrags) {
CERROR("Dst buffer exhausted: %d frags\n", dstidx);
rc = -EPROTO;
break;
}
- if (tx->tx_nwrq == IBLND_MAX_RDMA_FRAGS) {
- CERROR("RDMA too fragmented: %d/%d src %d/%d dst frags\n",
+ if (tx->tx_nwrq == IBLND_RDMA_FRAGS(conn->ibc_version)) {
+ CERROR("RDMA too fragmented for %s (%d): "
+ "%d/%d src %d/%d dst frags\n",
+ libcfs_nid2str(conn->ibc_peer->ibp_nid),
+ IBLND_RDMA_FRAGS(conn->ibc_version),
srcidx, srcrd->rd_nfrags,
dstidx, dstrd->rd_nfrags);
rc = -EMSGSIZE;
break;
}
- wrknob = MIN(MIN(srcfrag->rf_nob, dstfrag->rf_nob), resid);
+ wrknob = MIN(MIN(kiblnd_rd_frag_size(srcrd, srcidx),
+ kiblnd_rd_frag_size(dstrd, dstidx)), resid);
sge = &tx->tx_sge[tx->tx_nwrq];
- sge->addr = srcfrag->rf_addr;
+ sge->addr = kiblnd_rd_frag_addr(srcrd, srcidx);
+ sge->lkey = kiblnd_rd_frag_key(srcrd, srcidx);
sge->length = wrknob;
- sge->lkey = srcrd->rd_key;
wrq = &tx->tx_wrq[tx->tx_nwrq];
wrq->opcode = IB_WR_RDMA_WRITE;
wrq->send_flags = 0;
- wrq->wr.rdma.remote_addr = dstfrag->rf_addr;
- wrq->wr.rdma.rkey = dstrd->rd_key;
+ wrq->wr.rdma.remote_addr = kiblnd_rd_frag_addr(dstrd, dstidx);
+ wrq->wr.rdma.rkey = kiblnd_rd_frag_key(dstrd, dstidx);
- wrq++;
- sge++;
+ srcidx = kiblnd_rd_consume_frag(srcrd, srcidx, wrknob);
+ dstidx = kiblnd_rd_consume_frag(dstrd, dstidx, wrknob);
resid -= wrknob;
- if (wrknob < srcfrag->rf_nob) {
- srcfrag->rf_nob -= wrknob;
- srcfrag->rf_addr += wrknob;
- } else {
- srcfrag++;
- srcidx++;
- }
-
- if (wrknob < dstfrag->rf_nob) {
- dstfrag->rf_nob -= wrknob;
- dstfrag->rf_addr += wrknob;
- } else {
- dstfrag++;
- dstidx++;
- }
tx->tx_nwrq++;
+ wrq++;
+ sge++;
}
if (rc < 0) /* no RDMA if completing with failure */
tx->tx_nwrq = 0;
-#endif
+
ibmsg->ibm_u.completion.ibcm_status = rc;
ibmsg->ibm_u.completion.ibcm_cookie = dstcookie;
- kiblnd_init_tx_msg(ni, tx, type, sizeof (kib_completion_msg_t));
+ kiblnd_init_tx_msg(conn->ibc_peer->ibp_ni, tx,
+ type, sizeof (kib_completion_msg_t));
return rc;
}
LASSERT (tx->tx_nwrq > 0); /* work items set up */
LASSERT (!tx->tx_queued); /* not queued for sending already */
+ LASSERT (conn->ibc_state >= IBLND_CONN_ESTABLISHED);
tx->tx_queued = 1;
tx->tx_deadline = jiffies + (*kiblnd_tunables.kib_timeout * HZ);
break;
case IBLND_MSG_NOOP:
- q = &conn->ibc_tx_noops;
+ if (IBLND_OOB_CAPABLE(conn->ibc_version))
+ q = &conn->ibc_tx_queue_nocred;
+ else
+ q = &conn->ibc_tx_queue;
break;
case IBLND_MSG_IMMEDIATE:
} else {
conn = kiblnd_get_conn_locked(peer);
kiblnd_conn_addref(conn); /* 1 ref for me... */
-
+
write_unlock_irqrestore(g_lock, flags);
if (tx != NULL)
}
ibmsg = tx->tx_msg;
- ibmsg->ibm_u.get.ibgm_hdr = *hdr;
- ibmsg->ibm_u.get.ibgm_cookie = tx->tx_cookie;
if ((lntmsg->msg_md->md_options & LNET_MD_KIOV) == 0)
rc = kiblnd_setup_rd_iov(ni, tx,
kiblnd_tx_done(ni, tx);
return -EIO;
}
-#if IBLND_MAP_ON_DEMAND
- nob = sizeof(kib_get_msg_t);
-#else
+
nob = offsetof(kib_get_msg_t, ibgm_rd.rd_frags[tx->tx_nfrags]);
-#endif
+ ibmsg->ibm_u.get.ibgm_cookie = tx->tx_cookie;
+ ibmsg->ibm_u.get.ibgm_hdr = *hdr;
+
kiblnd_init_tx_msg(ni, tx, IBLND_MSG_GET_REQ, nob);
tx->tx_lntmsg[1] = lnet_create_reply_msg(ni, lntmsg);
goto failed_1;
}
- rc = kiblnd_init_rdma(ni, tx, IBLND_MSG_GET_DONE, nob,
+ rc = kiblnd_init_rdma(rx->rx_conn, tx,
+ IBLND_MSG_GET_DONE, nob,
&rx->rx_msg->ibm_u.get.ibgm_rd,
rx->rx_msg->ibm_u.get.ibgm_cookie);
if (rc < 0) {
&txmsg->ibm_u.putack.ibpam_rd,
niov, iov, offset, mlen);
else
- rc = kiblnd_setup_rd_kiov(ni, tx,
+ rc = kiblnd_setup_rd_kiov(ni, tx,
&txmsg->ibm_u.putack.ibpam_rd,
niov, kiov, offset, mlen);
if (rc != 0) {
break;
}
+ nob = offsetof(kib_putack_msg_t, ibpam_rd.rd_frags[tx->tx_nfrags]);
txmsg->ibm_u.putack.ibpam_src_cookie = rxmsg->ibm_u.putreq.ibprm_cookie;
txmsg->ibm_u.putack.ibpam_dst_cookie = tx->tx_cookie;
-#if IBLND_MAP_ON_DEMAND
- nob = sizeof(kib_putack_msg_t);
-#else
- nob = offsetof(kib_putack_msg_t, ibpam_rd.rd_frags[tx->tx_nfrags]);
-#endif
+
kiblnd_init_tx_msg(ni, tx, IBLND_MSG_PUT_ACK, nob);
tx->tx_lntmsg[0] = lntmsg; /* finalise lntmsg on completion */
return; /* already being handled */
if (error == 0 &&
- list_empty(&conn->ibc_tx_noops) &&
list_empty(&conn->ibc_tx_queue) &&
list_empty(&conn->ibc_tx_queue_rsrvd) &&
list_empty(&conn->ibc_tx_queue_nocred) &&
CDEBUG(D_NET, "closing conn to %s\n",
libcfs_nid2str(peer->ibp_nid));
} else {
- CDEBUG(D_NETERROR, "Closing conn to %s: error %d%s%s%s%s%s\n",
+ CDEBUG(D_NETERROR, "Closing conn to %s: error %d%s%s%s%s\n",
libcfs_nid2str(peer->ibp_nid), error,
list_empty(&conn->ibc_tx_queue) ? "" : "(sending)",
- list_empty(&conn->ibc_tx_noops) ? "" : "(sending_noops)",
list_empty(&conn->ibc_tx_queue_rsrvd) ? "" : "(sending_rsrvd)",
list_empty(&conn->ibc_tx_queue_nocred) ? "" : "(sending_nocred)",
list_empty(&conn->ibc_active_txs) ? "" : "(waiting)");
}
- list_del (&conn->ibc_list);
+ list_del(&conn->ibc_list);
/* connd (see below) takes over ibc_list's ref */
if (list_empty (&peer->ibp_conns) && /* no more conns */
void
kiblnd_abort_txs(kib_conn_t *conn, struct list_head *txs)
{
- LIST_HEAD (zombies);
+ LIST_HEAD (zombies);
struct list_head *tmp;
struct list_head *nxt;
kib_tx_t *tx;
/* Complete all tx descs not waiting for sends to complete.
* NB we should be safe from RDMA now that the QP has changed state */
- kiblnd_abort_txs(conn, &conn->ibc_tx_noops);
kiblnd_abort_txs(conn, &conn->ibc_tx_queue);
kiblnd_abort_txs(conn, &conn->ibc_tx_queue_rsrvd);
kiblnd_abort_txs(conn, &conn->ibc_tx_queue_nocred);
void
kiblnd_connreq_done(kib_conn_t *conn, int status)
{
- struct list_head txs;
-
kib_peer_t *peer = conn->ibc_peer;
- int active;
- unsigned long flags;
kib_tx_t *tx;
+ struct list_head txs;
+ unsigned long flags;
+ int active;
active = (conn->ibc_state == IBLND_CONN_ACTIVE_CONNECT);
- CDEBUG(D_NET,"%s: %d, %d\n", libcfs_nid2str(peer->ibp_nid),
- active, status);
+ CDEBUG(D_NET,"%s: active(%d), version(%x), status(%d)\n",
+ libcfs_nid2str(peer->ibp_nid), active,
+ conn->ibc_version, status);
LASSERT (!in_interrupt());
LASSERT ((conn->ibc_state == IBLND_CONN_ACTIVE_CONNECT &&
else
peer->ibp_accepting--;
- kiblnd_close_stale_conns_locked(peer, conn->ibc_incarnation);
+ if (peer->ibp_version == 0) {
+ peer->ibp_version = conn->ibc_version;
+ peer->ibp_incarnation = conn->ibc_incarnation;
+ }
+
+ if (peer->ibp_version != conn->ibc_version ||
+ peer->ibp_incarnation != conn->ibc_incarnation) {
+ kiblnd_close_stale_conns_locked(peer, conn->ibc_version,
+ conn->ibc_incarnation);
+ peer->ibp_version = conn->ibc_version;
+ peer->ibp_incarnation = conn->ibc_incarnation;
+ }
/* grab pending txs while I have the lock */
list_add(&txs, &peer->ibp_tx_queue);
spin_lock (&conn->ibc_lock);
while (!list_empty (&txs)) {
tx = list_entry (txs.next, kib_tx_t, tx_list);
- list_del (&tx->tx_list);
+ list_del(&tx->tx_list);
kiblnd_queue_tx_locked(tx, conn);
}
}
void
-kiblnd_reject(struct rdma_cm_id *cmid, int why)
+kiblnd_reject(struct rdma_cm_id *cmid, kib_rej_t *rej)
{
int rc;
- kib_rej_t rej = {.ibr_magic = IBLND_MSG_MAGIC,
- .ibr_version = IBLND_MSG_VERSION,
- .ibr_why = why};
- rc = rdma_reject(cmid, &rej, sizeof(rej));
+ rc = rdma_reject(cmid, rej, sizeof(*rej));
if (rc != 0)
CWARN("Error %d sending reject\n", rc);
int
kiblnd_passive_connect (struct rdma_cm_id *cmid, void *priv, int priv_nob)
{
- kib_msg_t *ackmsg;
- kib_msg_t *reqmsg = priv;
rwlock_t *g_lock = &kiblnd_data.kib_global_lock;
- struct rdma_conn_param cp;
- unsigned long flags;
- lnet_ni_t *ni = NULL;
+ kib_msg_t *reqmsg = priv;
+ kib_msg_t *ackmsg;
kib_dev_t *ibdev;
kib_peer_t *peer;
kib_peer_t *peer2;
kib_conn_t *conn;
+ lnet_ni_t *ni = NULL;
+ kib_net_t *net = NULL;
lnet_nid_t nid;
+ struct rdma_conn_param cp;
+ kib_rej_t rej;
+ int version = IBLND_MSG_VERSION;
+ unsigned long flags;
int rc;
- int rej = IBLND_REJECT_FATAL;
LASSERT (!in_interrupt());
ibdev = (kib_dev_t *)cmid->context;
LASSERT (ibdev != NULL);
+ memset(&rej, 0, sizeof(rej));
+ rej.ibr_magic = IBLND_MSG_MAGIC;
+ rej.ibr_why = IBLND_REJECT_FATAL;
+ rej.ibr_cp.ibcp_max_msg_size = IBLND_MSG_SIZE;
+
if (priv_nob < offsetof(kib_msg_t, ibm_type)) {
CERROR("Short connection request\n");
goto failed;
}
+ /* Future protocol version compatibility support! If the
+ * o2iblnd-specific protocol changes, or when LNET unifies
+ * protocols over all LNDs, the initial connection will
+ * negotiate a protocol version. I trap this here to avoid
+ * console errors; the reject tells the peer which protocol I
+ * speak. */
if (reqmsg->ibm_magic == LNET_PROTO_MAGIC ||
- reqmsg->ibm_magic == __swab32(LNET_PROTO_MAGIC) ||
- (reqmsg->ibm_magic == IBLND_MSG_MAGIC &&
- reqmsg->ibm_version != IBLND_MSG_VERSION) ||
- (reqmsg->ibm_magic == __swab32(IBLND_MSG_MAGIC) &&
- reqmsg->ibm_version != __swab16(IBLND_MSG_VERSION))) {
- /* Future protocol version compatibility support! If the
- * o2iblnd-specific protocol changes, or when LNET unifies
- * protocols over all LNDs, the initial connection will
- * negotiate a protocol version. I trap this here to avoid
- * console errors; the reject tells the peer which protocol I
- * speak. */
+ reqmsg->ibm_magic == __swab32(LNET_PROTO_MAGIC))
+ goto failed;
+ if (reqmsg->ibm_magic == IBLND_MSG_MAGIC &&
+ reqmsg->ibm_version != IBLND_MSG_VERSION &&
+ reqmsg->ibm_version != IBLND_MSG_VERSION_1)
+ goto failed;
+ if (reqmsg->ibm_magic == __swab32(IBLND_MSG_MAGIC) &&
+ reqmsg->ibm_version != __swab16(IBLND_MSG_VERSION) &&
+ reqmsg->ibm_version != __swab16(IBLND_MSG_VERSION_1))
goto failed;
- }
rc = kiblnd_unpack_msg(reqmsg, priv_nob);
if (rc != 0) {
}
nid = reqmsg->ibm_srcnid;
+ ni = lnet_net2ni(LNET_NIDNET(reqmsg->ibm_dstnid));
+
+ if (ni != NULL) {
+ net = (kib_net_t *)ni->ni_data;
+ rej.ibr_incarnation = net->ibn_incarnation;
+ }
+
+ if (ni == NULL || /* no matching net */
+ ni->ni_nid != reqmsg->ibm_dstnid || /* right NET, wrong NID! */
+ net->ibn_dev != ibdev) { /* wrong device */
+ CERROR("Can't accept %s: bad dst nid %s\n",
+ libcfs_nid2str(nid),
+ libcfs_nid2str(reqmsg->ibm_dstnid));
+
+ goto failed;
+ }
+
+ /* check time stamp as soon as possible */
+ if (reqmsg->ibm_dststamp != 0 &&
+ reqmsg->ibm_dststamp != net->ibn_incarnation) {
+ CWARN("Stale connection request\n");
+ rej.ibr_why = IBLND_REJECT_CONN_STALE;
+ goto failed;
+ }
+
+ /* I can accept peer's version */
+ version = reqmsg->ibm_version;
if (reqmsg->ibm_type != IBLND_MSG_CONNREQ) {
CERROR("Unexpected connreq msg type: %x from %s\n",
goto failed;
}
- if (reqmsg->ibm_u.connparams.ibcp_queue_depth != IBLND_MSG_QUEUE_SIZE) {
+ if (reqmsg->ibm_u.connparams.ibcp_queue_depth !=
+ IBLND_MSG_QUEUE_SIZE(version)) {
CERROR("Can't accept %s: incompatible queue depth %d (%d wanted)\n",
- libcfs_nid2str(nid),
- reqmsg->ibm_u.connparams.ibcp_queue_depth,
- IBLND_MSG_QUEUE_SIZE);
+ libcfs_nid2str(nid), reqmsg->ibm_u.connparams.ibcp_queue_depth,
+ IBLND_MSG_QUEUE_SIZE(version));
+
+ if (version == IBLND_MSG_VERSION)
+ rej.ibr_why = IBLND_REJECT_MSG_QUEUE_SIZE;
+
goto failed;
}
- if (reqmsg->ibm_u.connparams.ibcp_max_frags != IBLND_MAX_RDMA_FRAGS) {
- CERROR("Can't accept %s: incompatible max_frags %d (%d wanted)\n",
- libcfs_nid2str(nid),
+ if (reqmsg->ibm_u.connparams.ibcp_max_frags !=
+ IBLND_RDMA_FRAGS(version)) {
+ CERROR("Can't accept %s(version %x): "
+ "incompatible max_frags %d (%d wanted)\n",
+ libcfs_nid2str(nid), version,
reqmsg->ibm_u.connparams.ibcp_max_frags,
- IBLND_MAX_RDMA_FRAGS);
+ IBLND_RDMA_FRAGS(version));
+
+ if (version == IBLND_MSG_VERSION)
+ rej.ibr_why = IBLND_REJECT_RDMA_FRAGS;
+
goto failed;
+
}
if (reqmsg->ibm_u.connparams.ibcp_max_msg_size > IBLND_MSG_SIZE) {
goto failed;
}
- ni = lnet_net2ni(LNET_NIDNET(reqmsg->ibm_dstnid));
- if (ni == NULL || /* no matching net */
- ni->ni_nid != reqmsg->ibm_dstnid || /* right NET, wrong NID! */
- ((kib_net_t*)ni->ni_data)->ibn_dev != ibdev) { /* wrong device */
- CERROR("Can't accept %s: bad dst nid %s\n",
- libcfs_nid2str(nid),
- libcfs_nid2str(reqmsg->ibm_dstnid));
-
- goto failed;
- }
-
/* assume 'nid' is a new peer; create */
rc = kiblnd_create_peer(ni, &peer, nid);
if (rc != 0) {
CERROR("Can't create peer for %s\n", libcfs_nid2str(nid));
- rej = IBLND_REJECT_NO_RESOURCES;
+ rej.ibr_why = IBLND_REJECT_NO_RESOURCES;
goto failed;
}
peer2 = kiblnd_find_peer_locked(nid);
if (peer2 != NULL) {
- /* tie-break connection race in favour of the higher NID */
+ if (peer2->ibp_version == 0) {
+ peer2->ibp_version = version;
+ peer2->ibp_incarnation = reqmsg->ibm_srcstamp;
+ }
+
+ /* not the guy I've talked with */
+ if (peer2->ibp_incarnation != reqmsg->ibm_srcstamp ||
+ peer2->ibp_version != version) {
+ kiblnd_close_peer_conns_locked(peer2, -ESTALE);
+ write_unlock_irqrestore(g_lock, flags);
+
+ CWARN("Conn stale %s [old ver: %x, new ver: %x]\n",
+ libcfs_nid2str(nid), peer2->ibp_version, version);
+
+ kiblnd_peer_decref(peer);
+ rej.ibr_why = IBLND_REJECT_CONN_STALE;
+ goto failed;
+ }
+
+ /* tie-break connection race in favour of the higher NID */
if (peer2->ibp_connecting != 0 &&
nid < ni->ni_nid) {
write_unlock_irqrestore(g_lock, flags);
- CWARN("Conn race %s\n",
- libcfs_nid2str(peer2->ibp_nid));
+ CWARN("Conn race %s\n", libcfs_nid2str(peer2->ibp_nid));
kiblnd_peer_decref(peer);
- rej = IBLND_REJECT_CONN_RACE;
+ rej.ibr_why = IBLND_REJECT_CONN_RACE;
goto failed;
}
} else {
/* Brand new peer */
LASSERT (peer->ibp_accepting == 0);
- peer->ibp_accepting = 1;
+ LASSERT (peer->ibp_version == 0 &&
+ peer->ibp_incarnation == 0);
+
+ peer->ibp_accepting = 1;
+ peer->ibp_version = version;
+ peer->ibp_incarnation = reqmsg->ibm_srcstamp;
/* I have a ref on ni that prevents it being shutdown */
- LASSERT (((kib_net_t *)ni->ni_data)->ibn_shutdown == 0);
+ LASSERT (net->ibn_shutdown == 0);
kiblnd_peer_addref(peer);
list_add_tail(&peer->ibp_list, kiblnd_nid2peerlist(nid));
write_unlock_irqrestore(g_lock, flags);
}
- conn = kiblnd_create_conn(peer, cmid, IBLND_CONN_PASSIVE_WAIT);
+ conn = kiblnd_create_conn(peer, cmid, IBLND_CONN_PASSIVE_WAIT, version);
if (conn == NULL) {
kiblnd_peer_connect_failed(peer, 0, -ENOMEM);
kiblnd_peer_decref(peer);
- rej = IBLND_REJECT_NO_RESOURCES;
+ rej.ibr_why = IBLND_REJECT_NO_RESOURCES;
goto failed;
}
* CM callback doesn't destroy cmid. */
conn->ibc_incarnation = reqmsg->ibm_srcstamp;
- conn->ibc_credits = IBLND_MSG_QUEUE_SIZE;
- conn->ibc_reserved_credits = IBLND_MSG_QUEUE_SIZE;
- LASSERT (conn->ibc_credits + conn->ibc_reserved_credits
- <= IBLND_RX_MSGS);
+ conn->ibc_credits = IBLND_MSG_QUEUE_SIZE(version);
+ conn->ibc_reserved_credits = IBLND_MSG_QUEUE_SIZE(version);
+ LASSERT (conn->ibc_credits + conn->ibc_reserved_credits + IBLND_OOB_MSGS(version)
+ <= IBLND_RX_MSGS(version));
ackmsg = &conn->ibc_connvars->cv_msg;
memset(ackmsg, 0, sizeof(*ackmsg));
kiblnd_init_msg(ackmsg, IBLND_MSG_CONNACK,
sizeof(ackmsg->ibm_u.connparams));
- ackmsg->ibm_u.connparams.ibcp_queue_depth = IBLND_MSG_QUEUE_SIZE;
- ackmsg->ibm_u.connparams.ibcp_max_frags = IBLND_MAX_RDMA_FRAGS;
+ ackmsg->ibm_u.connparams.ibcp_queue_depth = IBLND_MSG_QUEUE_SIZE(version);
ackmsg->ibm_u.connparams.ibcp_max_msg_size = IBLND_MSG_SIZE;
- kiblnd_pack_msg(ni, ackmsg, 0, nid, reqmsg->ibm_srcstamp);
+ ackmsg->ibm_u.connparams.ibcp_max_frags = IBLND_RDMA_FRAGS(version);
+
+ kiblnd_pack_msg(ni, ackmsg, version, 0, nid, reqmsg->ibm_srcstamp);
memset(&cp, 0, sizeof(cp));
cp.private_data = ackmsg;
rc = rdma_accept(cmid, &cp);
if (rc != 0) {
CERROR("Can't accept %s: %d\n", libcfs_nid2str(nid), rc);
- kiblnd_reject(cmid, IBLND_REJECT_FATAL);
+ rej.ibr_version = version;
+ rej.ibr_why = IBLND_REJECT_FATAL;
+
+ kiblnd_reject(cmid, &rej);
kiblnd_connreq_done(conn, rc);
kiblnd_conn_decref(conn);
}
if (ni != NULL)
lnet_ni_decref(ni);
- kiblnd_reject(cmid, rej);
+ rej.ibr_version = version;
+ rej.ibr_cp.ibcp_queue_depth = IBLND_MSG_QUEUE_SIZE(version);
+ rej.ibr_cp.ibcp_max_frags = IBLND_RDMA_FRAGS(version);
+ kiblnd_reject(cmid, &rej);
+
return -ECONNREFUSED;
}
void
-kiblnd_reconnect (kib_conn_t *conn, char *why)
+kiblnd_reconnect (kib_conn_t *conn, int version,
+ __u64 incarnation, int why, kib_connparams_t *cp)
{
kib_peer_t *peer = conn->ibc_peer;
+ char *reason;
int retry = 0;
unsigned long flags;
-
+
LASSERT (conn->ibc_state == IBLND_CONN_ACTIVE_CONNECT);
LASSERT (peer->ibp_connecting > 0); /* 'conn' at least */
peer->ibp_accepting == 0) {
retry = 1;
peer->ibp_connecting++;
+
+ peer->ibp_version = version;
+ peer->ibp_incarnation = incarnation;
}
-
+
write_unlock_irqrestore(&kiblnd_data.kib_global_lock, flags);
- if (retry) {
- CDEBUG(D_NETERROR, "%s: retrying (%s)\n",
- libcfs_nid2str(peer->ibp_nid), why);
- kiblnd_connect_peer(peer);
+ if (!retry)
+ return;
+
+ switch (why) {
+ default:
+ reason = "Unknown";
+ break;
+
+ case IBLND_REJECT_CONN_STALE:
+ reason = "stale";
+ break;
+
+ case IBLND_REJECT_CONN_RACE:
+ reason = "conn race";
+ break;
+
+ case IBLND_REJECT_CONN_UNCOMPAT:
+ reason = "version negotiation";
+ break;
}
+
+ CDEBUG(D_NETERROR, "%s: retrying (%s), %x, %x, "
+ "queue_dep: %d, max_frag: %d, msg_size: %d\n",
+ libcfs_nid2str(peer->ibp_nid),
+ reason, IBLND_MSG_VERSION, version,
+ cp != NULL ? cp->ibcp_queue_depth : IBLND_MSG_QUEUE_SIZE(version),
+ cp != NULL ? cp->ibcp_max_frags : IBLND_RDMA_FRAGS(version),
+ cp != NULL ? cp->ibcp_max_msg_size: IBLND_MSG_SIZE);
+
+ kiblnd_connect_peer(peer);
}
void
switch (reason) {
case IB_CM_REJ_STALE_CONN:
- kiblnd_reconnect(conn, "stale");
+ kiblnd_reconnect(conn, IBLND_MSG_VERSION, 0,
+ IBLND_REJECT_CONN_STALE, NULL);
break;
case IB_CM_REJ_CONSUMER_DEFINED:
- if (priv_nob >= sizeof(kib_rej_t)) {
- kib_rej_t *rej = priv;
+ if (priv_nob >= offsetof(kib_rej_t, ibr_padding)) {
+ kib_rej_t *rej = priv;
+ kib_connparams_t *cp = NULL;
+ int flip = 0;
+ __u64 incarnation = -1;
+
+ /* NB. default incarnation is -1 because:
+ * a) V1 will ignore dst incarnation in connreq.
+ * b) V2 will provide incarnation while rejecting me,
+ * -1 will be overwrote.
+ *
+ * if I try to connect to a V1 peer with V2 protocol,
+ * it rejected me then upgrade to V2, I have no idea
+ * about the upgrading and try to reconnect with V1,
+ * in this case upgraded V2 can find out I'm trying to
+ * talk to the old guy and reject me(incarnation is -1).
+ */
if (rej->ibr_magic == __swab32(IBLND_MSG_MAGIC) ||
rej->ibr_magic == __swab32(LNET_PROTO_MAGIC)) {
__swab32s(&rej->ibr_magic);
__swab16s(&rej->ibr_version);
+ flip = 1;
+ }
+
+ if (priv_nob >= sizeof(kib_rej_t) &&
+ rej->ibr_version > IBLND_MSG_VERSION_1) {
+ /* priv_nob is always 148 in current version
+ * of OFED, so we still need to check version.
+ * (define of IB_CM_REJ_PRIVATE_DATA_SIZE) */
+ cp = &rej->ibr_cp;
+
+ if (flip) {
+ __swab64s(&rej->ibr_incarnation);
+ __swab16s(&cp->ibcp_queue_depth);
+ __swab16s(&cp->ibcp_max_frags);
+ __swab32s(&cp->ibcp_max_msg_size);
+ }
+
+ incarnation = rej->ibr_incarnation;
}
if (rej->ibr_magic != IBLND_MSG_MAGIC &&
libcfs_nid2str(peer->ibp_nid));
break;
}
-
- if (rej->ibr_version != IBLND_MSG_VERSION) {
- CERROR("%s rejected: o2iblnd version %d error\n",
+
+ if (rej->ibr_version != IBLND_MSG_VERSION &&
+ rej->ibr_version != IBLND_MSG_VERSION_1) {
+ CERROR("%s rejected: o2iblnd version %x error\n",
libcfs_nid2str(peer->ibp_nid),
rej->ibr_version);
break;
}
-
+
+ if (rej->ibr_why == IBLND_REJECT_FATAL &&
+ rej->ibr_version == IBLND_MSG_VERSION_1) {
+ CDEBUG(D_NET, "rejected by old version peer %s: %x\n",
+ libcfs_nid2str(peer->ibp_nid), rej->ibr_version);
+
+ if (conn->ibc_version != IBLND_MSG_VERSION_1)
+ rej->ibr_why = IBLND_REJECT_CONN_UNCOMPAT;
+ }
+
switch (rej->ibr_why) {
case IBLND_REJECT_CONN_RACE:
- kiblnd_reconnect(conn, "conn race");
+ case IBLND_REJECT_CONN_STALE:
+ case IBLND_REJECT_CONN_UNCOMPAT:
+ kiblnd_reconnect(conn, rej->ibr_version,
+ incarnation, rej->ibr_why, cp);
+ break;
+
+ case IBLND_REJECT_MSG_QUEUE_SIZE:
+ CERROR("%s rejected: incompatible message queue depth %d, %d\n",
+ libcfs_nid2str(peer->ibp_nid), cp->ibcp_queue_depth,
+ IBLND_MSG_QUEUE_SIZE(conn->ibc_version));
+ break;
+
+ case IBLND_REJECT_RDMA_FRAGS:
+ CERROR("%s rejected: incompatible # of RDMA fragments %d, %d\n",
+ libcfs_nid2str(peer->ibp_nid), cp->ibcp_max_frags,
+ IBLND_RDMA_FRAGS(conn->ibc_version));
break;
-
+
case IBLND_REJECT_NO_RESOURCES:
CERROR("%s rejected: o2iblnd no resources\n",
libcfs_nid2str(peer->ibp_nid));
break;
+
case IBLND_REJECT_FATAL:
CERROR("%s rejected: o2iblnd fatal error\n",
libcfs_nid2str(peer->ibp_nid));
break;
+
default:
CERROR("%s rejected: o2iblnd reason %d\n",
libcfs_nid2str(peer->ibp_nid),
kiblnd_check_connreply (kib_conn_t *conn, void *priv, int priv_nob)
{
kib_peer_t *peer = conn->ibc_peer;
- lnet_ni_t *ni = peer->ibp_ni;
- kib_net_t *net = ni->ni_data;
- kib_msg_t *msg = priv;
- int rc = kiblnd_unpack_msg(msg, priv_nob);
+ lnet_ni_t *ni = peer->ibp_ni;
+ kib_net_t *net = ni->ni_data;
+ kib_msg_t *msg = priv;
+ int ver = conn->ibc_version;
+ int rc = kiblnd_unpack_msg(msg, priv_nob);
unsigned long flags;
LASSERT (net != NULL);
goto failed;
}
- if (msg->ibm_u.connparams.ibcp_queue_depth != IBLND_MSG_QUEUE_SIZE) {
+ if (ver != msg->ibm_version) {
+ CERROR("%s replied version %x is different with "
+ "requested version %x\n",
+ libcfs_nid2str(peer->ibp_nid), msg->ibm_version, ver);
+ rc = -EPROTO;
+ goto failed;
+ }
+
+ if (msg->ibm_u.connparams.ibcp_queue_depth !=
+ IBLND_MSG_QUEUE_SIZE(ver)) {
CERROR("%s has incompatible queue depth %d(%d wanted)\n",
libcfs_nid2str(peer->ibp_nid),
msg->ibm_u.connparams.ibcp_queue_depth,
- IBLND_MSG_QUEUE_SIZE);
+ IBLND_MSG_QUEUE_SIZE(ver));
rc = -EPROTO;
goto failed;
}
- if (msg->ibm_u.connparams.ibcp_max_frags != IBLND_MAX_RDMA_FRAGS) {
+ if (msg->ibm_u.connparams.ibcp_max_frags !=
+ IBLND_RDMA_FRAGS(ver)) {
CERROR("%s has incompatible max_frags %d (%d wanted)\n",
libcfs_nid2str(peer->ibp_nid),
msg->ibm_u.connparams.ibcp_max_frags,
- IBLND_MAX_RDMA_FRAGS);
+ IBLND_RDMA_FRAGS(ver));
rc = -EPROTO;
goto failed;
}
read_unlock_irqrestore(&kiblnd_data.kib_global_lock, flags);
if (rc != 0) {
- CERROR("Stale connection reply from %s\n",
- libcfs_nid2str(peer->ibp_nid));
+ CERROR("Bad connection reply from %s, rc = %d, "
+ "version: %x max_frags: %d\n",
+ libcfs_nid2str(peer->ibp_nid), rc,
+ msg->ibm_version, msg->ibm_u.connparams.ibcp_max_frags);
goto failed;
}
conn->ibc_incarnation = msg->ibm_srcstamp;
- conn->ibc_credits = IBLND_MSG_QUEUE_SIZE;
- conn->ibc_reserved_credits = IBLND_MSG_QUEUE_SIZE;
- LASSERT (conn->ibc_credits + conn->ibc_reserved_credits
- <= IBLND_RX_MSGS);
+ conn->ibc_credits =
+ conn->ibc_reserved_credits = IBLND_MSG_QUEUE_SIZE(ver);
+ LASSERT (conn->ibc_credits + conn->ibc_reserved_credits + IBLND_OOB_MSGS(ver)
+ <= IBLND_RX_MSGS(ver));
kiblnd_connreq_done(conn, 0);
return;
kib_conn_t *conn;
kib_msg_t *msg;
struct rdma_conn_param cp;
+ int version;
+ __u64 incarnation;
+ long flags;
int rc;
- conn = kiblnd_create_conn(peer, cmid, IBLND_CONN_ACTIVE_CONNECT);
+ read_lock_irqsave(&kiblnd_data.kib_global_lock, flags);
+
+ incarnation = peer->ibp_incarnation;
+ version = (peer->ibp_version == 0) ? IBLND_MSG_VERSION : peer->ibp_version;
+
+ read_unlock_irqrestore(&kiblnd_data.kib_global_lock, flags);
+
+ conn = kiblnd_create_conn(peer, cmid, IBLND_CONN_ACTIVE_CONNECT, version);
if (conn == NULL) {
kiblnd_peer_connect_failed(peer, 1, -ENOMEM);
kiblnd_peer_decref(peer); /* lose cmid's ref */
memset(msg, 0, sizeof(*msg));
kiblnd_init_msg(msg, IBLND_MSG_CONNREQ, sizeof(msg->ibm_u.connparams));
- msg->ibm_u.connparams.ibcp_queue_depth = IBLND_MSG_QUEUE_SIZE;
- msg->ibm_u.connparams.ibcp_max_frags = IBLND_MAX_RDMA_FRAGS;
+ msg->ibm_u.connparams.ibcp_queue_depth = IBLND_MSG_QUEUE_SIZE(version);
+ msg->ibm_u.connparams.ibcp_max_frags = IBLND_RDMA_FRAGS(version);
msg->ibm_u.connparams.ibcp_max_msg_size = IBLND_MSG_SIZE;
- kiblnd_pack_msg(peer->ibp_ni, msg, 0, peer->ibp_nid, 0);
-
+
+ kiblnd_pack_msg(peer->ibp_ni, msg, version,
+ 0, peer->ibp_nid, incarnation);
+
memset(&cp, 0, sizeof(cp));
cp.private_data = msg;
cp.private_data_len = msg->ibm_nob;
LASSERT(cmid->context == (void *)conn);
LASSERT(conn->ibc_cmid == cmid);
-
+
rc = rdma_connect(cmid, &cp);
if (rc != 0) {
CERROR("Can't connect to %s: %d\n",
if (time_after_eq (jiffies, tx->tx_deadline)) {
timed_out = 1;
+ CERROR("Timed out tx: %s, %lu seconds\n",
+ kiblnd_queue2str(conn, txs),
+ cfs_duration_sec(jiffies - tx->tx_deadline));
break;
}
}
kiblnd_conn_timed_out (kib_conn_t *conn)
{
return kiblnd_check_txs(conn, &conn->ibc_tx_queue) ||
- kiblnd_check_txs(conn, &conn->ibc_tx_noops) ||
kiblnd_check_txs(conn, &conn->ibc_tx_queue_rsrvd) ||
kiblnd_check_txs(conn, &conn->ibc_tx_queue_nocred) ||
kiblnd_check_txs(conn, &conn->ibc_active_txs);
read_unlock_irqrestore(&kiblnd_data.kib_global_lock,
flags);
- CERROR("Timed out RDMA with %s\n",
- libcfs_nid2str(peer->ibp_nid));
+ CERROR("Timed out RDMA with %s (%lu)\n",
+ libcfs_nid2str(peer->ibp_nid),
+ cfs_duration_sec(cfs_time_current() -
+ peer->ibp_last_alive));
kiblnd_close_conn(conn, -ETIMEDOUT);
kiblnd_conn_decref(conn); /* ...until here */
if (!list_empty (&kiblnd_data.kib_connd_zombies)) {
conn = list_entry (kiblnd_data.kib_connd_zombies.next,
kib_conn_t, ibc_list);
- list_del (&conn->ibc_list);
+ list_del(&conn->ibc_list);
spin_unlock_irqrestore (&kiblnd_data.kib_connd_lock, flags);
dropped_lock = 1;
if (!list_empty (&kiblnd_data.kib_connd_conns)) {
conn = list_entry (kiblnd_data.kib_connd_conns.next,
kib_conn_t, ibc_list);
- list_del (&conn->ibc_list);
+ list_del(&conn->ibc_list);
spin_unlock_irqrestore (&kiblnd_data.kib_connd_lock, flags);
dropped_lock = 1;
LASSERT(conn->ibc_scheduled);
list_del(&conn->ibc_sched_list);
conn->ibc_ready = 0;
-
+
spin_unlock_irqrestore(&kiblnd_data.kib_sched_lock,
flags);
CFS_MODULE_PARM(peer_credits, "i", int, 0444,
"# concurrent sends to 1 peer");
+static int peer_credits_hiw = 0;
+CFS_MODULE_PARM(peer_credits_hiw, "i", int, 0444,
+ "when eagerly to return credits");
+
static int peer_timeout = 0;
CFS_MODULE_PARM(peer_timeout, "i", int, 0444,
"Seconds without aliveness news to declare peer dead (<=0 to disable)");
CFS_MODULE_PARM(ib_mtu, "i", int, 0444,
"IB MTU 256/512/1024/2048/4096");
-#if IBLND_MAP_ON_DEMAND
-static int concurrent_sends = IBLND_RX_MSGS;
-#else
-static int concurrent_sends = IBLND_MSG_QUEUE_SIZE;
-#endif
+static int concurrent_sends = 0;
CFS_MODULE_PARM(concurrent_sends, "i", int, 0444,
"send work-queue sizing");
-#if IBLND_MAP_ON_DEMAND
+static int map_on_demand = 0;
+CFS_MODULE_PARM(map_on_demand, "i", int, 0444,
+ "map on demand");
+
static int fmr_pool_size = 512;
CFS_MODULE_PARM(fmr_pool_size, "i", int, 0444,
"size of the fmr pool (>= ntx)");
static int fmr_cache = 1;
CFS_MODULE_PARM(fmr_cache, "i", int, 0444,
"non-zero to enable FMR caching");
-#endif
+
+static int pmr_pool_size = 512;
+CFS_MODULE_PARM(pmr_pool_size, "i", int, 0444,
+ "size of the MR cache pmr pool");
kib_tunables_t kiblnd_tunables = {
.kib_service = &service,
.kib_ntx = &ntx,
.kib_credits = &credits,
.kib_peercredits = &peer_credits,
+ .kib_peercredits_hiw = &peer_credits_hiw,
.kib_peertimeout = &peer_timeout,
.kib_default_ipif = &ipif_name,
.kib_retry_count = &retry_count,
.kib_rnr_retry_count = &rnr_retry_count,
.kib_concurrent_sends = &concurrent_sends,
.kib_ib_mtu = &ib_mtu,
-#if IBLND_MAP_ON_DEMAND
+ .kib_map_on_demand = &map_on_demand,
.kib_fmr_pool_size = &fmr_pool_size,
.kib_fmr_flush_trigger = &fmr_flush_trigger,
.kib_fmr_cache = &fmr_cache,
-#endif
+ .kib_pmr_pool_size = &pmr_pool_size,
};
#if defined(CONFIG_SYSCTL) && !CFS_SYSFS_MODULE_PARM
O2IBLND_NTX,
O2IBLND_CREDITS,
O2IBLND_PEER_CREDITS,
+ O2IBLND_PEER_CREDITS_HIW,
O2IBLND_PEER_TIMEOUT,
O2IBLND_IPIF_BASENAME,
O2IBLND_RETRY_COUNT,
O2IBLND_KEEPALIVE,
O2IBLND_CONCURRENT_SENDS,
O2IBLND_IB_MTU,
+ O2IBLND_MAP_ON_DEMAND,
O2IBLND_FMR_POOL_SIZE,
O2IBLND_FMR_FLUSH_TRIGGER,
- O2IBLND_FMR_CACHE
+ O2IBLND_FMR_CACHE,
+ O2IBLND_PMR_POOL_SIZE
};
#else
#define O2IBLND_NTX CTL_UNNUMBERED
#define O2IBLND_CREDITS CTL_UNNUMBERED
#define O2IBLND_PEER_CREDITS CTL_UNNUMBERED
+#define O2IBLND_PEER_CREDITS_HIW CTL_UNNUMBERED
#define O2IBLND_PEER_TIMEOUT CTL_UNNUMBERED
#define O2IBLND_IPIF_BASENAME CTL_UNNUMBERED
#define O2IBLND_RETRY_COUNT CTL_UNNUMBERED
#define O2IBLND_KEEPALIVE CTL_UNNUMBERED
#define O2IBLND_CONCURRENT_SENDS CTL_UNNUMBERED
#define O2IBLND_IB_MTU CTL_UNNUMBERED
+#define O2IBLND_MAP_ON_DEMAND CTL_UNNUMBERED
#define O2IBLND_FMR_POOL_SIZE CTL_UNNUMBERED
#define O2IBLND_FMR_FLUSH_TRIGGER CTL_UNNUMBERED
#define O2IBLND_FMR_CACHE CTL_UNNUMBERED
+#define O2IBLND_PMR_POOL_SIZE CTL_UNNUMBERED
#endif
.proc_handler = &proc_dointvec
},
{
+ .ctl_name = O2IBLND_PEER_CREDITS_HIW,
+ .procname = "peer_credits_hiw",
+ .data = &peer_credits_hiw,
+ .maxlen = sizeof(int),
+ .mode = 0444,
+ .proc_handler = &proc_dointvec
+ },
+ {
.ctl_name = O2IBLND_PEER_TIMEOUT,
.procname = "peer_timeout",
.data = &peer_timeout,
.mode = 0444,
.proc_handler = &proc_dointvec
},
-#if IBLND_MAP_ON_DEMAND
+ {
+ .ctl_name = O2IBLND_MAP_ON_DEMAND,
+ .procname = "map_on_demand",
+ .data = &map_on_demand,
+ .maxlen = sizeof(int),
+ .mode = 0444,
+ .proc_handler = &proc_dointvec
+ },
+
{
.ctl_name = O2IBLND_FMR_POOL_SIZE,
.procname = "fmr_pool_size",
.mode = 0444,
.proc_handler = &proc_dointvec
},
-#endif
+ {
+ .ctl_name = O2IBLND_PMR_POOL_SIZE,
+ .procname = "pmr_pool_size",
+ .data = &pmr_pool_size,
+ .maxlen = sizeof(int),
+ .mode = 0444,
+ .proc_handler = &proc_dointvec
+ },
{0}
};
return -EINVAL;
}
- if (*kiblnd_tunables.kib_ib_mtu != 0 &&
- *kiblnd_tunables.kib_ib_mtu != 256 &&
- *kiblnd_tunables.kib_ib_mtu != 512 &&
- *kiblnd_tunables.kib_ib_mtu != 1024 &&
- *kiblnd_tunables.kib_ib_mtu != 2048 &&
- *kiblnd_tunables.kib_ib_mtu != 4096) {
+ if (kiblnd_translate_mtu(*kiblnd_tunables.kib_ib_mtu) < 0) {
CERROR("Invalid ib_mtu %d, expected 256/512/1024/2048/4096\n",
*kiblnd_tunables.kib_ib_mtu);
return -EINVAL;
}
- if (*kiblnd_tunables.kib_concurrent_sends > IBLND_RX_MSGS)
- *kiblnd_tunables.kib_concurrent_sends = IBLND_RX_MSGS;
- if (*kiblnd_tunables.kib_concurrent_sends < IBLND_MSG_QUEUE_SIZE / 2)
- *kiblnd_tunables.kib_concurrent_sends = IBLND_MSG_QUEUE_SIZE / 2;
+ if (*kiblnd_tunables.kib_peercredits < IBLND_CREDITS_DEFAULT)
+ *kiblnd_tunables.kib_peercredits = IBLND_CREDITS_DEFAULT;
+
+ if (*kiblnd_tunables.kib_peercredits > IBLND_CREDITS_MAX)
+ *kiblnd_tunables.kib_peercredits = IBLND_CREDITS_MAX;
+
+ if (*kiblnd_tunables.kib_peercredits_hiw < *kiblnd_tunables.kib_peercredits / 2)
+ *kiblnd_tunables.kib_peercredits_hiw = *kiblnd_tunables.kib_peercredits / 2;
+
+ if (*kiblnd_tunables.kib_peercredits_hiw >= *kiblnd_tunables.kib_peercredits)
+ *kiblnd_tunables.kib_peercredits_hiw = *kiblnd_tunables.kib_peercredits - 1;
+
+ if (*kiblnd_tunables.kib_map_on_demand < 0 ||
+ *kiblnd_tunables.kib_map_on_demand >= IBLND_MAX_RDMA_FRAGS)
+ *kiblnd_tunables.kib_map_on_demand = 0; /* disable map-on-demand */
+
+ if (*kiblnd_tunables.kib_concurrent_sends == 0) {
+ if (*kiblnd_tunables.kib_map_on_demand > 0 &&
+ *kiblnd_tunables.kib_map_on_demand <= IBLND_MAX_RDMA_FRAGS / 8)
+ *kiblnd_tunables.kib_concurrent_sends = (*kiblnd_tunables.kib_peercredits) * 2;
+ else
+ *kiblnd_tunables.kib_concurrent_sends = (*kiblnd_tunables.kib_peercredits);
+ }
+
+ if (*kiblnd_tunables.kib_concurrent_sends > *kiblnd_tunables.kib_peercredits * 2)
+ *kiblnd_tunables.kib_concurrent_sends = *kiblnd_tunables.kib_peercredits * 2;
+
+ if (*kiblnd_tunables.kib_concurrent_sends < *kiblnd_tunables.kib_peercredits / 2)
+ *kiblnd_tunables.kib_concurrent_sends = *kiblnd_tunables.kib_peercredits / 2;
- if (*kiblnd_tunables.kib_concurrent_sends < IBLND_MSG_QUEUE_SIZE) {
+ if (*kiblnd_tunables.kib_concurrent_sends < *kiblnd_tunables.kib_peercredits) {
CWARN("Concurrent sends %d is lower than message queue size: %d, "
"performance may drop slightly.\n",
- *kiblnd_tunables.kib_concurrent_sends, IBLND_MSG_QUEUE_SIZE);
+ *kiblnd_tunables.kib_concurrent_sends, *kiblnd_tunables.kib_peercredits);
}
kiblnd_sysctl_init();
git://git.whamcloud.com / fs / lustre-release.git / commitdiff