1 /* -*- mode: c; c-basic-offset: 8; indent-tabs-mode: nil; -*-
2 * vim:expandtab:shiftwidth=8:tabstop=8:
4 * Copyright (C) 2006 Cluster File Systems, Inc.
5 * Author: Eric Barton <eric@bartonsoftware.com>
7 * This file is part of Lustre, http://www.lustre.org.
9 * Lustre is free software; you can redistribute it and/or
10 * modify it under the terms of version 2 of the GNU General Public
11 * License as published by the Free Software Foundation.
13 * Lustre is distributed in the hope that it will be useful,
14 * but WITHOUT ANY WARRANTY; without even the implied warranty of
15 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
16 * GNU General Public License for more details.
18 * You should have received a copy of the GNU General Public License
19 * along with Lustre; if not, write to the Free Software
20 * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
28 .lnd_startup = kiblnd_startup,
29 .lnd_shutdown = kiblnd_shutdown,
30 .lnd_ctl = kiblnd_ctl,
31 .lnd_send = kiblnd_send,
32 .lnd_recv = kiblnd_recv,
35 kib_data_t kiblnd_data;
38 kiblnd_cksum (void *ptr, int nob)
44 sum = ((sum << 1) | (sum >> 31)) + *c++;
46 /* ensure I don't return 0 (== no checksum) */
47 return (sum == 0) ? 1 : sum;
51 kiblnd_init_msg (kib_msg_t *msg, int type, int body_nob)
54 msg->ibm_nob = offsetof(kib_msg_t, ibm_u) + body_nob;
58 kiblnd_pack_msg (lnet_ni_t *ni, kib_msg_t *msg,
59 int credits, lnet_nid_t dstnid, __u64 dststamp)
61 kib_net_t *net = ni->ni_data;
63 /* CAVEAT EMPTOR! all message fields not set here should have been
64 * initialised previously. */
65 msg->ibm_magic = IBLND_MSG_MAGIC;
66 msg->ibm_version = IBLND_MSG_VERSION;
68 msg->ibm_credits = credits;
71 msg->ibm_srcnid = lnet_ptlcompat_srcnid(ni->ni_nid, dstnid);
72 msg->ibm_srcstamp = net->ibn_incarnation;
73 msg->ibm_dstnid = dstnid;
74 msg->ibm_dststamp = dststamp;
76 if (*kiblnd_tunables.kib_cksum) {
77 /* NB ibm_cksum zero while computing cksum */
78 msg->ibm_cksum = kiblnd_cksum(msg, msg->ibm_nob);
83 kiblnd_unpack_msg(kib_msg_t *msg, int nob)
85 const int hdr_size = offsetof(kib_msg_t, ibm_u);
89 #if !IBLND_MAP_ON_DEMAND
93 /* 6 bytes are enough to have received magic + version */
95 CERROR("Short message: %d\n", nob);
99 if (msg->ibm_magic == IBLND_MSG_MAGIC) {
101 } else if (msg->ibm_magic == __swab32(IBLND_MSG_MAGIC)) {
104 CERROR("Bad magic: %08x\n", msg->ibm_magic);
108 if (msg->ibm_version !=
109 (flip ? __swab16(IBLND_MSG_VERSION) : IBLND_MSG_VERSION)) {
110 CERROR("Bad version: %d\n", msg->ibm_version);
114 if (nob < hdr_size) {
115 CERROR("Short message: %d\n", nob);
119 msg_nob = flip ? __swab32(msg->ibm_nob) : msg->ibm_nob;
121 CERROR("Short message: got %d, wanted %d\n", nob, msg_nob);
125 /* checksum must be computed with ibm_cksum zero and BEFORE anything
127 msg_cksum = flip ? __swab32(msg->ibm_cksum) : msg->ibm_cksum;
129 if (msg_cksum != 0 &&
130 msg_cksum != kiblnd_cksum(msg, msg_nob)) {
131 CERROR("Bad checksum\n");
134 msg->ibm_cksum = msg_cksum;
137 /* leave magic unflipped as a clue to peer endianness */
138 __swab16s(&msg->ibm_version);
139 CLASSERT (sizeof(msg->ibm_type) == 1);
140 CLASSERT (sizeof(msg->ibm_credits) == 1);
141 msg->ibm_nob = msg_nob;
142 __swab64s(&msg->ibm_srcnid);
143 __swab64s(&msg->ibm_srcstamp);
144 __swab64s(&msg->ibm_dstnid);
145 __swab64s(&msg->ibm_dststamp);
148 if (msg->ibm_srcnid == LNET_NID_ANY) {
149 CERROR("Bad src nid: %s\n", libcfs_nid2str(msg->ibm_srcnid));
153 switch (msg->ibm_type) {
155 CERROR("Unknown message type %x\n", msg->ibm_type);
161 case IBLND_MSG_IMMEDIATE:
162 if (msg_nob < offsetof(kib_msg_t, ibm_u.immediate.ibim_payload[0])) {
163 CERROR("Short IMMEDIATE: %d(%d)\n", msg_nob,
164 (int)offsetof(kib_msg_t, ibm_u.immediate.ibim_payload[0]));
169 case IBLND_MSG_PUT_REQ:
170 if (msg_nob < hdr_size + sizeof(msg->ibm_u.putreq)) {
171 CERROR("Short PUT_REQ: %d(%d)\n", msg_nob,
172 (int)(hdr_size + sizeof(msg->ibm_u.putreq)));
177 case IBLND_MSG_PUT_ACK:
178 if (msg_nob < hdr_size + sizeof(msg->ibm_u.putack)) {
179 CERROR("Short PUT_ACK: %d(%d)\n", msg_nob,
180 (int)(hdr_size + sizeof(msg->ibm_u.putack)));
183 #if IBLND_MAP_ON_DEMAND
185 __swab64s(&msg->ibm_u.putack.ibpam_rd.rd_addr);
186 __swab32s(&msg->ibm_u.putack.ibpam_rd.rd_nob);
187 __swab32s(&msg->ibm_u.putack.ibpam_rd.rd_key);
191 __swab32s(&msg->ibm_u.putack.ibpam_rd.rd_key);
192 __swab32s(&msg->ibm_u.putack.ibpam_rd.rd_nfrags);
195 n = msg->ibm_u.putack.ibpam_rd.rd_nfrags;
196 if (n <= 0 || n > IBLND_MAX_RDMA_FRAGS) {
197 CERROR("Bad PUT_ACK nfrags: %d, should be 0 < n <= %d\n",
198 n, IBLND_MAX_RDMA_FRAGS);
202 if (msg_nob < offsetof(kib_msg_t, ibm_u.putack.ibpam_rd.rd_frags[n])) {
203 CERROR("Short PUT_ACK: %d(%d)\n", msg_nob,
204 (int)offsetof(kib_msg_t, ibm_u.putack.ibpam_rd.rd_frags[n]));
209 for (i = 0; i < n; i++) {
210 __swab32s(&msg->ibm_u.putack.ibpam_rd.rd_frags[i].rf_nob);
211 __swab64s(&msg->ibm_u.putack.ibpam_rd.rd_frags[i].rf_addr);
217 case IBLND_MSG_GET_REQ:
218 if (msg_nob < hdr_size + sizeof(msg->ibm_u.get)) {
219 CERROR("Short GET_REQ: %d(%d)\n", msg_nob,
220 (int)(hdr_size + sizeof(msg->ibm_u.get)));
223 #if IBLND_MAP_ON_DEMAND
225 __swab64s(&msg->ibm_u.get.ibgm_rd.rd_addr);
226 __swab32s(&msg->ibm_u.get.ibgm_rd.rd_nob);
227 __swab32s(&msg->ibm_u.get.ibgm_rd.rd_key);
231 __swab32s(&msg->ibm_u.get.ibgm_rd.rd_key);
232 __swab32s(&msg->ibm_u.get.ibgm_rd.rd_nfrags);
235 n = msg->ibm_u.get.ibgm_rd.rd_nfrags;
236 if (n <= 0 || n > IBLND_MAX_RDMA_FRAGS) {
237 CERROR("Bad GET_REQ nfrags: %d, should be 0 < n <= %d\n",
238 n, IBLND_MAX_RDMA_FRAGS);
242 if (msg_nob < offsetof(kib_msg_t, ibm_u.get.ibgm_rd.rd_frags[n])) {
243 CERROR("Short GET_REQ: %d(%d)\n", msg_nob,
244 (int)offsetof(kib_msg_t, ibm_u.get.ibgm_rd.rd_frags[n]));
249 for (i = 0; i < msg->ibm_u.get.ibgm_rd.rd_nfrags; i++) {
250 __swab32s(&msg->ibm_u.get.ibgm_rd.rd_frags[i].rf_nob);
251 __swab64s(&msg->ibm_u.get.ibgm_rd.rd_frags[i].rf_addr);
256 case IBLND_MSG_PUT_NAK:
257 case IBLND_MSG_PUT_DONE:
258 case IBLND_MSG_GET_DONE:
259 if (msg_nob < hdr_size + sizeof(msg->ibm_u.completion)) {
260 CERROR("Short RDMA completion: %d(%d)\n", msg_nob,
261 (int)(hdr_size + sizeof(msg->ibm_u.completion)));
265 __swab32s(&msg->ibm_u.completion.ibcm_status);
268 case IBLND_MSG_CONNREQ:
269 case IBLND_MSG_CONNACK:
270 if (msg_nob < hdr_size + sizeof(msg->ibm_u.connparams)) {
271 CERROR("Short connreq/ack: %d(%d)\n", msg_nob,
272 (int)(hdr_size + sizeof(msg->ibm_u.connparams)));
276 __swab16s(&msg->ibm_u.connparams.ibcp_queue_depth);
277 __swab16s(&msg->ibm_u.connparams.ibcp_max_frags);
278 __swab32s(&msg->ibm_u.connparams.ibcp_max_msg_size);
286 kiblnd_create_peer (lnet_ni_t *ni, kib_peer_t **peerp, lnet_nid_t nid)
289 kib_net_t *net = ni->ni_data;
292 LASSERT (net != NULL);
293 LASSERT (nid != LNET_NID_ANY);
295 LIBCFS_ALLOC(peer, sizeof(*peer));
297 CERROR("Cannot allocate peer\n");
301 memset(peer, 0, sizeof(*peer)); /* zero flags etc */
306 peer->ibp_last_alive = cfs_time_current();
307 atomic_set(&peer->ibp_refcount, 1); /* 1 ref for caller */
309 INIT_LIST_HEAD(&peer->ibp_list); /* not in the peer table yet */
310 INIT_LIST_HEAD(&peer->ibp_conns);
311 INIT_LIST_HEAD(&peer->ibp_tx_queue);
313 write_lock_irqsave(&kiblnd_data.kib_global_lock, flags);
315 /* always called with a ref on ni, which prevents ni being shutdown */
316 LASSERT (net->ibn_shutdown == 0);
318 /* npeers only grows with the global lock held */
319 atomic_inc(&net->ibn_npeers);
321 write_unlock_irqrestore(&kiblnd_data.kib_global_lock, flags);
328 kiblnd_destroy_peer (kib_peer_t *peer)
330 kib_net_t *net = peer->ibp_ni->ni_data;
332 LASSERT (net != NULL);
333 LASSERT (atomic_read(&peer->ibp_refcount) == 0);
334 LASSERT (!kiblnd_peer_active(peer));
335 LASSERT (peer->ibp_connecting == 0);
336 LASSERT (peer->ibp_accepting == 0);
337 LASSERT (list_empty(&peer->ibp_conns));
338 LASSERT (list_empty(&peer->ibp_tx_queue));
340 LIBCFS_FREE(peer, sizeof(*peer));
342 /* NB a peer's connections keep a reference on their peer until
343 * they are destroyed, so we can be assured that _all_ state to do
344 * with this peer has been cleaned up when its refcount drops to
346 atomic_dec(&net->ibn_npeers);
350 kiblnd_destroy_dev (kib_dev_t *dev)
352 LASSERT (dev->ibd_nnets == 0);
354 if (!list_empty(&dev->ibd_list)) /* on kib_devs? */
355 list_del_init(&dev->ibd_list);
357 if (dev->ibd_mr != NULL)
358 ib_dereg_mr(dev->ibd_mr);
360 if (dev->ibd_pd != NULL)
361 ib_dealloc_pd(dev->ibd_pd);
363 if (dev->ibd_cmid != NULL)
364 rdma_destroy_id(dev->ibd_cmid);
366 LIBCFS_FREE(dev, sizeof(*dev));
370 kiblnd_find_peer_locked (lnet_nid_t nid)
372 /* the caller is responsible for accounting the additional reference
373 * that this creates */
374 struct list_head *peer_list = kiblnd_nid2peerlist(nid);
375 struct list_head *tmp;
378 list_for_each (tmp, peer_list) {
380 peer = list_entry(tmp, kib_peer_t, ibp_list);
382 LASSERT (peer->ibp_connecting > 0 || /* creating conns */
383 peer->ibp_accepting > 0 ||
384 !list_empty(&peer->ibp_conns)); /* active conn */
386 if (peer->ibp_nid != nid)
389 CDEBUG(D_NET, "got peer [%p] -> %s (%d)\n",
390 peer, libcfs_nid2str(nid),
391 atomic_read(&peer->ibp_refcount));
398 kiblnd_unlink_peer_locked (kib_peer_t *peer)
400 LASSERT (list_empty(&peer->ibp_conns));
402 LASSERT (kiblnd_peer_active(peer));
403 list_del_init(&peer->ibp_list);
404 /* lose peerlist's ref */
405 kiblnd_peer_decref(peer);
409 kiblnd_get_peer_info (lnet_ni_t *ni, int index,
410 lnet_nid_t *nidp, int *count)
413 struct list_head *ptmp;
417 read_lock_irqsave(&kiblnd_data.kib_global_lock, flags);
419 for (i = 0; i < kiblnd_data.kib_peer_hash_size; i++) {
421 list_for_each (ptmp, &kiblnd_data.kib_peers[i]) {
423 peer = list_entry(ptmp, kib_peer_t, ibp_list);
424 LASSERT (peer->ibp_connecting > 0 ||
425 peer->ibp_accepting > 0 ||
426 !list_empty(&peer->ibp_conns));
428 if (peer->ibp_ni != ni)
434 *nidp = peer->ibp_nid;
435 *count = atomic_read(&peer->ibp_refcount);
437 read_unlock_irqrestore(&kiblnd_data.kib_global_lock,
443 read_unlock_irqrestore(&kiblnd_data.kib_global_lock, flags);
448 kiblnd_del_peer_locked (kib_peer_t *peer)
450 struct list_head *ctmp;
451 struct list_head *cnxt;
454 if (list_empty(&peer->ibp_conns)) {
455 kiblnd_unlink_peer_locked(peer);
457 list_for_each_safe (ctmp, cnxt, &peer->ibp_conns) {
458 conn = list_entry(ctmp, kib_conn_t, ibc_list);
460 kiblnd_close_conn_locked(conn, 0);
462 /* NB closing peer's last conn unlinked it. */
464 /* NB peer now unlinked; might even be freed if the peer table had the
469 kiblnd_del_peer (lnet_ni_t *ni, lnet_nid_t nid)
471 CFS_LIST_HEAD (zombies);
472 struct list_head *ptmp;
473 struct list_head *pnxt;
481 write_lock_irqsave(&kiblnd_data.kib_global_lock, flags);
483 if (nid != LNET_NID_ANY) {
484 lo = hi = kiblnd_nid2peerlist(nid) - kiblnd_data.kib_peers;
487 hi = kiblnd_data.kib_peer_hash_size - 1;
490 for (i = lo; i <= hi; i++) {
491 list_for_each_safe (ptmp, pnxt, &kiblnd_data.kib_peers[i]) {
492 peer = list_entry(ptmp, kib_peer_t, ibp_list);
493 LASSERT (peer->ibp_connecting > 0 ||
494 peer->ibp_accepting > 0 ||
495 !list_empty(&peer->ibp_conns));
497 if (peer->ibp_ni != ni)
500 if (!(nid == LNET_NID_ANY || peer->ibp_nid == nid))
503 if (!list_empty(&peer->ibp_tx_queue)) {
504 LASSERT (list_empty(&peer->ibp_conns));
506 list_splice_init(&peer->ibp_tx_queue, &zombies);
509 kiblnd_del_peer_locked(peer);
510 rc = 0; /* matched something */
514 write_unlock_irqrestore(&kiblnd_data.kib_global_lock, flags);
516 kiblnd_txlist_done(ni, &zombies, -EIO);
522 kiblnd_get_conn_by_idx (lnet_ni_t *ni, int index)
525 struct list_head *ptmp;
527 struct list_head *ctmp;
531 read_lock_irqsave(&kiblnd_data.kib_global_lock, flags);
533 for (i = 0; i < kiblnd_data.kib_peer_hash_size; i++) {
534 list_for_each (ptmp, &kiblnd_data.kib_peers[i]) {
536 peer = list_entry(ptmp, kib_peer_t, ibp_list);
537 LASSERT (peer->ibp_connecting > 0 ||
538 peer->ibp_accepting > 0 ||
539 !list_empty(&peer->ibp_conns));
541 if (peer->ibp_ni != ni)
544 list_for_each (ctmp, &peer->ibp_conns) {
548 conn = list_entry(ctmp, kib_conn_t, ibc_list);
549 kiblnd_conn_addref(conn);
550 read_unlock_irqrestore(&kiblnd_data.kib_global_lock,
557 read_unlock_irqrestore(&kiblnd_data.kib_global_lock, flags);
562 kiblnd_debug_rx (kib_rx_t *rx)
564 CDEBUG(D_CONSOLE, " %p status %d msg_type %x cred %d\n",
565 rx, rx->rx_status, rx->rx_msg->ibm_type,
566 rx->rx_msg->ibm_credits);
570 kiblnd_debug_tx (kib_tx_t *tx)
572 CDEBUG(D_CONSOLE, " %p snd %d q %d w %d rc %d dl %lx "
573 "cookie "LPX64" msg %s%s type %x cred %d\n",
574 tx, tx->tx_sending, tx->tx_queued, tx->tx_waiting,
575 tx->tx_status, tx->tx_deadline, tx->tx_cookie,
576 tx->tx_lntmsg[0] == NULL ? "-" : "!",
577 tx->tx_lntmsg[1] == NULL ? "-" : "!",
578 tx->tx_msg->ibm_type, tx->tx_msg->ibm_credits);
582 kiblnd_debug_conn (kib_conn_t *conn)
584 struct list_head *tmp;
587 spin_lock(&conn->ibc_lock);
589 CDEBUG(D_CONSOLE, "conn[%d] %p -> %s: \n",
590 atomic_read(&conn->ibc_refcount), conn,
591 libcfs_nid2str(conn->ibc_peer->ibp_nid));
592 CDEBUG(D_CONSOLE, " state %d nposted %d cred %d o_cred %d r_cred %d\n",
593 conn->ibc_state, conn->ibc_nsends_posted, conn->ibc_credits,
594 conn->ibc_outstanding_credits, conn->ibc_reserved_credits);
595 CDEBUG(D_CONSOLE, " comms_err %d\n", conn->ibc_comms_error);
597 CDEBUG(D_CONSOLE, " early_rxs:\n");
598 list_for_each(tmp, &conn->ibc_early_rxs)
599 kiblnd_debug_rx(list_entry(tmp, kib_rx_t, rx_list));
601 CDEBUG(D_CONSOLE, " tx_queue_nocred:\n");
602 list_for_each(tmp, &conn->ibc_tx_queue_nocred)
603 kiblnd_debug_tx(list_entry(tmp, kib_tx_t, tx_list));
605 CDEBUG(D_CONSOLE, " tx_queue_rsrvd:\n");
606 list_for_each(tmp, &conn->ibc_tx_queue_rsrvd)
607 kiblnd_debug_tx(list_entry(tmp, kib_tx_t, tx_list));
609 CDEBUG(D_CONSOLE, " tx_queue:\n");
610 list_for_each(tmp, &conn->ibc_tx_queue)
611 kiblnd_debug_tx(list_entry(tmp, kib_tx_t, tx_list));
613 CDEBUG(D_CONSOLE, " active_txs:\n");
614 list_for_each(tmp, &conn->ibc_active_txs)
615 kiblnd_debug_tx(list_entry(tmp, kib_tx_t, tx_list));
617 CDEBUG(D_CONSOLE, " rxs:\n");
618 for (i = 0; i < IBLND_RX_MSGS; i++)
619 kiblnd_debug_rx(&conn->ibc_rxs[i]);
621 spin_unlock(&conn->ibc_lock);
625 kiblnd_create_conn (kib_peer_t *peer, struct rdma_cm_id *cmid, int state)
628 * If the new conn is created successfully it takes over the caller's
629 * ref on 'peer'. It also "owns" 'cmid' and destroys it when it itself
630 * is destroyed. On failure, the caller's ref on 'peer' remains and
631 * she must dispose of 'cmid'. (Actually I'd block forever if I tried
632 * to destroy 'cmid' here since I'm called from the CM which still has
633 * its ref on 'cmid'). */
635 kib_net_t *net = peer->ibp_ni->ni_data;
641 struct ib_qp_init_attr *init_qp_attr;
644 LASSERT (net != NULL);
645 LASSERT (!in_interrupt());
647 LIBCFS_ALLOC(init_qp_attr, sizeof(*init_qp_attr));
648 if (init_qp_attr == NULL) {
649 CERROR("Can't allocate qp_attr for %s\n",
650 libcfs_nid2str(peer->ibp_nid));
654 LIBCFS_ALLOC(conn, sizeof(*conn));
656 CERROR("Can't allocate connection for %s\n",
657 libcfs_nid2str(peer->ibp_nid));
661 memset(conn, 0, sizeof(*conn)); /* zero flags, NULL pointers etc... */
663 conn->ibc_state = IBLND_CONN_INIT;
664 conn->ibc_peer = peer; /* I take the caller's ref */
665 cmid->context = conn; /* for future CM callbacks */
666 conn->ibc_cmid = cmid;
668 INIT_LIST_HEAD(&conn->ibc_early_rxs);
669 INIT_LIST_HEAD(&conn->ibc_tx_queue);
670 INIT_LIST_HEAD(&conn->ibc_tx_queue_rsrvd);
671 INIT_LIST_HEAD(&conn->ibc_tx_queue_nocred);
672 INIT_LIST_HEAD(&conn->ibc_active_txs);
673 spin_lock_init(&conn->ibc_lock);
675 LIBCFS_ALLOC(conn->ibc_connvars, sizeof(*conn->ibc_connvars));
676 if (conn->ibc_connvars == NULL) {
677 CERROR("Can't allocate in-progress connection state\n");
680 memset(conn->ibc_connvars, 0, sizeof(*conn->ibc_connvars));
682 LIBCFS_ALLOC(conn->ibc_rxs, IBLND_RX_MSGS * sizeof(kib_rx_t));
683 if (conn->ibc_rxs == NULL) {
684 CERROR("Cannot allocate RX buffers\n");
687 memset(conn->ibc_rxs, 0, IBLND_RX_MSGS * sizeof(kib_rx_t));
689 rc = kiblnd_alloc_pages(&conn->ibc_rx_pages, IBLND_RX_MSG_PAGES);
693 for (i = ipage = page_offset = 0; i < IBLND_RX_MSGS; i++) {
694 struct page *page = conn->ibc_rx_pages->ibp_pages[ipage];
695 kib_rx_t *rx = &conn->ibc_rxs[i];
698 rx->rx_msg = (kib_msg_t *)(((char *)page_address(page)) +
700 rx->rx_msgaddr = dma_map_single(cmid->device->dma_device,
704 pci_unmap_addr_set(rx, rx_msgunmap, rx->rx_msgaddr);
706 CDEBUG(D_NET,"rx %d: %p "LPX64"("LPX64")\n",
707 i, rx->rx_msg, rx->rx_msgaddr,
708 lnet_page2phys(page) + page_offset);
710 page_offset += IBLND_MSG_SIZE;
711 LASSERT (page_offset <= PAGE_SIZE);
713 if (page_offset == PAGE_SIZE) {
716 LASSERT (ipage <= IBLND_RX_MSG_PAGES);
720 cq = ib_create_cq(cmid->device,
721 kiblnd_cq_completion, kiblnd_cq_event, conn,
726 CERROR("Can't create CQ: %ld\n", PTR_ERR(cq));
730 rc = ib_req_notify_cq(cq, IB_CQ_NEXT_COMP);
732 CERROR("Can't request completion notificiation: %d\n", rc);
736 memset(init_qp_attr, 0, sizeof(*init_qp_attr));
737 init_qp_attr->event_handler = kiblnd_qp_event;
738 init_qp_attr->qp_context = conn;
739 init_qp_attr->cap.max_send_wr = (*kiblnd_tunables.kib_concurrent_sends) *
740 (1 + IBLND_MAX_RDMA_FRAGS);
741 init_qp_attr->cap.max_recv_wr = IBLND_RX_MSGS;
742 init_qp_attr->cap.max_send_sge = 1;
743 init_qp_attr->cap.max_recv_sge = 1;
744 init_qp_attr->sq_sig_type = IB_SIGNAL_REQ_WR;
745 init_qp_attr->qp_type = IB_QPT_RC;
746 init_qp_attr->send_cq = cq;
747 init_qp_attr->recv_cq = cq;
750 write_lock_irqsave(&kiblnd_data.kib_global_lock, flags);
751 switch (*kiblnd_tunables.kib_ib_mtu) {
753 rc = *kiblnd_tunables.kib_ib_mtu;
754 /* fall through to... */
755 case 0: /* set tunable to the default
756 * CAVEAT EMPTOR! this assumes the default is one of the MTUs
757 * below, otherwise we'll WARN on the next QP create */
758 *kiblnd_tunables.kib_ib_mtu =
759 ib_mtu_enum_to_int(cmid->route.path_rec->mtu);
762 cmid->route.path_rec->mtu = IB_MTU_256;
765 cmid->route.path_rec->mtu = IB_MTU_512;
768 cmid->route.path_rec->mtu = IB_MTU_1024;
771 cmid->route.path_rec->mtu = IB_MTU_2048;
774 cmid->route.path_rec->mtu = IB_MTU_4096;
777 write_unlock_irqrestore(&kiblnd_data.kib_global_lock, flags);
780 CWARN("Invalid IB MTU value %d, using default value %d\n",
781 rc, *kiblnd_tunables.kib_ib_mtu);
783 rc = rdma_create_qp(cmid, net->ibn_dev->ibd_pd, init_qp_attr);
785 CERROR("Can't create QP: %d\n", rc);
789 LIBCFS_FREE(init_qp_attr, sizeof(*init_qp_attr));
791 /* 1 ref for caller and each rxmsg */
792 atomic_set(&conn->ibc_refcount, 1 + IBLND_RX_MSGS);
793 conn->ibc_nrx = IBLND_RX_MSGS;
796 for (i = 0; i < IBLND_RX_MSGS; i++) {
797 rc = kiblnd_post_rx(&conn->ibc_rxs[i],
798 IBLND_POSTRX_NO_CREDIT);
800 CERROR("Can't post rxmsg: %d\n", rc);
802 /* Make posted receives complete */
803 kiblnd_abort_receives(conn);
805 /* correct # of posted buffers
806 * NB locking needed now I'm racing with completion */
807 spin_lock_irqsave(&kiblnd_data.kib_sched_lock, flags);
808 conn->ibc_nrx -= IBLND_RX_MSGS - i;
809 spin_unlock_irqrestore(&kiblnd_data.kib_sched_lock,
812 /* Drop my own and unused rxbuffer refcounts */
813 while (i++ <= IBLND_RX_MSGS)
814 kiblnd_conn_decref(conn);
820 /* Init successful! */
821 LASSERT (state == IBLND_CONN_ACTIVE_CONNECT ||
822 state == IBLND_CONN_PASSIVE_WAIT);
823 conn->ibc_state = state;
826 atomic_inc(&net->ibn_nconns);
830 kiblnd_destroy_conn(conn);
832 LIBCFS_FREE(init_qp_attr, sizeof(*init_qp_attr));
838 kiblnd_destroy_conn (kib_conn_t *conn)
840 struct rdma_cm_id *cmid = conn->ibc_cmid;
841 kib_peer_t *peer = conn->ibc_peer;
845 LASSERT (!in_interrupt());
846 LASSERT (atomic_read(&conn->ibc_refcount) == 0);
847 LASSERT (list_empty(&conn->ibc_early_rxs));
848 LASSERT (list_empty(&conn->ibc_tx_queue));
849 LASSERT (list_empty(&conn->ibc_tx_queue_rsrvd));
850 LASSERT (list_empty(&conn->ibc_tx_queue_nocred));
851 LASSERT (list_empty(&conn->ibc_active_txs));
852 LASSERT (conn->ibc_nsends_posted == 0);
854 switch (conn->ibc_state) {
856 /* conn must be completely disengaged from the network */
859 case IBLND_CONN_DISCONNECTED:
860 /* connvars should have been freed already */
861 LASSERT (conn->ibc_connvars == NULL);
864 case IBLND_CONN_INIT:
868 if (conn->ibc_cmid->qp != NULL)
869 rdma_destroy_qp(conn->ibc_cmid);
871 if (conn->ibc_cq != NULL) {
872 rc = ib_destroy_cq(conn->ibc_cq);
874 CWARN("Error destroying CQ: %d\n", rc);
877 if (conn->ibc_rx_pages != NULL) {
878 LASSERT (conn->ibc_rxs != NULL);
880 for (i = 0; i < IBLND_RX_MSGS; i++) {
881 kib_rx_t *rx = &conn->ibc_rxs[i];
883 LASSERT (rx->rx_nob >= 0); /* not posted */
885 dma_unmap_single(conn->ibc_cmid->device->dma_device,
886 pci_unmap_addr(rx, rx_msgunmap),
887 IBLND_MSG_SIZE, DMA_FROM_DEVICE);
890 kiblnd_free_pages(conn->ibc_rx_pages);
893 if (conn->ibc_rxs != NULL) {
894 LIBCFS_FREE(conn->ibc_rxs,
895 IBLND_RX_MSGS * sizeof(kib_rx_t));
898 if (conn->ibc_connvars != NULL)
899 LIBCFS_FREE(conn->ibc_connvars, sizeof(*conn->ibc_connvars));
901 /* See CAVEAT EMPTOR above in kiblnd_create_conn */
902 if (conn->ibc_state != IBLND_CONN_INIT) {
903 kib_net_t *net = peer->ibp_ni->ni_data;
905 kiblnd_peer_decref(peer);
906 rdma_destroy_id(cmid);
907 atomic_dec(&net->ibn_nconns);
910 LIBCFS_FREE(conn, sizeof(*conn));
914 kiblnd_close_peer_conns_locked (kib_peer_t *peer, int why)
917 struct list_head *ctmp;
918 struct list_head *cnxt;
921 list_for_each_safe (ctmp, cnxt, &peer->ibp_conns) {
922 conn = list_entry(ctmp, kib_conn_t, ibc_list);
925 kiblnd_close_conn_locked(conn, why);
932 kiblnd_close_stale_conns_locked (kib_peer_t *peer, __u64 incarnation)
935 struct list_head *ctmp;
936 struct list_head *cnxt;
939 list_for_each_safe (ctmp, cnxt, &peer->ibp_conns) {
940 conn = list_entry(ctmp, kib_conn_t, ibc_list);
942 if (conn->ibc_incarnation == incarnation)
945 CDEBUG(D_NET, "Closing stale conn -> %s incarnation:"LPX64"("LPX64")\n",
946 libcfs_nid2str(peer->ibp_nid),
947 conn->ibc_incarnation, incarnation);
950 kiblnd_close_conn_locked(conn, -ESTALE);
957 kiblnd_close_matching_conns (lnet_ni_t *ni, lnet_nid_t nid)
960 struct list_head *ptmp;
961 struct list_head *pnxt;
968 write_lock_irqsave(&kiblnd_data.kib_global_lock, flags);
970 if (nid != LNET_NID_ANY)
971 lo = hi = kiblnd_nid2peerlist(nid) - kiblnd_data.kib_peers;
974 hi = kiblnd_data.kib_peer_hash_size - 1;
977 for (i = lo; i <= hi; i++) {
978 list_for_each_safe (ptmp, pnxt, &kiblnd_data.kib_peers[i]) {
980 peer = list_entry(ptmp, kib_peer_t, ibp_list);
981 LASSERT (peer->ibp_connecting > 0 ||
982 peer->ibp_accepting > 0 ||
983 !list_empty(&peer->ibp_conns));
985 if (peer->ibp_ni != ni)
988 if (!(nid == LNET_NID_ANY || nid == peer->ibp_nid))
991 count += kiblnd_close_peer_conns_locked(peer, 0);
995 write_unlock_irqrestore(&kiblnd_data.kib_global_lock, flags);
997 /* wildcards always succeed */
998 if (nid == LNET_NID_ANY)
1001 return (count == 0) ? -ENOENT : 0;
1005 kiblnd_ctl(lnet_ni_t *ni, unsigned int cmd, void *arg)
1007 struct libcfs_ioctl_data *data = arg;
1011 case IOC_LIBCFS_GET_PEER: {
1015 rc = kiblnd_get_peer_info(ni, data->ioc_count,
1017 data->ioc_nid = nid;
1018 data->ioc_count = count;
1022 case IOC_LIBCFS_DEL_PEER: {
1023 rc = kiblnd_del_peer(ni, data->ioc_nid);
1026 case IOC_LIBCFS_GET_CONN: {
1027 kib_conn_t *conn = kiblnd_get_conn_by_idx(ni, data->ioc_count);
1032 // kiblnd_debug_conn(conn);
1034 data->ioc_nid = conn->ibc_peer->ibp_nid;
1035 kiblnd_conn_decref(conn);
1039 case IOC_LIBCFS_CLOSE_CONNECTION: {
1040 rc = kiblnd_close_matching_conns(ni, data->ioc_nid);
1052 kiblnd_free_pages (kib_pages_t *p)
1054 int npages = p->ibp_npages;
1057 for (i = 0; i < npages; i++)
1058 if (p->ibp_pages[i] != NULL)
1059 __free_page(p->ibp_pages[i]);
1061 LIBCFS_FREE (p, offsetof(kib_pages_t, ibp_pages[npages]));
1065 kiblnd_alloc_pages (kib_pages_t **pp, int npages)
1070 LIBCFS_ALLOC(p, offsetof(kib_pages_t, ibp_pages[npages]));
1072 CERROR("Can't allocate descriptor for %d pages\n", npages);
1076 memset(p, 0, offsetof(kib_pages_t, ibp_pages[npages]));
1077 p->ibp_npages = npages;
1079 for (i = 0; i < npages; i++) {
1080 p->ibp_pages[i] = alloc_page(GFP_KERNEL);
1081 if (p->ibp_pages[i] == NULL) {
1082 CERROR("Can't allocate page %d of %d\n", i, npages);
1083 kiblnd_free_pages(p);
1093 kiblnd_free_tx_descs (lnet_ni_t *ni)
1096 kib_net_t *net = ni->ni_data;
1098 LASSERT (net != NULL);
1100 if (net->ibn_tx_descs != NULL) {
1101 for (i = 0; i < IBLND_TX_MSGS(); i++) {
1102 kib_tx_t *tx = &net->ibn_tx_descs[i];
1104 #if IBLND_MAP_ON_DEMAND
1105 if (tx->tx_pages != NULL)
1106 LIBCFS_FREE(tx->tx_pages, LNET_MAX_IOV *
1107 sizeof(*tx->tx_pages));
1109 if (tx->tx_wrq != NULL)
1110 LIBCFS_FREE(tx->tx_wrq,
1111 (1 + IBLND_MAX_RDMA_FRAGS) *
1112 sizeof(*tx->tx_wrq));
1114 if (tx->tx_sge != NULL)
1115 LIBCFS_FREE(tx->tx_sge,
1116 (1 + IBLND_MAX_RDMA_FRAGS) *
1117 sizeof(*tx->tx_sge));
1119 if (tx->tx_rd != NULL)
1120 LIBCFS_FREE(tx->tx_rd,
1121 offsetof(kib_rdma_desc_t,
1122 rd_frags[IBLND_MAX_RDMA_FRAGS]));
1124 if (tx->tx_frags != NULL)
1125 LIBCFS_FREE(tx->tx_frags,
1126 IBLND_MAX_RDMA_FRAGS *
1127 sizeof(*tx->tx_frags));
1131 LIBCFS_FREE(net->ibn_tx_descs,
1132 IBLND_TX_MSGS() * sizeof(kib_tx_t));
1135 if (net->ibn_tx_pages != NULL)
1136 kiblnd_free_pages(net->ibn_tx_pages);
1140 kiblnd_alloc_tx_descs (lnet_ni_t *ni)
1144 kib_net_t *net = ni->ni_data;
1146 LASSERT (net != NULL);
1148 rc = kiblnd_alloc_pages(&net->ibn_tx_pages, IBLND_TX_MSG_PAGES());
1151 CERROR("Can't allocate tx pages\n");
1155 LIBCFS_ALLOC (net->ibn_tx_descs,
1156 IBLND_TX_MSGS() * sizeof(kib_tx_t));
1157 if (net->ibn_tx_descs == NULL) {
1158 CERROR("Can't allocate %d tx descriptors\n", IBLND_TX_MSGS());
1162 memset(net->ibn_tx_descs, 0,
1163 IBLND_TX_MSGS() * sizeof(kib_tx_t));
1165 for (i = 0; i < IBLND_TX_MSGS(); i++) {
1166 kib_tx_t *tx = &net->ibn_tx_descs[i];
1168 #if IBLND_MAP_ON_DEMAND
1169 LIBCFS_ALLOC(tx->tx_pages, LNET_MAX_IOV *
1170 sizeof(*tx->tx_pages));
1171 if (tx->tx_pages == NULL) {
1172 CERROR("Can't allocate phys page vector[%d]\n",
1177 LIBCFS_ALLOC(tx->tx_wrq,
1178 (1 + IBLND_MAX_RDMA_FRAGS) *
1179 sizeof(*tx->tx_wrq));
1180 if (tx->tx_wrq == NULL)
1183 LIBCFS_ALLOC(tx->tx_sge,
1184 (1 + IBLND_MAX_RDMA_FRAGS) *
1185 sizeof(*tx->tx_sge));
1186 if (tx->tx_sge == NULL)
1189 LIBCFS_ALLOC(tx->tx_rd,
1190 offsetof(kib_rdma_desc_t,
1191 rd_frags[IBLND_MAX_RDMA_FRAGS]));
1192 if (tx->tx_rd == NULL)
1195 LIBCFS_ALLOC(tx->tx_frags,
1196 IBLND_MAX_RDMA_FRAGS *
1197 sizeof(*tx->tx_frags));
1198 if (tx->tx_frags == NULL)
1207 kiblnd_unmap_tx_descs (lnet_ni_t *ni)
1211 kib_net_t *net = ni->ni_data;
1213 LASSERT (net != NULL);
1215 for (i = 0; i < IBLND_TX_MSGS(); i++) {
1216 tx = &net->ibn_tx_descs[i];
1218 dma_unmap_single(net->ibn_dev->ibd_cmid->device->dma_device,
1219 pci_unmap_addr(tx, tx_msgunmap),
1220 IBLND_MSG_SIZE, DMA_TO_DEVICE);
1225 kiblnd_map_tx_descs (lnet_ni_t *ni)
1228 int page_offset = 0;
1232 kib_net_t *net = ni->ni_data;
1234 LASSERT (net != NULL);
1236 /* pre-mapped messages are not bigger than 1 page */
1237 CLASSERT (IBLND_MSG_SIZE <= PAGE_SIZE);
1239 /* No fancy arithmetic when we do the buffer calculations */
1240 CLASSERT (PAGE_SIZE % IBLND_MSG_SIZE == 0);
1242 for (i = 0; i < IBLND_TX_MSGS(); i++) {
1243 page = net->ibn_tx_pages->ibp_pages[ipage];
1244 tx = &net->ibn_tx_descs[i];
1246 tx->tx_msg = (kib_msg_t *)(((char *)page_address(page)) +
1249 tx->tx_msgaddr = dma_map_single(
1250 net->ibn_dev->ibd_cmid->device->dma_device,
1251 tx->tx_msg, IBLND_MSG_SIZE, DMA_TO_DEVICE);
1252 pci_unmap_addr_set(tx, tx_msgunmap, tx->tx_msgaddr);
1254 list_add(&tx->tx_list, &net->ibn_idle_txs);
1256 page_offset += IBLND_MSG_SIZE;
1257 LASSERT (page_offset <= PAGE_SIZE);
1259 if (page_offset == PAGE_SIZE) {
1262 LASSERT (ipage <= IBLND_TX_MSG_PAGES());
1268 kiblnd_base_shutdown (void)
1272 LASSERT (list_empty(&kiblnd_data.kib_devs));
1274 CDEBUG(D_MALLOC, "before LND base cleanup: kmem %d\n",
1275 atomic_read(&libcfs_kmemory));
1277 switch (kiblnd_data.kib_init) {
1281 case IBLND_INIT_ALL:
1282 case IBLND_INIT_DATA:
1283 LASSERT (kiblnd_data.kib_peers != NULL);
1284 for (i = 0; i < kiblnd_data.kib_peer_hash_size; i++) {
1285 LASSERT (list_empty(&kiblnd_data.kib_peers[i]));
1287 LASSERT (list_empty(&kiblnd_data.kib_connd_zombies));
1288 LASSERT (list_empty(&kiblnd_data.kib_connd_conns));
1290 /* flag threads to terminate; wake and wait for them to die */
1291 kiblnd_data.kib_shutdown = 1;
1292 wake_up_all(&kiblnd_data.kib_sched_waitq);
1293 wake_up_all(&kiblnd_data.kib_connd_waitq);
1296 while (atomic_read(&kiblnd_data.kib_nthreads) != 0) {
1298 CDEBUG(((i & (-i)) == i) ? D_WARNING : D_NET, /* power of 2? */
1299 "Waiting for %d threads to terminate\n",
1300 atomic_read(&kiblnd_data.kib_nthreads));
1301 cfs_pause(cfs_time_seconds(1));
1306 case IBLND_INIT_NOTHING:
1310 if (kiblnd_data.kib_peers != NULL)
1311 LIBCFS_FREE(kiblnd_data.kib_peers,
1312 sizeof(struct list_head) *
1313 kiblnd_data.kib_peer_hash_size);
1315 CDEBUG(D_MALLOC, "after LND base cleanup: kmem %d\n",
1316 atomic_read(&libcfs_kmemory));
1318 kiblnd_data.kib_init = IBLND_INIT_NOTHING;
1319 PORTAL_MODULE_UNUSE;
1323 kiblnd_shutdown (lnet_ni_t *ni)
1325 kib_net_t *net = ni->ni_data;
1326 rwlock_t *g_lock = &kiblnd_data.kib_global_lock;
1328 unsigned long flags;
1330 LASSERT(kiblnd_data.kib_init == IBLND_INIT_ALL);
1335 CDEBUG(D_MALLOC, "before LND net cleanup: kmem %d\n",
1336 atomic_read(&libcfs_kmemory));
1338 write_lock_irqsave(g_lock, flags);
1339 net->ibn_shutdown = 1;
1340 write_unlock_irqrestore(g_lock, flags);
1342 switch (net->ibn_init) {
1346 case IBLND_INIT_ALL:
1347 /* nuke all existing peers within this net */
1348 kiblnd_del_peer(ni, LNET_NID_ANY);
1350 /* Wait for all peer state to clean up */
1352 while (atomic_read(&net->ibn_npeers) != 0) {
1354 CDEBUG(((i & (-i)) == i) ? D_WARNING : D_NET, /* 2**n? */
1355 "%s: waiting for %d peers to disconnect\n",
1356 libcfs_nid2str(ni->ni_nid),
1357 atomic_read(&net->ibn_npeers));
1358 cfs_pause(cfs_time_seconds(1));
1361 kiblnd_unmap_tx_descs(ni);
1363 LASSERT (net->ibn_dev->ibd_nnets > 0);
1364 net->ibn_dev->ibd_nnets--;
1368 case IBLND_INIT_NOTHING:
1369 LASSERT (atomic_read(&net->ibn_nconns) == 0);
1371 #if IBLND_MAP_ON_DEMAND
1372 if (net->ibn_fmrpool != NULL)
1373 ib_destroy_fmr_pool(net->ibn_fmrpool);
1375 if (net->ibn_dev != NULL &&
1376 net->ibn_dev->ibd_nnets == 0)
1377 kiblnd_destroy_dev(net->ibn_dev);
1382 kiblnd_free_tx_descs(ni);
1384 CDEBUG(D_MALLOC, "after LND net cleanup: kmem %d\n",
1385 atomic_read(&libcfs_kmemory));
1387 net->ibn_init = IBLND_INIT_NOTHING;
1390 LIBCFS_FREE(net, sizeof(*net));
1393 if (list_empty(&kiblnd_data.kib_devs))
1394 kiblnd_base_shutdown();
1399 kiblnd_base_startup (void)
1404 LASSERT (kiblnd_data.kib_init == IBLND_INIT_NOTHING);
1406 if (*kiblnd_tunables.kib_credits > *kiblnd_tunables.kib_ntx) {
1407 CERROR("Can't set credits(%d) > ntx(%d)\n",
1408 *kiblnd_tunables.kib_credits,
1409 *kiblnd_tunables.kib_ntx);
1414 memset(&kiblnd_data, 0, sizeof(kiblnd_data)); /* zero pointers, flags etc */
1416 rwlock_init(&kiblnd_data.kib_global_lock);
1418 INIT_LIST_HEAD(&kiblnd_data.kib_devs);
1420 kiblnd_data.kib_peer_hash_size = IBLND_PEER_HASH_SIZE;
1421 LIBCFS_ALLOC(kiblnd_data.kib_peers,
1422 sizeof(struct list_head) * kiblnd_data.kib_peer_hash_size);
1423 if (kiblnd_data.kib_peers == NULL) {
1426 for (i = 0; i < kiblnd_data.kib_peer_hash_size; i++)
1427 INIT_LIST_HEAD(&kiblnd_data.kib_peers[i]);
1429 spin_lock_init(&kiblnd_data.kib_connd_lock);
1430 INIT_LIST_HEAD(&kiblnd_data.kib_connd_conns);
1431 INIT_LIST_HEAD(&kiblnd_data.kib_connd_zombies);
1432 init_waitqueue_head(&kiblnd_data.kib_connd_waitq);
1434 spin_lock_init(&kiblnd_data.kib_sched_lock);
1435 INIT_LIST_HEAD(&kiblnd_data.kib_sched_conns);
1436 init_waitqueue_head(&kiblnd_data.kib_sched_waitq);
1438 kiblnd_data.kib_error_qpa.qp_state = IB_QPS_ERR;
1440 /* lists/ptrs/locks initialised */
1441 kiblnd_data.kib_init = IBLND_INIT_DATA;
1442 /*****************************************************/
1444 for (i = 0; i < IBLND_N_SCHED; i++) {
1445 rc = kiblnd_thread_start(kiblnd_scheduler, (void *)((long)i));
1447 CERROR("Can't spawn o2iblnd scheduler[%d]: %d\n",
1453 rc = kiblnd_thread_start(kiblnd_connd, NULL);
1455 CERROR("Can't spawn o2iblnd connd: %d\n", rc);
1459 /* flag everything initialised */
1460 kiblnd_data.kib_init = IBLND_INIT_ALL;
1461 /*****************************************************/
1466 kiblnd_base_shutdown();
1471 kiblnd_startup (lnet_ni_t *ni)
1476 struct list_head *tmp;
1480 LASSERT (ni->ni_lnd == &the_kiblnd);
1482 if (kiblnd_data.kib_init == IBLND_INIT_NOTHING) {
1483 rc = kiblnd_base_startup();
1488 LIBCFS_ALLOC(net, sizeof(*net));
1493 memset(net, 0, sizeof(*net));
1495 do_gettimeofday(&tv);
1496 net->ibn_incarnation = (((__u64)tv.tv_sec) * 1000000) + tv.tv_usec;
1498 ni->ni_maxtxcredits = *kiblnd_tunables.kib_credits;
1499 ni->ni_peertxcredits = *kiblnd_tunables.kib_peercredits;
1501 spin_lock_init(&net->ibn_tx_lock);
1502 INIT_LIST_HEAD(&net->ibn_idle_txs);
1504 rc = kiblnd_alloc_tx_descs(ni);
1506 CERROR("Can't allocate tx descs\n");
1510 if (ni->ni_interfaces[0] != NULL) {
1511 /* Use the IPoIB interface specified in 'networks=' */
1513 CLASSERT (LNET_MAX_INTERFACES > 1);
1514 if (ni->ni_interfaces[1] != NULL) {
1515 CERROR("Multiple interfaces not supported\n");
1519 ifname = ni->ni_interfaces[0];
1521 ifname = *kiblnd_tunables.kib_default_ipif;
1524 if (strlen(ifname) >= sizeof(ibdev->ibd_ifname)) {
1525 CERROR("IPoIB interface name too long: %s\n", ifname);
1530 list_for_each (tmp, &kiblnd_data.kib_devs) {
1531 ibdev = list_entry(tmp, kib_dev_t, ibd_list);
1533 if (!strcmp(&ibdev->ibd_ifname[0], ifname))
1539 if (ibdev == NULL) {
1543 struct rdma_cm_id *id;
1546 struct sockaddr_in addr;
1548 rc = libcfs_ipif_query(ifname, &up, &ip, &netmask);
1550 CERROR("Can't query IPoIB interface %s: %d\n",
1556 CERROR("Can't query IPoIB interface %s: it's down\n",
1561 LIBCFS_ALLOC(ibdev, sizeof(*ibdev));
1565 memset(ibdev, 0, sizeof(*ibdev));
1567 INIT_LIST_HEAD(&ibdev->ibd_list); /* not yet in kib_devs */
1568 ibdev->ibd_ifip = ip;
1569 strcpy(&ibdev->ibd_ifname[0], ifname);
1571 id = rdma_create_id(kiblnd_cm_callback, ibdev, RDMA_PS_TCP);
1573 ibdev->ibd_cmid = id;
1575 CERROR("Can't create listen ID: %ld\n", PTR_ERR(id));
1579 memset(&addr, 0, sizeof(addr));
1580 addr.sin_family = AF_INET;
1581 addr.sin_port = htons(*kiblnd_tunables.kib_service);
1582 addr.sin_addr.s_addr = htonl(ip);
1584 rc = rdma_bind_addr(id, (struct sockaddr *)&addr);
1586 CERROR("Can't bind to %s: %d\n", ifname, rc);
1590 /* Binding should have assigned me an IB device */
1591 LASSERT (id->device != NULL);
1593 pd = ib_alloc_pd(id->device);
1597 CERROR("Can't allocate PD: %ld\n", PTR_ERR(pd));
1601 #if IBLND_MAP_ON_DEMAND
1602 /* MR for sends and receives */
1603 mr = ib_get_dma_mr(pd, IB_ACCESS_LOCAL_WRITE);
1605 /* MR for sends, recieves _and_ RDMA...........v */
1606 mr = ib_get_dma_mr(pd, IB_ACCESS_LOCAL_WRITE |
1607 IB_ACCESS_REMOTE_WRITE);
1612 CERROR("Can't get MR: %ld\n", PTR_ERR(pd));
1616 rc = rdma_listen(id, 0);
1618 CERROR("Can't start listener: %d\n", rc);
1622 list_add_tail(&ibdev->ibd_list,
1623 &kiblnd_data.kib_devs);
1626 ni->ni_nid = LNET_MKNID(LNET_NIDNET(ni->ni_nid), ibdev->ibd_ifip);
1627 net->ibn_dev = ibdev;
1629 #if IBLND_MAP_ON_DEMAND
1630 /* FMR pool for RDMA */
1632 struct ib_fmr_pool *fmrpool;
1633 struct ib_fmr_pool_param param = {
1634 .max_pages_per_fmr = LNET_MAX_PAYLOAD/PAGE_SIZE,
1635 .page_shift = PAGE_SHIFT,
1636 .access = (IB_ACCESS_LOCAL_WRITE |
1637 IB_ACCESS_REMOTE_WRITE),
1638 .pool_size = *kiblnd_tunables.kib_fmr_pool_size,
1639 .dirty_watermark = *kiblnd_tunables.kib_fmr_flush_trigger,
1640 .flush_function = NULL,
1642 .cache = *kiblnd_tunables.kib_fmr_cache};
1644 if (*kiblnd_tunables.kib_fmr_pool_size <
1645 *kiblnd_tunables.kib_ntx) {
1646 CERROR("Can't set fmr pool size (%d) < ntx(%d)\n",
1647 *kiblnd_tunables.kib_fmr_pool_size,
1648 *kiblnd_tunables.kib_ntx);
1652 fmrpool = ib_create_fmr_pool(ibdev->ibd_pd, ¶m);
1653 if (!IS_ERR(fmrpool)) {
1654 net->ibn_fmrpool = fmrpool;
1656 CERROR("Can't create FMR pool: %ld\n",
1663 kiblnd_map_tx_descs(ni);
1666 net->ibn_init = IBLND_INIT_ALL;
1671 kiblnd_shutdown(ni);
1673 CDEBUG(D_NET, "kiblnd_startup failed\n");
1678 kiblnd_module_fini (void)
1680 lnet_unregister_lnd(&the_kiblnd);
1681 kiblnd_tunables_fini();
1685 kiblnd_module_init (void)
1689 CLASSERT (sizeof(kib_msg_t) <= IBLND_MSG_SIZE);
1690 #if !IBLND_MAP_ON_DEMAND
1691 CLASSERT (offsetof(kib_msg_t, ibm_u.get.ibgm_rd.rd_frags[IBLND_MAX_RDMA_FRAGS])
1693 CLASSERT (offsetof(kib_msg_t, ibm_u.putack.ibpam_rd.rd_frags[IBLND_MAX_RDMA_FRAGS])
1696 rc = kiblnd_tunables_init();
1700 lnet_register_lnd(&the_kiblnd);
1705 MODULE_AUTHOR("Cluster File Systems, Inc. <info@clusterfs.com>");
1706 MODULE_DESCRIPTION("Kernel OpenIB gen2 LND v1.00");
1707 MODULE_LICENSE("GPL");
1709 module_init(kiblnd_module_init);
1710 module_exit(kiblnd_module_fini);