1 /* -*- mode: c; c-basic-offset: 8; indent-tabs-mode: nil; -*-
2 * vim:expandtab:shiftwidth=8:tabstop=8:
4 * Copyright (C) 2004 Cluster File Systems, Inc.
5 * Author: Eric Barton <eric@bartonsoftware.com>
6 * Author: Frank Zago <fzago@systemfabricworks.com>
8 * This file is part of Lustre, http://www.lustre.org.
10 * Lustre is free software; you can redistribute it and/or
11 * modify it under the terms of version 2 of the GNU General Public
12 * License as published by the Free Software Foundation.
14 * Lustre is distributed in the hope that it will be useful,
15 * but WITHOUT ANY WARRANTY; without even the implied warranty of
16 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17 * GNU General Public License for more details.
19 * You should have received a copy of the GNU General Public License
20 * along with Lustre; if not, write to the Free Software
21 * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
29 .lnd_startup = kibnal_startup,
30 .lnd_shutdown = kibnal_shutdown,
31 .lnd_ctl = kibnal_ctl,
32 .lnd_send = kibnal_send,
33 .lnd_recv = kibnal_recv,
34 .lnd_eager_recv = kibnal_eager_recv,
37 kib_data_t kibnal_data;
39 void vibnal_assert_wire_constants (void)
41 /* Wire protocol assertions generated by 'wirecheck'
42 * running on Linux robert 2.6.11-1.27_FC3 #1 Tue May 17 20:27:37 EDT 2005 i686 athlon i386 G
43 * with gcc version 3.4.3 20050227 (Red Hat 3.4.3-22.fc3) */
47 CLASSERT (IBNAL_MSG_MAGIC == 0x0be91b91);
48 CLASSERT (IBNAL_MSG_VERSION == 0x11);
49 CLASSERT (IBNAL_MSG_CONNREQ == 0xc0);
50 CLASSERT (IBNAL_MSG_CONNACK == 0xc1);
51 CLASSERT (IBNAL_MSG_NOOP == 0xd0);
52 CLASSERT (IBNAL_MSG_IMMEDIATE == 0xd1);
53 CLASSERT (IBNAL_MSG_PUT_REQ == 0xd2);
54 CLASSERT (IBNAL_MSG_PUT_NAK == 0xd3);
55 CLASSERT (IBNAL_MSG_PUT_ACK == 0xd4);
56 CLASSERT (IBNAL_MSG_PUT_DONE == 0xd5);
57 CLASSERT (IBNAL_MSG_GET_REQ == 0xd6);
58 CLASSERT (IBNAL_MSG_GET_DONE == 0xd7);
60 /* Checks for struct kib_connparams_t */
61 CLASSERT ((int)sizeof(kib_connparams_t) == 12);
62 CLASSERT ((int)offsetof(kib_connparams_t, ibcp_queue_depth) == 0);
63 CLASSERT ((int)sizeof(((kib_connparams_t *)0)->ibcp_queue_depth) == 4);
64 CLASSERT ((int)offsetof(kib_connparams_t, ibcp_max_msg_size) == 4);
65 CLASSERT ((int)sizeof(((kib_connparams_t *)0)->ibcp_max_msg_size) == 4);
66 CLASSERT ((int)offsetof(kib_connparams_t, ibcp_max_frags) == 8);
67 CLASSERT ((int)sizeof(((kib_connparams_t *)0)->ibcp_max_frags) == 4);
69 /* Checks for struct kib_immediate_msg_t */
70 CLASSERT ((int)sizeof(kib_immediate_msg_t) == 72);
71 CLASSERT ((int)offsetof(kib_immediate_msg_t, ibim_hdr) == 0);
72 CLASSERT ((int)sizeof(((kib_immediate_msg_t *)0)->ibim_hdr) == 72);
73 CLASSERT ((int)offsetof(kib_immediate_msg_t, ibim_payload[13]) == 85);
74 CLASSERT ((int)sizeof(((kib_immediate_msg_t *)0)->ibim_payload[13]) == 1);
75 CLASSERT (IBNAL_USE_FMR == 1);
77 /* Checks for struct kib_rdma_desc_t */
78 CLASSERT ((int)sizeof(kib_rdma_desc_t) == 16);
79 CLASSERT ((int)offsetof(kib_rdma_desc_t, rd_addr) == 0);
80 CLASSERT ((int)sizeof(((kib_rdma_desc_t *)0)->rd_addr) == 8);
81 CLASSERT ((int)offsetof(kib_rdma_desc_t, rd_nob) == 8);
82 CLASSERT ((int)sizeof(((kib_rdma_desc_t *)0)->rd_nob) == 4);
83 CLASSERT ((int)offsetof(kib_rdma_desc_t, rd_key) == 12);
84 CLASSERT ((int)sizeof(((kib_rdma_desc_t *)0)->rd_key) == 4);
86 /* Checks for struct kib_putreq_msg_t */
87 CLASSERT ((int)sizeof(kib_putreq_msg_t) == 80);
88 CLASSERT ((int)offsetof(kib_putreq_msg_t, ibprm_hdr) == 0);
89 CLASSERT ((int)sizeof(((kib_putreq_msg_t *)0)->ibprm_hdr) == 72);
90 CLASSERT ((int)offsetof(kib_putreq_msg_t, ibprm_cookie) == 72);
91 CLASSERT ((int)sizeof(((kib_putreq_msg_t *)0)->ibprm_cookie) == 8);
93 /* Checks for struct kib_putack_msg_t */
94 CLASSERT ((int)sizeof(kib_putack_msg_t) == 32);
95 CLASSERT ((int)offsetof(kib_putack_msg_t, ibpam_src_cookie) == 0);
96 CLASSERT ((int)sizeof(((kib_putack_msg_t *)0)->ibpam_src_cookie) == 8);
97 CLASSERT ((int)offsetof(kib_putack_msg_t, ibpam_dst_cookie) == 8);
98 CLASSERT ((int)sizeof(((kib_putack_msg_t *)0)->ibpam_dst_cookie) == 8);
99 CLASSERT ((int)offsetof(kib_putack_msg_t, ibpam_rd) == 16);
100 CLASSERT ((int)sizeof(((kib_putack_msg_t *)0)->ibpam_rd) == 16);
102 /* Checks for struct kib_get_msg_t */
103 CLASSERT ((int)sizeof(kib_get_msg_t) == 96);
104 CLASSERT ((int)offsetof(kib_get_msg_t, ibgm_hdr) == 0);
105 CLASSERT ((int)sizeof(((kib_get_msg_t *)0)->ibgm_hdr) == 72);
106 CLASSERT ((int)offsetof(kib_get_msg_t, ibgm_cookie) == 72);
107 CLASSERT ((int)sizeof(((kib_get_msg_t *)0)->ibgm_cookie) == 8);
108 CLASSERT ((int)offsetof(kib_get_msg_t, ibgm_rd) == 80);
109 CLASSERT ((int)sizeof(((kib_get_msg_t *)0)->ibgm_rd) == 16);
111 /* Checks for struct kib_completion_msg_t */
112 CLASSERT ((int)sizeof(kib_completion_msg_t) == 12);
113 CLASSERT ((int)offsetof(kib_completion_msg_t, ibcm_cookie) == 0);
114 CLASSERT ((int)sizeof(((kib_completion_msg_t *)0)->ibcm_cookie) == 8);
115 CLASSERT ((int)offsetof(kib_completion_msg_t, ibcm_status) == 8);
116 CLASSERT ((int)sizeof(((kib_completion_msg_t *)0)->ibcm_status) == 4);
118 /* Checks for struct kib_msg_t */
119 CLASSERT ((int)sizeof(kib_msg_t) == 152);
120 CLASSERT ((int)offsetof(kib_msg_t, ibm_magic) == 0);
121 CLASSERT ((int)sizeof(((kib_msg_t *)0)->ibm_magic) == 4);
122 CLASSERT ((int)offsetof(kib_msg_t, ibm_version) == 4);
123 CLASSERT ((int)sizeof(((kib_msg_t *)0)->ibm_version) == 2);
124 CLASSERT ((int)offsetof(kib_msg_t, ibm_type) == 6);
125 CLASSERT ((int)sizeof(((kib_msg_t *)0)->ibm_type) == 1);
126 CLASSERT ((int)offsetof(kib_msg_t, ibm_credits) == 7);
127 CLASSERT ((int)sizeof(((kib_msg_t *)0)->ibm_credits) == 1);
128 CLASSERT ((int)offsetof(kib_msg_t, ibm_nob) == 8);
129 CLASSERT ((int)sizeof(((kib_msg_t *)0)->ibm_nob) == 4);
130 CLASSERT ((int)offsetof(kib_msg_t, ibm_cksum) == 12);
131 CLASSERT ((int)sizeof(((kib_msg_t *)0)->ibm_cksum) == 4);
132 CLASSERT ((int)offsetof(kib_msg_t, ibm_srcnid) == 16);
133 CLASSERT ((int)sizeof(((kib_msg_t *)0)->ibm_srcnid) == 8);
134 CLASSERT ((int)offsetof(kib_msg_t, ibm_srcstamp) == 24);
135 CLASSERT ((int)sizeof(((kib_msg_t *)0)->ibm_srcstamp) == 8);
136 CLASSERT ((int)offsetof(kib_msg_t, ibm_dstnid) == 32);
137 CLASSERT ((int)sizeof(((kib_msg_t *)0)->ibm_dstnid) == 8);
138 CLASSERT ((int)offsetof(kib_msg_t, ibm_dststamp) == 40);
139 CLASSERT ((int)sizeof(((kib_msg_t *)0)->ibm_dststamp) == 8);
140 CLASSERT ((int)offsetof(kib_msg_t, ibm_seq) == 48);
141 CLASSERT ((int)sizeof(((kib_msg_t *)0)->ibm_seq) == 8);
142 CLASSERT ((int)offsetof(kib_msg_t, ibm_u.connparams) == 56);
143 CLASSERT ((int)sizeof(((kib_msg_t *)0)->ibm_u.connparams) == 12);
144 CLASSERT ((int)offsetof(kib_msg_t, ibm_u.immediate) == 56);
145 CLASSERT ((int)sizeof(((kib_msg_t *)0)->ibm_u.immediate) == 72);
146 CLASSERT ((int)offsetof(kib_msg_t, ibm_u.putreq) == 56);
147 CLASSERT ((int)sizeof(((kib_msg_t *)0)->ibm_u.putreq) == 80);
148 CLASSERT ((int)offsetof(kib_msg_t, ibm_u.putack) == 56);
149 CLASSERT ((int)sizeof(((kib_msg_t *)0)->ibm_u.putack) == 32);
150 CLASSERT ((int)offsetof(kib_msg_t, ibm_u.get) == 56);
151 CLASSERT ((int)sizeof(((kib_msg_t *)0)->ibm_u.get) == 96);
152 CLASSERT ((int)offsetof(kib_msg_t, ibm_u.completion) == 56);
153 CLASSERT ((int)sizeof(((kib_msg_t *)0)->ibm_u.completion) == 12);
157 kibnal_cksum (void *ptr, int nob)
163 sum = ((sum << 1) | (sum >> 31)) + *c++;
165 /* ensure I don't return 0 (== no checksum) */
166 return (sum == 0) ? 1 : sum;
170 kibnal_init_msg(kib_msg_t *msg, int type, int body_nob)
172 msg->ibm_type = type;
173 msg->ibm_nob = offsetof(kib_msg_t, ibm_u) + body_nob;
177 kibnal_pack_msg(kib_msg_t *msg, __u32 version, int credits,
178 lnet_nid_t dstnid, __u64 dststamp, __u64 seq)
180 /* CAVEAT EMPTOR! all message fields not set here should have been
181 * initialised previously. */
182 msg->ibm_magic = IBNAL_MSG_MAGIC;
183 msg->ibm_version = version;
185 msg->ibm_credits = credits;
188 msg->ibm_srcnid = lnet_ptlcompat_srcnid(kibnal_data.kib_ni->ni_nid,
190 msg->ibm_srcstamp = kibnal_data.kib_incarnation;
191 msg->ibm_dstnid = dstnid;
192 msg->ibm_dststamp = dststamp;
195 if (*kibnal_tunables.kib_cksum) {
196 /* NB ibm_cksum zero while computing cksum */
197 msg->ibm_cksum = kibnal_cksum(msg, msg->ibm_nob);
202 kibnal_unpack_msg(kib_msg_t *msg, __u32 expected_version, int nob)
204 const int hdr_size = offsetof(kib_msg_t, ibm_u);
213 /* 6 bytes are enough to have received magic + version */
215 CERROR("Short message: %d\n", nob);
219 /* Future protocol version compatibility support!
220 * If the viblnd-specific protocol changes, or when LNET unifies
221 * protocols over all LNDs, the initial connection will negotiate a
222 * protocol version. If I find this, I avoid any console errors. If
223 * my is doing connection establishment, the reject will tell the peer
224 * which version I'm running. */
226 if (msg->ibm_magic == IBNAL_MSG_MAGIC) {
228 } else if (msg->ibm_magic == __swab32(IBNAL_MSG_MAGIC)) {
231 if (msg->ibm_magic == LNET_PROTO_MAGIC ||
232 msg->ibm_magic == __swab32(LNET_PROTO_MAGIC))
235 /* Completely out to lunch */
236 CERROR("Bad magic: %08x\n", msg->ibm_magic);
240 msg_version = flip ? __swab16(msg->ibm_version) : msg->ibm_version;
241 if (expected_version == 0) {
242 if (msg_version != IBNAL_MSG_VERSION_RDMAREPLYNOTRSRVD &&
243 msg_version != IBNAL_MSG_VERSION)
245 } else if (msg_version != expected_version) {
246 CERROR("Bad version: %x(%x expected)\n",
247 msg_version, expected_version);
251 if (nob < hdr_size) {
252 CERROR("Short message: %d\n", nob);
256 msg_nob = flip ? __swab32(msg->ibm_nob) : msg->ibm_nob;
258 CERROR("Short message: got %d, wanted %d\n", nob, msg_nob);
262 /* checksum must be computed with ibm_cksum zero and BEFORE anything
264 msg_cksum = flip ? __swab32(msg->ibm_cksum) : msg->ibm_cksum;
266 if (msg_cksum != 0 &&
267 msg_cksum != kibnal_cksum(msg, msg_nob)) {
268 CERROR("Bad checksum\n");
271 msg->ibm_cksum = msg_cksum;
274 /* leave magic unflipped as a clue to peer endianness */
275 msg->ibm_version = msg_version;
276 CLASSERT (sizeof(msg->ibm_type) == 1);
277 CLASSERT (sizeof(msg->ibm_credits) == 1);
278 msg->ibm_nob = msg_nob;
279 __swab64s(&msg->ibm_srcnid);
280 __swab64s(&msg->ibm_srcstamp);
281 __swab64s(&msg->ibm_dstnid);
282 __swab64s(&msg->ibm_dststamp);
283 __swab64s(&msg->ibm_seq);
286 if (msg->ibm_srcnid == LNET_NID_ANY) {
287 CERROR("Bad src nid: %s\n", libcfs_nid2str(msg->ibm_srcnid));
291 switch (msg->ibm_type) {
293 CERROR("Unknown message type %x\n", msg->ibm_type);
299 case IBNAL_MSG_IMMEDIATE:
300 if (msg_nob < offsetof(kib_msg_t, ibm_u.immediate.ibim_payload[0])) {
301 CERROR("Short IMMEDIATE: %d(%d)\n", msg_nob,
302 (int)offsetof(kib_msg_t, ibm_u.immediate.ibim_payload[0]));
307 case IBNAL_MSG_PUT_REQ:
308 if (msg_nob < hdr_size + sizeof(msg->ibm_u.putreq)) {
309 CERROR("Short PUT_REQ: %d(%d)\n", msg_nob,
310 (int)(hdr_size + sizeof(msg->ibm_u.putreq)));
315 case IBNAL_MSG_PUT_ACK:
316 if (msg_nob < hdr_size + sizeof(msg->ibm_u.putack)) {
317 CERROR("Short PUT_ACK: %d(%d)\n", msg_nob,
318 (int)(hdr_size + sizeof(msg->ibm_u.putack)));
323 __swab64s(&msg->ibm_u.putack.ibpam_rd.rd_addr);
324 __swab32s(&msg->ibm_u.putack.ibpam_rd.rd_nob);
325 __swab32s(&msg->ibm_u.putack.ibpam_rd.rd_key);
329 __swab32s(&msg->ibm_u.putack.ibpam_rd.rd_key);
330 __swab32s(&msg->ibm_u.putack.ibpam_rd.rd_nfrag);
333 n = msg->ibm_u.putack.ibpam_rd.rd_nfrag;
334 if (n <= 0 || n > IBNAL_MAX_RDMA_FRAGS) {
335 CERROR("Bad PUT_ACK nfrags: %d, should be 0 < n <= %d\n",
336 n, IBNAL_MAX_RDMA_FRAGS);
340 if (msg_nob < offsetof(kib_msg_t, ibm_u.putack.ibpam_rd.rd_frags[n])) {
341 CERROR("Short PUT_ACK: %d(%d)\n", msg_nob,
342 (int)offsetof(kib_msg_t, ibm_u.putack.ibpam_rd.rd_frags[n]));
347 for (i = 0; i < n; i++) {
348 __swab32s(&msg->ibm_u.putack.ibpam_rd.rd_frags[i].rf_nob);
349 __swab32s(&msg->ibm_u.putack.ibpam_rd.rd_frags[i].rf_addr_lo);
350 __swab32s(&msg->ibm_u.putack.ibpam_rd.rd_frags[i].rf_addr_hi);
356 case IBNAL_MSG_GET_REQ:
357 if (msg_nob < hdr_size + sizeof(msg->ibm_u.get)) {
358 CERROR("Short GET_REQ: %d(%d)\n", msg_nob,
359 (int)(hdr_size + sizeof(msg->ibm_u.get)));
364 __swab64s(&msg->ibm_u.get.ibgm_rd.rd_addr);
365 __swab32s(&msg->ibm_u.get.ibgm_rd.rd_nob);
366 __swab32s(&msg->ibm_u.get.ibgm_rd.rd_key);
370 __swab32s(&msg->ibm_u.get.ibgm_rd.rd_key);
371 __swab32s(&msg->ibm_u.get.ibgm_rd.rd_nfrag);
374 n = msg->ibm_u.get.ibgm_rd.rd_nfrag;
375 if (n <= 0 || n > IBNAL_MAX_RDMA_FRAGS) {
376 CERROR("Bad GET_REQ nfrags: %d, should be 0 < n <= %d\n",
377 n, IBNAL_MAX_RDMA_FRAGS);
381 if (msg_nob < offsetof(kib_msg_t, ibm_u.get.ibgm_rd.rd_frags[n])) {
382 CERROR("Short GET_REQ: %d(%d)\n", msg_nob,
383 (int)offsetof(kib_msg_t, ibm_u.get.ibgm_rd.rd_frags[n]));
388 for (i = 0; i < msg->ibm_u.get.ibgm_rd.rd_nfrag; i++) {
389 __swab32s(&msg->ibm_u.get.ibgm_rd.rd_frags[i].rf_nob);
390 __swab32s(&msg->ibm_u.get.ibgm_rd.rd_frags[i].rf_addr_lo);
391 __swab32s(&msg->ibm_u.get.ibgm_rd.rd_frags[i].rf_addr_hi);
396 case IBNAL_MSG_PUT_NAK:
397 case IBNAL_MSG_PUT_DONE:
398 case IBNAL_MSG_GET_DONE:
399 if (msg_nob < hdr_size + sizeof(msg->ibm_u.completion)) {
400 CERROR("Short RDMA completion: %d(%d)\n", msg_nob,
401 (int)(hdr_size + sizeof(msg->ibm_u.completion)));
405 __swab32s(&msg->ibm_u.completion.ibcm_status);
408 case IBNAL_MSG_CONNREQ:
409 case IBNAL_MSG_CONNACK:
410 if (msg_nob < hdr_size + sizeof(msg->ibm_u.connparams)) {
411 CERROR("Short connreq/ack: %d(%d)\n", msg_nob,
412 (int)(hdr_size + sizeof(msg->ibm_u.connparams)));
416 __swab32s(&msg->ibm_u.connparams.ibcp_queue_depth);
417 __swab32s(&msg->ibm_u.connparams.ibcp_max_msg_size);
418 __swab32s(&msg->ibm_u.connparams.ibcp_max_frags);
426 kibnal_start_listener (lnet_ni_t *ni)
428 static cm_listen_data_t info;
432 LASSERT (kibnal_data.kib_listen_handle == NULL);
434 kibnal_data.kib_listen_handle =
435 cm_create_cep(cm_cep_transp_rc);
436 if (kibnal_data.kib_listen_handle == NULL) {
437 CERROR ("Can't create listen CEP\n");
441 CDEBUG(D_NET, "Created CEP %p for listening\n",
442 kibnal_data.kib_listen_handle);
444 memset(&info, 0, sizeof(info));
445 info.listen_addr.end_pt.sid =
446 (__u64)(*kibnal_tunables.kib_service_number);
448 cmrc = cm_listen(kibnal_data.kib_listen_handle, &info,
449 kibnal_listen_callback, NULL);
450 if (cmrc == cm_stat_success)
453 CERROR ("cm_listen error: %d\n", cmrc);
455 cmrc = cm_destroy_cep(kibnal_data.kib_listen_handle);
456 LASSERT (cmrc == cm_stat_success);
458 kibnal_data.kib_listen_handle = NULL;
463 kibnal_stop_listener(lnet_ni_t *ni)
467 LASSERT (kibnal_data.kib_listen_handle != NULL);
469 cmrc = cm_cancel(kibnal_data.kib_listen_handle);
470 if (cmrc != cm_stat_success)
471 CERROR ("Error %d stopping listener\n", cmrc);
473 cfs_pause(cfs_time_seconds(1)/10); /* ensure no more callbacks */
475 cmrc = cm_destroy_cep(kibnal_data.kib_listen_handle);
476 if (cmrc != vv_return_ok)
477 CERROR ("Error %d destroying CEP\n", cmrc);
479 kibnal_data.kib_listen_handle = NULL;
483 kibnal_create_peer (kib_peer_t **peerp, lnet_nid_t nid)
489 LASSERT (nid != LNET_NID_ANY);
491 LIBCFS_ALLOC(peer, sizeof (*peer));
493 CERROR("Cannot allocate peer\n");
497 memset(peer, 0, sizeof(*peer)); /* zero flags etc */
500 atomic_set (&peer->ibp_refcount, 1); /* 1 ref for caller */
502 INIT_LIST_HEAD (&peer->ibp_list); /* not in the peer table yet */
503 INIT_LIST_HEAD (&peer->ibp_conns);
504 INIT_LIST_HEAD (&peer->ibp_tx_queue);
507 peer->ibp_last_alive = cfs_time_current();
508 peer->ibp_reconnect_interval = 0; /* OK to connect at any time */
510 write_lock_irqsave(&kibnal_data.kib_global_lock, flags);
512 if (atomic_read(&kibnal_data.kib_npeers) >=
513 *kibnal_tunables.kib_concurrent_peers) {
514 rc = -EOVERFLOW; /* !! but at least it distinguishes */
515 } else if (kibnal_data.kib_listen_handle == NULL) {
516 rc = -ESHUTDOWN; /* shutdown has started */
519 /* npeers only grows with the global lock held */
520 atomic_inc(&kibnal_data.kib_npeers);
523 write_unlock_irqrestore(&kibnal_data.kib_global_lock, flags);
526 CERROR("Can't create peer: %s\n",
527 (rc == -ESHUTDOWN) ? "shutting down" :
529 LIBCFS_FREE(peer, sizeof(*peer));
538 kibnal_destroy_peer (kib_peer_t *peer)
540 LASSERT (atomic_read (&peer->ibp_refcount) == 0);
541 LASSERT (peer->ibp_persistence == 0);
542 LASSERT (!kibnal_peer_active(peer));
543 LASSERT (peer->ibp_connecting == 0);
544 LASSERT (peer->ibp_accepting == 0);
545 LASSERT (list_empty (&peer->ibp_conns));
546 LASSERT (list_empty (&peer->ibp_tx_queue));
548 LIBCFS_FREE (peer, sizeof (*peer));
550 /* NB a peer's connections keep a reference on their peer until
551 * they are destroyed, so we can be assured that _all_ state to do
552 * with this peer has been cleaned up when its refcount drops to
554 atomic_dec(&kibnal_data.kib_npeers);
558 kibnal_find_peer_locked (lnet_nid_t nid)
560 /* the caller is responsible for accounting the additional reference
561 * that this creates */
562 struct list_head *peer_list = kibnal_nid2peerlist (nid);
563 struct list_head *tmp;
566 list_for_each (tmp, peer_list) {
568 peer = list_entry (tmp, kib_peer_t, ibp_list);
570 LASSERT (peer->ibp_persistence != 0 || /* persistent peer */
571 peer->ibp_connecting != 0 || /* creating conns */
572 peer->ibp_accepting != 0 ||
573 !list_empty (&peer->ibp_conns)); /* active conn */
575 if (peer->ibp_nid != nid)
578 CDEBUG(D_NET, "got peer [%p] -> %s (%d)\n",
579 peer, libcfs_nid2str(nid),
580 atomic_read (&peer->ibp_refcount));
587 kibnal_unlink_peer_locked (kib_peer_t *peer)
589 LASSERT (peer->ibp_persistence == 0);
590 LASSERT (list_empty(&peer->ibp_conns));
592 LASSERT (kibnal_peer_active(peer));
593 list_del_init (&peer->ibp_list);
594 /* lose peerlist's ref */
595 kibnal_peer_decref(peer);
599 kibnal_get_peer_info (int index, lnet_nid_t *nidp, __u32 *ipp,
603 struct list_head *ptmp;
607 read_lock_irqsave(&kibnal_data.kib_global_lock, flags);
609 for (i = 0; i < kibnal_data.kib_peer_hash_size; i++) {
611 list_for_each (ptmp, &kibnal_data.kib_peers[i]) {
613 peer = list_entry (ptmp, kib_peer_t, ibp_list);
614 LASSERT (peer->ibp_persistence != 0 ||
615 peer->ibp_connecting != 0 ||
616 peer->ibp_accepting != 0 ||
617 !list_empty (&peer->ibp_conns));
622 *nidp = peer->ibp_nid;
624 *persistencep = peer->ibp_persistence;
626 read_unlock_irqrestore(&kibnal_data.kib_global_lock,
632 read_unlock_irqrestore(&kibnal_data.kib_global_lock, flags);
637 kibnal_add_persistent_peer (lnet_nid_t nid, __u32 ip)
644 CDEBUG(D_NET, "%s at %u.%u.%u.%u\n",
645 libcfs_nid2str(nid), HIPQUAD(ip));
647 if (nid == LNET_NID_ANY)
650 rc = kibnal_create_peer(&peer, nid);
654 write_lock_irqsave(&kibnal_data.kib_global_lock, flags);
656 peer2 = kibnal_find_peer_locked (nid);
658 kibnal_peer_decref (peer);
661 /* peer table takes existing ref on peer */
662 list_add_tail (&peer->ibp_list,
663 kibnal_nid2peerlist (nid));
667 peer->ibp_persistence++;
669 write_unlock_irqrestore(&kibnal_data.kib_global_lock, flags);
674 kibnal_del_peer_locked (kib_peer_t *peer)
676 struct list_head *ctmp;
677 struct list_head *cnxt;
680 peer->ibp_persistence = 0;
682 if (list_empty(&peer->ibp_conns)) {
683 kibnal_unlink_peer_locked(peer);
685 list_for_each_safe (ctmp, cnxt, &peer->ibp_conns) {
686 conn = list_entry(ctmp, kib_conn_t, ibc_list);
688 kibnal_close_conn_locked (conn, 0);
690 /* NB peer is no longer persistent; closing its last conn
693 /* NB peer now unlinked; might even be freed if the peer table had the
698 kibnal_del_peer (lnet_nid_t nid)
700 CFS_LIST_HEAD (zombies);
701 struct list_head *ptmp;
702 struct list_head *pnxt;
710 write_lock_irqsave(&kibnal_data.kib_global_lock, flags);
712 if (nid != LNET_NID_ANY)
713 lo = hi = kibnal_nid2peerlist(nid) - kibnal_data.kib_peers;
716 hi = kibnal_data.kib_peer_hash_size - 1;
719 for (i = lo; i <= hi; i++) {
720 list_for_each_safe (ptmp, pnxt, &kibnal_data.kib_peers[i]) {
721 peer = list_entry (ptmp, kib_peer_t, ibp_list);
722 LASSERT (peer->ibp_persistence != 0 ||
723 peer->ibp_connecting != 0 ||
724 peer->ibp_accepting != 0 ||
725 !list_empty (&peer->ibp_conns));
727 if (!(nid == LNET_NID_ANY || peer->ibp_nid == nid))
730 if (!list_empty(&peer->ibp_tx_queue)) {
731 LASSERT (list_empty(&peer->ibp_conns));
733 list_splice_init(&peer->ibp_tx_queue, &zombies);
736 kibnal_del_peer_locked (peer);
737 rc = 0; /* matched something */
741 write_unlock_irqrestore(&kibnal_data.kib_global_lock, flags);
743 kibnal_txlist_done(&zombies, -EIO);
749 kibnal_get_conn_by_idx (int index)
752 struct list_head *ptmp;
754 struct list_head *ctmp;
758 read_lock_irqsave(&kibnal_data.kib_global_lock, flags);
760 for (i = 0; i < kibnal_data.kib_peer_hash_size; i++) {
761 list_for_each (ptmp, &kibnal_data.kib_peers[i]) {
763 peer = list_entry (ptmp, kib_peer_t, ibp_list);
764 LASSERT (peer->ibp_persistence > 0 ||
765 peer->ibp_connecting != 0 ||
766 peer->ibp_accepting != 0 ||
767 !list_empty (&peer->ibp_conns));
769 list_for_each (ctmp, &peer->ibp_conns) {
773 conn = list_entry (ctmp, kib_conn_t, ibc_list);
774 kibnal_conn_addref(conn);
775 read_unlock_irqrestore(&kibnal_data.kib_global_lock,
782 read_unlock_irqrestore(&kibnal_data.kib_global_lock, flags);
787 kibnal_debug_rx (kib_rx_t *rx)
789 CDEBUG(D_CONSOLE, " %p nob %d msg_type %x "
790 "cred %d seq "LPD64"\n",
791 rx, rx->rx_nob, rx->rx_msg->ibm_type,
792 rx->rx_msg->ibm_credits, rx->rx_msg->ibm_seq);
796 kibnal_debug_tx (kib_tx_t *tx)
798 CDEBUG(D_CONSOLE, " %p snd %d q %d w %d rc %d dl %lx "
799 "cookie "LPX64" msg %s%s type %x cred %d seq "LPD64"\n",
800 tx, tx->tx_sending, tx->tx_queued, tx->tx_waiting,
801 tx->tx_status, tx->tx_deadline, tx->tx_cookie,
802 tx->tx_lntmsg[0] == NULL ? "-" : "!",
803 tx->tx_lntmsg[1] == NULL ? "-" : "!",
804 tx->tx_msg->ibm_type, tx->tx_msg->ibm_credits,
805 tx->tx_msg->ibm_seq);
809 kibnal_debug_conn (kib_conn_t *conn)
811 struct list_head *tmp;
814 spin_lock(&conn->ibc_lock);
816 CDEBUG(D_CONSOLE, "conn[%d] %p -> %s: \n",
817 atomic_read(&conn->ibc_refcount), conn,
818 libcfs_nid2str(conn->ibc_peer->ibp_nid));
819 CDEBUG(D_CONSOLE, " txseq "LPD64" rxseq "LPD64" state %d \n",
820 conn->ibc_txseq, conn->ibc_rxseq, conn->ibc_state);
821 CDEBUG(D_CONSOLE, " nposted %d cred %d o_cred %d r_cred %d\n",
822 conn->ibc_nsends_posted, conn->ibc_credits,
823 conn->ibc_outstanding_credits, conn->ibc_reserved_credits);
824 CDEBUG(D_CONSOLE, " disc %d comms_err %d\n",
825 conn->ibc_disconnect, conn->ibc_comms_error);
827 CDEBUG(D_CONSOLE, " early_rxs:\n");
828 list_for_each(tmp, &conn->ibc_early_rxs)
829 kibnal_debug_rx(list_entry(tmp, kib_rx_t, rx_list));
831 CDEBUG(D_CONSOLE, " tx_queue_nocred:\n");
832 list_for_each(tmp, &conn->ibc_tx_queue_nocred)
833 kibnal_debug_tx(list_entry(tmp, kib_tx_t, tx_list));
835 CDEBUG(D_CONSOLE, " tx_queue_rsrvd:\n");
836 list_for_each(tmp, &conn->ibc_tx_queue_rsrvd)
837 kibnal_debug_tx(list_entry(tmp, kib_tx_t, tx_list));
839 CDEBUG(D_CONSOLE, " tx_queue:\n");
840 list_for_each(tmp, &conn->ibc_tx_queue)
841 kibnal_debug_tx(list_entry(tmp, kib_tx_t, tx_list));
843 CDEBUG(D_CONSOLE, " active_txs:\n");
844 list_for_each(tmp, &conn->ibc_active_txs)
845 kibnal_debug_tx(list_entry(tmp, kib_tx_t, tx_list));
847 CDEBUG(D_CONSOLE, " rxs:\n");
848 for (i = 0; i < IBNAL_RX_MSGS; i++)
849 kibnal_debug_rx(&conn->ibc_rxs[i]);
851 spin_unlock(&conn->ibc_lock);
855 kibnal_set_qp_state (kib_conn_t *conn, vv_qp_state_t new_state)
857 static vv_qp_attr_t attr;
859 kib_connvars_t *cv = conn->ibc_connvars;
862 /* Only called by connd => static OK */
863 LASSERT (!in_interrupt());
864 LASSERT (current == kibnal_data.kib_connd);
866 memset(&attr, 0, sizeof(attr));
872 case vv_qp_state_init: {
873 struct vv_qp_modify_init_st *init = &attr.modify.params.init;
875 init->p_key_indx = cv->cv_pkey_index;
876 init->phy_port_num = cv->cv_port;
877 init->q_key = IBNAL_QKEY; /* XXX but VV_QP_AT_Q_KEY not set! */
878 init->access_control = vv_acc_r_mem_read |
879 vv_acc_r_mem_write; /* XXX vv_acc_l_mem_write ? */
881 attr.modify.vv_qp_attr_mask = VV_QP_AT_P_KEY_IX |
882 VV_QP_AT_PHY_PORT_NUM |
883 VV_QP_AT_ACCESS_CON_F;
886 case vv_qp_state_rtr: {
887 struct vv_qp_modify_rtr_st *rtr = &attr.modify.params.rtr;
888 vv_add_vec_t *av = &rtr->remote_add_vec;
890 av->dlid = cv->cv_path.dlid;
891 av->grh_flag = (!IBNAL_LOCAL_SUB);
892 av->max_static_rate = IBNAL_R_2_STATIC_RATE(cv->cv_path.rate);
893 av->service_level = cv->cv_path.sl;
894 av->source_path_bit = IBNAL_SOURCE_PATH_BIT;
895 av->pmtu = cv->cv_path.mtu;
896 av->rnr_retry_count = cv->cv_rnr_count;
897 av->global_dest.traffic_class = cv->cv_path.traffic_class;
898 av->global_dest.hope_limit = cv->cv_path.hop_limut;
899 av->global_dest.flow_lable = cv->cv_path.flow_label;
900 av->global_dest.s_gid_index = cv->cv_sgid_index;
901 // XXX other av fields zero?
903 rtr->destanation_qp = cv->cv_remote_qpn;
904 rtr->receive_psn = cv->cv_rxpsn;
905 rtr->responder_rdma_r_atom_num = IBNAL_OUS_DST_RD;
906 rtr->opt_min_rnr_nak_timer = *kibnal_tunables.kib_rnr_nak_timer;
909 // XXX sdp sets VV_QP_AT_OP_F but no actual optional options
910 attr.modify.vv_qp_attr_mask = VV_QP_AT_ADD_VEC |
913 VV_QP_AT_MIN_RNR_NAK_T |
914 VV_QP_AT_RESP_RDMA_ATOM_OUT_NUM |
918 case vv_qp_state_rts: {
919 struct vv_qp_modify_rts_st *rts = &attr.modify.params.rts;
921 rts->send_psn = cv->cv_txpsn;
922 rts->local_ack_timeout = *kibnal_tunables.kib_local_ack_timeout;
923 rts->retry_num = *kibnal_tunables.kib_retry_cnt;
924 rts->rnr_num = *kibnal_tunables.kib_rnr_cnt;
925 rts->dest_out_rdma_r_atom_num = IBNAL_OUS_DST_RD;
927 attr.modify.vv_qp_attr_mask = VV_QP_AT_S_PSN |
931 VV_QP_AT_DEST_RDMA_ATOM_OUT_NUM;
934 case vv_qp_state_error:
935 case vv_qp_state_reset:
936 attr.modify.vv_qp_attr_mask = 0;
940 attr.modify.qp_modify_into_state = new_state;
941 attr.modify.vv_qp_attr_mask |= VV_QP_AT_STATE;
943 vvrc = vv_qp_modify(kibnal_data.kib_hca, conn->ibc_qp, &attr, NULL);
944 if (vvrc != vv_return_ok) {
945 CERROR("Can't modify qp -> %s state to %d: %d\n",
946 libcfs_nid2str(conn->ibc_peer->ibp_nid),
955 kibnal_create_conn (cm_cep_handle_t cep)
964 static vv_qp_attr_t reqattr;
965 static vv_qp_attr_t rspattr;
967 /* Only the connd creates conns => single threaded */
968 LASSERT(!in_interrupt());
969 LASSERT(current == kibnal_data.kib_connd);
971 LIBCFS_ALLOC(conn, sizeof (*conn));
973 CERROR ("Can't allocate connection\n");
977 /* zero flags, NULL pointers etc... */
978 memset (conn, 0, sizeof (*conn));
980 conn->ibc_version = IBNAL_MSG_VERSION; /* Use latest version at first */
982 INIT_LIST_HEAD (&conn->ibc_early_rxs);
983 INIT_LIST_HEAD (&conn->ibc_tx_queue_nocred);
984 INIT_LIST_HEAD (&conn->ibc_tx_queue);
985 INIT_LIST_HEAD (&conn->ibc_tx_queue_rsrvd);
986 INIT_LIST_HEAD (&conn->ibc_active_txs);
987 spin_lock_init (&conn->ibc_lock);
989 atomic_inc (&kibnal_data.kib_nconns);
990 /* well not really, but I call destroy() on failure, which decrements */
994 LIBCFS_ALLOC(conn->ibc_connvars, sizeof(*conn->ibc_connvars));
995 if (conn->ibc_connvars == NULL) {
996 CERROR("Can't allocate in-progress connection state\n");
999 memset (conn->ibc_connvars, 0, sizeof(*conn->ibc_connvars));
1000 /* Random seed for QP sequence number */
1001 get_random_bytes(&conn->ibc_connvars->cv_rxpsn,
1002 sizeof(conn->ibc_connvars->cv_rxpsn));
1004 LIBCFS_ALLOC(conn->ibc_rxs, IBNAL_RX_MSGS * sizeof (kib_rx_t));
1005 if (conn->ibc_rxs == NULL) {
1006 CERROR("Cannot allocate RX buffers\n");
1009 memset (conn->ibc_rxs, 0, IBNAL_RX_MSGS * sizeof(kib_rx_t));
1011 rc = kibnal_alloc_pages(&conn->ibc_rx_pages, IBNAL_RX_MSG_PAGES, 1);
1015 for (i = ipage = page_offset = 0; i < IBNAL_RX_MSGS; i++) {
1016 struct page *page = conn->ibc_rx_pages->ibp_pages[ipage];
1017 kib_rx_t *rx = &conn->ibc_rxs[i];
1018 vv_mem_reg_h_t mem_h;
1022 rx->rx_msg = (kib_msg_t *)(((char *)page_address(page)) +
1025 vvrc = vv_get_gen_mr_attrib(kibnal_data.kib_hca,
1031 LASSERT (vvrc == vv_return_ok);
1033 CDEBUG(D_NET, "Rx[%d] %p->%p[%x]\n", i, rx,
1034 rx->rx_msg, rx->rx_lkey);
1036 page_offset += IBNAL_MSG_SIZE;
1037 LASSERT (page_offset <= PAGE_SIZE);
1039 if (page_offset == PAGE_SIZE) {
1042 LASSERT (ipage <= IBNAL_RX_MSG_PAGES);
1046 memset(&reqattr, 0, sizeof(reqattr));
1048 reqattr.create.qp_type = vv_qp_type_r_conn;
1049 reqattr.create.cq_send_h = kibnal_data.kib_cq;
1050 reqattr.create.cq_receive_h = kibnal_data.kib_cq;
1051 reqattr.create.send_max_outstand_wr = (1 + IBNAL_MAX_RDMA_FRAGS) *
1052 (*kibnal_tunables.kib_concurrent_sends);
1053 reqattr.create.receive_max_outstand_wr = IBNAL_RX_MSGS;
1054 reqattr.create.max_scatgat_per_send_wr = 1;
1055 reqattr.create.max_scatgat_per_receive_wr = 1;
1056 reqattr.create.signaling_type = vv_selectable_signaling;
1057 reqattr.create.pd_h = kibnal_data.kib_pd;
1058 reqattr.create.recv_solicited_events = vv_selectable_signaling; // vv_signal_all;
1060 vvrc = vv_qp_create(kibnal_data.kib_hca, &reqattr, NULL,
1061 &conn->ibc_qp, &rspattr);
1062 if (vvrc != vv_return_ok) {
1063 CERROR ("Failed to create queue pair: %d\n", vvrc);
1067 /* Mark QP created */
1068 conn->ibc_state = IBNAL_CONN_INIT_QP;
1069 conn->ibc_connvars->cv_local_qpn = rspattr.create_return.qp_num;
1071 if (rspattr.create_return.receive_max_outstand_wr <
1073 rspattr.create_return.send_max_outstand_wr <
1074 (1 + IBNAL_MAX_RDMA_FRAGS) * (*kibnal_tunables.kib_concurrent_sends)) {
1075 CERROR("Insufficient rx/tx work items: wanted %d/%d got %d/%d\n",
1077 (1 + IBNAL_MAX_RDMA_FRAGS) *
1078 (*kibnal_tunables.kib_concurrent_sends),
1079 rspattr.create_return.receive_max_outstand_wr,
1080 rspattr.create_return.send_max_outstand_wr);
1084 /* Mark init complete */
1085 conn->ibc_state = IBNAL_CONN_INIT;
1087 /* 1 ref for caller */
1088 atomic_set (&conn->ibc_refcount, 1);
1092 kibnal_destroy_conn (conn);
1097 kibnal_destroy_conn (kib_conn_t *conn)
1101 /* Only the connd does this (i.e. single threaded) */
1102 LASSERT (!in_interrupt());
1103 LASSERT (current == kibnal_data.kib_connd);
1105 CDEBUG (D_NET, "connection %p\n", conn);
1107 LASSERT (atomic_read (&conn->ibc_refcount) == 0);
1108 LASSERT (list_empty(&conn->ibc_early_rxs));
1109 LASSERT (list_empty(&conn->ibc_tx_queue));
1110 LASSERT (list_empty(&conn->ibc_tx_queue_rsrvd));
1111 LASSERT (list_empty(&conn->ibc_tx_queue_nocred));
1112 LASSERT (list_empty(&conn->ibc_active_txs));
1113 LASSERT (conn->ibc_nsends_posted == 0);
1115 switch (conn->ibc_state) {
1117 /* conn must be completely disengaged from the network */
1120 case IBNAL_CONN_DISCONNECTED:
1121 /* connvars should have been freed already */
1122 LASSERT (conn->ibc_connvars == NULL);
1125 case IBNAL_CONN_INIT:
1126 vvrc = cm_destroy_cep(conn->ibc_cep);
1127 LASSERT (vvrc == vv_return_ok);
1130 case IBNAL_CONN_INIT_QP:
1131 kibnal_set_qp_state(conn, vv_qp_state_reset);
1132 vvrc = vv_qp_destroy(kibnal_data.kib_hca, conn->ibc_qp);
1133 if (vvrc != vv_return_ok)
1134 CERROR("Can't destroy QP: %d\n", vvrc);
1137 case IBNAL_CONN_INIT_NOTHING:
1141 if (conn->ibc_rx_pages != NULL)
1142 kibnal_free_pages(conn->ibc_rx_pages);
1144 if (conn->ibc_rxs != NULL)
1145 LIBCFS_FREE(conn->ibc_rxs,
1146 IBNAL_RX_MSGS * sizeof(kib_rx_t));
1148 if (conn->ibc_connvars != NULL)
1149 LIBCFS_FREE(conn->ibc_connvars, sizeof(*conn->ibc_connvars));
1151 if (conn->ibc_peer != NULL)
1152 kibnal_peer_decref(conn->ibc_peer);
1154 LIBCFS_FREE(conn, sizeof (*conn));
1156 atomic_dec(&kibnal_data.kib_nconns);
1160 kibnal_close_peer_conns_locked (kib_peer_t *peer, int why)
1163 struct list_head *ctmp;
1164 struct list_head *cnxt;
1167 list_for_each_safe (ctmp, cnxt, &peer->ibp_conns) {
1168 conn = list_entry (ctmp, kib_conn_t, ibc_list);
1171 kibnal_close_conn_locked (conn, why);
1178 kibnal_close_stale_conns_locked (kib_peer_t *peer, __u64 incarnation)
1181 struct list_head *ctmp;
1182 struct list_head *cnxt;
1185 list_for_each_safe (ctmp, cnxt, &peer->ibp_conns) {
1186 conn = list_entry (ctmp, kib_conn_t, ibc_list);
1188 if (conn->ibc_incarnation == incarnation)
1191 CDEBUG(D_NET, "Closing stale conn -> %s incarnation:"LPX64"("LPX64")\n",
1192 libcfs_nid2str(peer->ibp_nid),
1193 conn->ibc_incarnation, incarnation);
1196 kibnal_close_conn_locked (conn, -ESTALE);
1203 kibnal_close_matching_conns (lnet_nid_t nid)
1206 struct list_head *ptmp;
1207 struct list_head *pnxt;
1211 unsigned long flags;
1214 write_lock_irqsave(&kibnal_data.kib_global_lock, flags);
1216 if (nid != LNET_NID_ANY)
1217 lo = hi = kibnal_nid2peerlist(nid) - kibnal_data.kib_peers;
1220 hi = kibnal_data.kib_peer_hash_size - 1;
1223 for (i = lo; i <= hi; i++) {
1224 list_for_each_safe (ptmp, pnxt, &kibnal_data.kib_peers[i]) {
1226 peer = list_entry (ptmp, kib_peer_t, ibp_list);
1227 LASSERT (peer->ibp_persistence != 0 ||
1228 peer->ibp_connecting != 0 ||
1229 peer->ibp_accepting != 0 ||
1230 !list_empty (&peer->ibp_conns));
1232 if (!(nid == LNET_NID_ANY || nid == peer->ibp_nid))
1235 count += kibnal_close_peer_conns_locked (peer, 0);
1239 write_unlock_irqrestore(&kibnal_data.kib_global_lock, flags);
1241 /* wildcards always succeed */
1242 if (nid == LNET_NID_ANY)
1245 return (count == 0 ? -ENOENT : 0);
1249 kibnal_ctl(lnet_ni_t *ni, unsigned int cmd, void *arg)
1251 struct libcfs_ioctl_data *data = arg;
1254 LASSERT (ni == kibnal_data.kib_ni);
1257 case IOC_LIBCFS_GET_PEER: {
1260 int share_count = 0;
1262 rc = kibnal_get_peer_info(data->ioc_count,
1263 &nid, &ip, &share_count);
1264 data->ioc_nid = nid;
1265 data->ioc_count = share_count;
1266 data->ioc_u32[0] = ip;
1267 data->ioc_u32[1] = *kibnal_tunables.kib_service_number; /* port */
1270 case IOC_LIBCFS_ADD_PEER: {
1271 rc = kibnal_add_persistent_peer (data->ioc_nid,
1272 data->ioc_u32[0]); /* IP */
1275 case IOC_LIBCFS_DEL_PEER: {
1276 rc = kibnal_del_peer (data->ioc_nid);
1279 case IOC_LIBCFS_GET_CONN: {
1280 kib_conn_t *conn = kibnal_get_conn_by_idx (data->ioc_count);
1285 // kibnal_debug_conn(conn);
1287 data->ioc_nid = conn->ibc_peer->ibp_nid;
1288 kibnal_conn_decref(conn);
1292 case IOC_LIBCFS_CLOSE_CONNECTION: {
1293 rc = kibnal_close_matching_conns (data->ioc_nid);
1296 case IOC_LIBCFS_REGISTER_MYNID: {
1297 if (ni->ni_nid == data->ioc_nid) {
1300 CERROR("obsolete IOC_LIBCFS_REGISTER_MYNID: %s(%s)\n",
1301 libcfs_nid2str(data->ioc_nid),
1302 libcfs_nid2str(ni->ni_nid));
1313 kibnal_free_pages (kib_pages_t *p)
1315 int npages = p->ibp_npages;
1318 for (i = 0; i < npages; i++)
1319 if (p->ibp_pages[i] != NULL)
1320 __free_page(p->ibp_pages[i]);
1322 LIBCFS_FREE (p, offsetof(kib_pages_t, ibp_pages[npages]));
1326 kibnal_alloc_pages (kib_pages_t **pp, int npages, int allow_write)
1331 LIBCFS_ALLOC(p, offsetof(kib_pages_t, ibp_pages[npages]));
1333 CERROR ("Can't allocate buffer %d\n", npages);
1337 memset (p, 0, offsetof(kib_pages_t, ibp_pages[npages]));
1338 p->ibp_npages = npages;
1340 for (i = 0; i < npages; i++) {
1341 p->ibp_pages[i] = alloc_page (GFP_KERNEL);
1342 if (p->ibp_pages[i] == NULL) {
1343 CERROR ("Can't allocate page %d of %d\n", i, npages);
1344 kibnal_free_pages(p);
1354 kibnal_alloc_tx_descs (void)
1358 LIBCFS_ALLOC (kibnal_data.kib_tx_descs,
1359 IBNAL_TX_MSGS() * sizeof(kib_tx_t));
1360 if (kibnal_data.kib_tx_descs == NULL)
1363 memset(kibnal_data.kib_tx_descs, 0,
1364 IBNAL_TX_MSGS() * sizeof(kib_tx_t));
1366 for (i = 0; i < IBNAL_TX_MSGS(); i++) {
1367 kib_tx_t *tx = &kibnal_data.kib_tx_descs[i];
1370 LIBCFS_ALLOC(tx->tx_pages, LNET_MAX_IOV *
1371 sizeof(*tx->tx_pages));
1372 if (tx->tx_pages == NULL)
1375 LIBCFS_ALLOC(tx->tx_wrq,
1376 (1 + IBNAL_MAX_RDMA_FRAGS) *
1377 sizeof(*tx->tx_wrq));
1378 if (tx->tx_wrq == NULL)
1381 LIBCFS_ALLOC(tx->tx_gl,
1382 (1 + IBNAL_MAX_RDMA_FRAGS) *
1383 sizeof(*tx->tx_gl));
1384 if (tx->tx_gl == NULL)
1387 LIBCFS_ALLOC(tx->tx_rd,
1388 offsetof(kib_rdma_desc_t,
1389 rd_frags[IBNAL_MAX_RDMA_FRAGS]));
1390 if (tx->tx_rd == NULL)
1399 kibnal_free_tx_descs (void)
1403 if (kibnal_data.kib_tx_descs == NULL)
1406 for (i = 0; i < IBNAL_TX_MSGS(); i++) {
1407 kib_tx_t *tx = &kibnal_data.kib_tx_descs[i];
1410 if (tx->tx_pages != NULL)
1411 LIBCFS_FREE(tx->tx_pages, LNET_MAX_IOV *
1412 sizeof(*tx->tx_pages));
1414 if (tx->tx_wrq != NULL)
1415 LIBCFS_FREE(tx->tx_wrq,
1416 (1 + IBNAL_MAX_RDMA_FRAGS) *
1417 sizeof(*tx->tx_wrq));
1419 if (tx->tx_gl != NULL)
1420 LIBCFS_FREE(tx->tx_gl,
1421 (1 + IBNAL_MAX_RDMA_FRAGS) *
1422 sizeof(*tx->tx_gl));
1424 if (tx->tx_rd != NULL)
1425 LIBCFS_FREE(tx->tx_rd,
1426 offsetof(kib_rdma_desc_t,
1427 rd_frags[IBNAL_MAX_RDMA_FRAGS]));
1431 LIBCFS_FREE(kibnal_data.kib_tx_descs,
1432 IBNAL_TX_MSGS() * sizeof(kib_tx_t));
1437 kibnal_free_fmrs (int n)
1443 for (i = 0; i < n; i++) {
1444 tx = &kibnal_data.kib_tx_descs[i];
1446 vvrc = vv_free_fmr(kibnal_data.kib_hca,
1447 tx->tx_md.md_fmrhandle);
1448 if (vvrc != vv_return_ok)
1449 CWARN("vv_free_fmr[%d]: %d\n", i, vvrc);
1455 kibnal_setup_tx_descs (void)
1458 int page_offset = 0;
1461 vv_mem_reg_h_t mem_h;
1470 /* pre-mapped messages are not bigger than 1 page */
1471 CLASSERT (IBNAL_MSG_SIZE <= PAGE_SIZE);
1473 /* No fancy arithmetic when we do the buffer calculations */
1474 CLASSERT (PAGE_SIZE % IBNAL_MSG_SIZE == 0);
1476 rc = kibnal_alloc_pages(&kibnal_data.kib_tx_pages,
1477 IBNAL_TX_MSG_PAGES(), 0);
1481 for (i = 0; i < IBNAL_TX_MSGS(); i++) {
1482 page = kibnal_data.kib_tx_pages->ibp_pages[ipage];
1483 tx = &kibnal_data.kib_tx_descs[i];
1486 memset(&fmr_props, 0, sizeof(fmr_props));
1487 fmr_props.pd_hndl = kibnal_data.kib_pd;
1488 fmr_props.acl = (vv_acc_r_mem_write |
1489 vv_acc_l_mem_write);
1490 fmr_props.max_pages = LNET_MAX_IOV;
1491 fmr_props.log2_page_sz = PAGE_SHIFT;
1492 fmr_props.max_outstanding_maps = *kibnal_tunables.kib_fmr_remaps;
1494 vvrc = vv_alloc_fmr(kibnal_data.kib_hca,
1496 &tx->tx_md.md_fmrhandle);
1497 if (vvrc != vv_return_ok) {
1498 CERROR("Can't allocate fmr %d: %d\n", i, vvrc);
1500 kibnal_free_fmrs(i);
1501 kibnal_free_pages (kibnal_data.kib_tx_pages);
1505 tx->tx_md.md_fmrcount = *kibnal_tunables.kib_fmr_remaps;
1506 tx->tx_md.md_active = 0;
1508 tx->tx_msg = (kib_msg_t *)(((char *)page_address(page)) +
1511 vvrc = vv_get_gen_mr_attrib(kibnal_data.kib_hca,
1517 LASSERT (vvrc == vv_return_ok);
1519 CDEBUG(D_NET, "Tx[%d] %p->%p[%x]\n", i, tx,
1520 tx->tx_msg, tx->tx_lkey);
1522 list_add (&tx->tx_list, &kibnal_data.kib_idle_txs);
1524 page_offset += IBNAL_MSG_SIZE;
1525 LASSERT (page_offset <= PAGE_SIZE);
1527 if (page_offset == PAGE_SIZE) {
1530 LASSERT (ipage <= IBNAL_TX_MSG_PAGES());
1538 kibnal_shutdown (lnet_ni_t *ni)
1543 LASSERT (ni == kibnal_data.kib_ni);
1544 LASSERT (ni->ni_data == &kibnal_data);
1546 CDEBUG(D_MALLOC, "before NAL cleanup: kmem %d\n",
1547 atomic_read (&libcfs_kmemory));
1549 switch (kibnal_data.kib_init) {
1551 case IBNAL_INIT_ALL:
1552 /* stop accepting connections and prevent new peers */
1553 kibnal_stop_listener(ni);
1555 /* nuke all existing peers */
1556 kibnal_del_peer(LNET_NID_ANY);
1558 /* Wait for all peer state to clean up */
1560 while (atomic_read(&kibnal_data.kib_npeers) != 0) {
1562 CDEBUG(((i & (-i)) == i) ? D_WARNING : D_NET, /* 2**n? */
1563 "waiting for %d peers to disconnect\n",
1564 atomic_read(&kibnal_data.kib_npeers));
1565 cfs_pause(cfs_time_seconds(1));
1570 vvrc = vv_cq_destroy(kibnal_data.kib_hca, kibnal_data.kib_cq);
1571 if (vvrc != vv_return_ok)
1572 CERROR ("Destroy CQ error: %d\n", vvrc);
1575 case IBNAL_INIT_TXD:
1576 kibnal_free_pages (kibnal_data.kib_tx_pages);
1578 kibnal_free_fmrs(IBNAL_TX_MSGS());
1584 /* Only deallocate a PD if we actually allocated one */
1585 vvrc = vv_pd_deallocate(kibnal_data.kib_hca,
1586 kibnal_data.kib_pd);
1587 if (vvrc != vv_return_ok)
1588 CERROR ("Destroy PD error: %d\n", vvrc);
1592 case IBNAL_INIT_ASYNC:
1593 vvrc = vv_dell_async_event_cb (kibnal_data.kib_hca,
1594 kibnal_async_callback);
1595 if (vvrc != vv_return_ok)
1596 CERROR("vv_dell_async_event_cb error: %d\n", vvrc);
1600 case IBNAL_INIT_HCA:
1601 vvrc = vv_hca_close(kibnal_data.kib_hca);
1602 if (vvrc != vv_return_ok)
1603 CERROR ("Close HCA error: %d\n", vvrc);
1606 case IBNAL_INIT_DATA:
1607 LASSERT (atomic_read(&kibnal_data.kib_npeers) == 0);
1608 LASSERT (kibnal_data.kib_peers != NULL);
1609 for (i = 0; i < kibnal_data.kib_peer_hash_size; i++) {
1610 LASSERT (list_empty (&kibnal_data.kib_peers[i]));
1612 LASSERT (atomic_read (&kibnal_data.kib_nconns) == 0);
1613 LASSERT (list_empty (&kibnal_data.kib_connd_zombies));
1614 LASSERT (list_empty (&kibnal_data.kib_connd_conns));
1615 LASSERT (list_empty (&kibnal_data.kib_connd_pcreqs));
1616 LASSERT (list_empty (&kibnal_data.kib_connd_peers));
1618 /* flag threads to terminate; wake and wait for them to die */
1619 kibnal_data.kib_shutdown = 1;
1620 wake_up_all (&kibnal_data.kib_sched_waitq);
1621 wake_up_all (&kibnal_data.kib_connd_waitq);
1624 while (atomic_read (&kibnal_data.kib_nthreads) != 0) {
1626 CDEBUG(((i & (-i)) == i) ? D_WARNING : D_NET, /* power of 2? */
1627 "Waiting for %d threads to terminate\n",
1628 atomic_read (&kibnal_data.kib_nthreads));
1629 cfs_pause(cfs_time_seconds(1));
1633 case IBNAL_INIT_NOTHING:
1637 kibnal_free_tx_descs();
1639 if (kibnal_data.kib_peers != NULL)
1640 LIBCFS_FREE (kibnal_data.kib_peers,
1641 sizeof (struct list_head) *
1642 kibnal_data.kib_peer_hash_size);
1644 CDEBUG(D_MALLOC, "after NAL cleanup: kmem %d\n",
1645 atomic_read (&libcfs_kmemory));
1647 kibnal_data.kib_init = IBNAL_INIT_NOTHING;
1648 PORTAL_MODULE_UNUSE;
1652 kibnal_startup (lnet_ni_t *ni)
1665 vv_request_event_record_t req_er;
1668 LASSERT (ni->ni_lnd == &the_kiblnd);
1670 /* Only 1 instance supported */
1671 if (kibnal_data.kib_init != IBNAL_INIT_NOTHING) {
1672 CERROR ("Only 1 instance supported\n");
1676 if (*kibnal_tunables.kib_credits > *kibnal_tunables.kib_ntx) {
1677 CERROR ("Can't set credits(%d) > ntx(%d)\n",
1678 *kibnal_tunables.kib_credits,
1679 *kibnal_tunables.kib_ntx);
1683 ni->ni_maxtxcredits = *kibnal_tunables.kib_credits;
1684 ni->ni_peertxcredits = *kibnal_tunables.kib_peercredits;
1686 CLASSERT (LNET_MAX_INTERFACES > 1);
1688 if (ni->ni_interfaces[0] != NULL) {
1689 /* Use the HCA specified in 'networks=' */
1691 if (ni->ni_interfaces[1] != NULL) {
1692 CERROR("Multiple interfaces not supported\n");
1696 /* Parse <hca base name><number> */
1697 hca_name = ni->ni_interfaces[0];
1698 nob = strlen(*kibnal_tunables.kib_hca_basename);
1700 if (strncmp(hca_name, *kibnal_tunables.kib_hca_basename, nob) ||
1701 sscanf(hca_name + nob, "%d%n", &devno, &nob) < 1) {
1702 CERROR("Unrecognised HCA %s\n", hca_name);
1707 /* Use <hca base name>0 */
1711 snprintf(hca_name, sizeof(scratch), "%s%d",
1712 *kibnal_tunables.kib_hca_basename, devno);
1713 if (strlen(hca_name) == sizeof(scratch) - 1) {
1714 CERROR("HCA name %s truncated\n", hca_name);
1719 /* Find IP address from <ipif base name><hca number> */
1720 snprintf(ipif_name, sizeof(ipif_name), "%s%d",
1721 *kibnal_tunables.kib_ipif_basename, devno);
1722 if (strlen(ipif_name) == sizeof(ipif_name) - 1) {
1723 CERROR("IPoIB interface name %s truncated\n", ipif_name);
1727 rc = libcfs_ipif_query(ipif_name, &up, &ip, &netmask);
1729 CERROR("Can't query IPoIB interface %s: %d\n", ipif_name, rc);
1734 CERROR("Can't query IPoIB interface %s: it's down\n", ipif_name);
1738 ni->ni_nid = LNET_MKNID(LNET_NIDNET(ni->ni_nid), ip);
1741 memset (&kibnal_data, 0, sizeof (kibnal_data)); /* zero pointers, flags etc */
1743 kibnal_data.kib_ni = ni;
1744 ni->ni_data = &kibnal_data;
1746 do_gettimeofday(&tv);
1747 kibnal_data.kib_incarnation = (((__u64)tv.tv_sec) * 1000000) + tv.tv_usec;
1749 rwlock_init(&kibnal_data.kib_global_lock);
1751 kibnal_data.kib_peer_hash_size = IBNAL_PEER_HASH_SIZE;
1752 LIBCFS_ALLOC (kibnal_data.kib_peers,
1753 sizeof (struct list_head) * kibnal_data.kib_peer_hash_size);
1754 if (kibnal_data.kib_peers == NULL) {
1757 for (i = 0; i < kibnal_data.kib_peer_hash_size; i++)
1758 INIT_LIST_HEAD(&kibnal_data.kib_peers[i]);
1760 spin_lock_init (&kibnal_data.kib_connd_lock);
1761 INIT_LIST_HEAD (&kibnal_data.kib_connd_peers);
1762 INIT_LIST_HEAD (&kibnal_data.kib_connd_pcreqs);
1763 INIT_LIST_HEAD (&kibnal_data.kib_connd_conns);
1764 INIT_LIST_HEAD (&kibnal_data.kib_connd_zombies);
1765 init_waitqueue_head (&kibnal_data.kib_connd_waitq);
1767 spin_lock_init (&kibnal_data.kib_sched_lock);
1768 init_waitqueue_head (&kibnal_data.kib_sched_waitq);
1770 spin_lock_init (&kibnal_data.kib_tx_lock);
1771 INIT_LIST_HEAD (&kibnal_data.kib_idle_txs);
1773 rc = kibnal_alloc_tx_descs();
1775 CERROR("Can't allocate tx descs\n");
1779 /* lists/ptrs/locks initialised */
1780 kibnal_data.kib_init = IBNAL_INIT_DATA;
1781 /*****************************************************/
1783 for (i = 0; i < IBNAL_N_SCHED; i++) {
1784 rc = kibnal_thread_start (kibnal_scheduler, (void *)((long)i));
1786 CERROR("Can't spawn vibnal scheduler[%d]: %d\n",
1792 rc = kibnal_thread_start (kibnal_connd, NULL);
1794 CERROR ("Can't spawn vibnal connd: %d\n", rc);
1798 vvrc = vv_hca_open(hca_name, NULL, &kibnal_data.kib_hca);
1799 if (vvrc != vv_return_ok) {
1800 CERROR ("Can't open HCA %s: %d\n", hca_name, vvrc);
1804 /* Channel Adapter opened */
1805 kibnal_data.kib_init = IBNAL_INIT_HCA;
1807 /* register to get HCA's asynchronous events. */
1808 req_er.req_event_type = VV_ASYNC_EVENT_ALL_MASK;
1809 vvrc = vv_set_async_event_cb (kibnal_data.kib_hca, req_er,
1810 kibnal_async_callback);
1811 if (vvrc != vv_return_ok) {
1812 CERROR ("Can't set HCA %s callback: %d\n", hca_name, vvrc);
1816 kibnal_data.kib_init = IBNAL_INIT_ASYNC;
1818 /*****************************************************/
1820 vvrc = vv_hca_query(kibnal_data.kib_hca, &kibnal_data.kib_hca_attrs);
1821 if (vvrc != vv_return_ok) {
1822 CERROR ("Can't size port attrs for %s: %d\n", hca_name, vvrc);
1826 kibnal_data.kib_port = -1;
1828 for (i = 0; i<kibnal_data.kib_hca_attrs.port_num; i++) {
1831 u_int32_t tbl_count;
1832 vv_port_attrib_t *pattr = &kibnal_data.kib_port_attr;
1834 vvrc = vv_port_query(kibnal_data.kib_hca, port_num, pattr);
1835 if (vvrc != vv_return_ok) {
1836 CERROR("vv_port_query failed for %s port %d: %d\n",
1837 hca_name, port_num, vvrc);
1841 switch (pattr->port_state) {
1842 case vv_state_linkDoun:
1843 CDEBUG(D_NET, "port[%d] Down\n", port_num);
1845 case vv_state_linkInit:
1846 CDEBUG(D_NET, "port[%d] Init\n", port_num);
1848 case vv_state_linkArm:
1849 CDEBUG(D_NET, "port[%d] Armed\n", port_num);
1851 case vv_state_linkActive:
1852 CDEBUG(D_NET, "port[%d] Active\n", port_num);
1854 /* Found a suitable port. Get its GUID and PKEY. */
1856 vvrc = vv_get_port_gid_tbl(kibnal_data.kib_hca,
1857 port_num, &tbl_count,
1858 &kibnal_data.kib_port_gid);
1859 if (vvrc != vv_return_ok) {
1860 CERROR("vv_get_port_gid_tbl failed "
1861 "for %s port %d: %d\n",
1862 hca_name, port_num, vvrc);
1867 vvrc = vv_get_port_partition_tbl(kibnal_data.kib_hca,
1868 port_num, &tbl_count,
1869 &kibnal_data.kib_port_pkey);
1870 if (vvrc != vv_return_ok) {
1871 CERROR("vv_get_port_partition_tbl failed "
1872 "for %s port %d: %d\n",
1873 hca_name, port_num, vvrc);
1877 kibnal_data.kib_port = port_num;
1880 case vv_state_linkActDefer: /* TODO: correct? */
1881 case vv_state_linkNoChange:
1882 CERROR("Unexpected %s port[%d] state %d\n",
1883 hca_name, i, pattr->port_state);
1889 if (kibnal_data.kib_port == -1) {
1890 CERROR ("Can't find an active port on %s\n", hca_name);
1894 CDEBUG(D_NET, "Using %s port %d - GID="LPX64":"LPX64"\n",
1895 hca_name, kibnal_data.kib_port,
1896 kibnal_data.kib_port_gid.scope.g.subnet,
1897 kibnal_data.kib_port_gid.scope.g.eui64);
1899 /*****************************************************/
1902 /* We use a pre-allocated PD */
1903 vvrc = vv_get_gen_pd_h(kibnal_data.kib_hca, &kibnal_data.kib_pd);
1905 vvrc = vv_pd_allocate(kibnal_data.kib_hca, &kibnal_data.kib_pd);
1907 if (vvrc != vv_return_ok) {
1908 CERROR ("Can't init PD: %d\n", vvrc);
1912 /* flag PD initialised */
1913 kibnal_data.kib_init = IBNAL_INIT_PD;
1914 /*****************************************************/
1916 rc = kibnal_setup_tx_descs();
1918 CERROR ("Can't register tx descs: %d\n", rc);
1922 /* flag TX descs initialised */
1923 kibnal_data.kib_init = IBNAL_INIT_TXD;
1924 /*****************************************************/
1929 vvrc = vv_cq_create(kibnal_data.kib_hca, IBNAL_CQ_ENTRIES(),
1932 &kibnal_data.kib_cq, &nentries);
1934 CERROR ("Can't create RX CQ: %d\n", vvrc);
1938 /* flag CQ initialised */
1939 kibnal_data.kib_init = IBNAL_INIT_CQ;
1941 if (nentries < IBNAL_CQ_ENTRIES()) {
1942 CERROR ("CQ only has %d entries, need %d\n",
1943 nentries, IBNAL_CQ_ENTRIES());
1947 vvrc = vv_request_completion_notification(kibnal_data.kib_hca,
1949 vv_next_solicit_unsolicit_event);
1951 CERROR ("Failed to re-arm completion queue: %d\n", rc);
1956 rc = kibnal_start_listener(ni);
1958 CERROR("Can't start listener: %d\n", rc);
1962 /* flag everything initialised */
1963 kibnal_data.kib_init = IBNAL_INIT_ALL;
1964 /*****************************************************/
1969 CDEBUG(D_NET, "kibnal_startup failed\n");
1970 kibnal_shutdown (ni);
1975 kibnal_module_fini (void)
1977 lnet_unregister_lnd(&the_kiblnd);
1978 kibnal_tunables_fini();
1982 kibnal_module_init (void)
1986 vibnal_assert_wire_constants();
1988 CLASSERT (offsetof(kib_msg_t, ibm_u) + sizeof(kib_connparams_t)
1989 <= cm_REQ_priv_data_len);
1990 CLASSERT (offsetof(kib_msg_t, ibm_u) + sizeof(kib_connparams_t)
1991 <= cm_REP_priv_data_len);
1992 CLASSERT (sizeof(kib_msg_t) <= IBNAL_MSG_SIZE);
1994 CLASSERT (offsetof(kib_msg_t, ibm_u.get.ibgm_rd.rd_frags[IBNAL_MAX_RDMA_FRAGS])
1996 CLASSERT (offsetof(kib_msg_t, ibm_u.putack.ibpam_rd.rd_frags[IBNAL_MAX_RDMA_FRAGS])
1999 rc = kibnal_tunables_init();
2003 lnet_register_lnd(&the_kiblnd);
2008 MODULE_AUTHOR("Cluster File Systems, Inc. <info@clusterfs.com>");
2009 MODULE_DESCRIPTION("Kernel Voltaire IB LND v1.00");
2010 MODULE_LICENSE("GPL");
2012 module_init(kibnal_module_init);
2013 module_exit(kibnal_module_fini);