1 /* -*- mode: c; c-basic-offset: 8; indent-tabs-mode: nil; -*-
2 * vim:expandtab:shiftwidth=8:tabstop=8:
4 * Copyright (C) 2004 Cluster File Systems, Inc.
5 * Author: Eric Barton <eric@bartonsoftware.com>
6 * Author: Frank Zago <fzago@systemfabricworks.com>
8 * This file is part of Lustre, http://www.lustre.org.
10 * Lustre is free software; you can redistribute it and/or
11 * modify it under the terms of version 2 of the GNU General Public
12 * License as published by the Free Software Foundation.
14 * Lustre is distributed in the hope that it will be useful,
15 * but WITHOUT ANY WARRANTY; without even the implied warranty of
16 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17 * GNU General Public License for more details.
19 * You should have received a copy of the GNU General Public License
20 * along with Lustre; if not, write to the Free Software
21 * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
29 .lnd_startup = kibnal_startup,
30 .lnd_shutdown = kibnal_shutdown,
31 .lnd_ctl = kibnal_ctl,
32 .lnd_send = kibnal_send,
33 .lnd_recv = kibnal_recv,
34 .lnd_eager_recv = kibnal_eager_recv,
37 kib_data_t kibnal_data;
39 void vibnal_assert_wire_constants (void)
41 /* Wire protocol assertions generated by 'wirecheck'
42 * running on Linux robert 2.6.11-1.27_FC3 #1 Tue May 17 20:27:37 EDT 2005 i686 athlon i386 G
43 * with gcc version 3.4.3 20050227 (Red Hat 3.4.3-22.fc3) */
47 CLASSERT (IBNAL_MSG_MAGIC == 0x0be91b91);
48 CLASSERT (IBNAL_MSG_VERSION == 0x11);
49 CLASSERT (IBNAL_MSG_CONNREQ == 0xc0);
50 CLASSERT (IBNAL_MSG_CONNACK == 0xc1);
51 CLASSERT (IBNAL_MSG_NOOP == 0xd0);
52 CLASSERT (IBNAL_MSG_IMMEDIATE == 0xd1);
53 CLASSERT (IBNAL_MSG_PUT_REQ == 0xd2);
54 CLASSERT (IBNAL_MSG_PUT_NAK == 0xd3);
55 CLASSERT (IBNAL_MSG_PUT_ACK == 0xd4);
56 CLASSERT (IBNAL_MSG_PUT_DONE == 0xd5);
57 CLASSERT (IBNAL_MSG_GET_REQ == 0xd6);
58 CLASSERT (IBNAL_MSG_GET_DONE == 0xd7);
60 /* Checks for struct kib_connparams_t */
61 CLASSERT ((int)sizeof(kib_connparams_t) == 12);
62 CLASSERT ((int)offsetof(kib_connparams_t, ibcp_queue_depth) == 0);
63 CLASSERT ((int)sizeof(((kib_connparams_t *)0)->ibcp_queue_depth) == 4);
64 CLASSERT ((int)offsetof(kib_connparams_t, ibcp_max_msg_size) == 4);
65 CLASSERT ((int)sizeof(((kib_connparams_t *)0)->ibcp_max_msg_size) == 4);
66 CLASSERT ((int)offsetof(kib_connparams_t, ibcp_max_frags) == 8);
67 CLASSERT ((int)sizeof(((kib_connparams_t *)0)->ibcp_max_frags) == 4);
69 /* Checks for struct kib_immediate_msg_t */
70 CLASSERT ((int)sizeof(kib_immediate_msg_t) == 72);
71 CLASSERT ((int)offsetof(kib_immediate_msg_t, ibim_hdr) == 0);
72 CLASSERT ((int)sizeof(((kib_immediate_msg_t *)0)->ibim_hdr) == 72);
73 CLASSERT ((int)offsetof(kib_immediate_msg_t, ibim_payload[13]) == 85);
74 CLASSERT ((int)sizeof(((kib_immediate_msg_t *)0)->ibim_payload[13]) == 1);
75 CLASSERT (IBNAL_USE_FMR == 1);
77 /* Checks for struct kib_rdma_desc_t */
78 CLASSERT ((int)sizeof(kib_rdma_desc_t) == 16);
79 CLASSERT ((int)offsetof(kib_rdma_desc_t, rd_addr) == 0);
80 CLASSERT ((int)sizeof(((kib_rdma_desc_t *)0)->rd_addr) == 8);
81 CLASSERT ((int)offsetof(kib_rdma_desc_t, rd_nob) == 8);
82 CLASSERT ((int)sizeof(((kib_rdma_desc_t *)0)->rd_nob) == 4);
83 CLASSERT ((int)offsetof(kib_rdma_desc_t, rd_key) == 12);
84 CLASSERT ((int)sizeof(((kib_rdma_desc_t *)0)->rd_key) == 4);
86 /* Checks for struct kib_putreq_msg_t */
87 CLASSERT ((int)sizeof(kib_putreq_msg_t) == 80);
88 CLASSERT ((int)offsetof(kib_putreq_msg_t, ibprm_hdr) == 0);
89 CLASSERT ((int)sizeof(((kib_putreq_msg_t *)0)->ibprm_hdr) == 72);
90 CLASSERT ((int)offsetof(kib_putreq_msg_t, ibprm_cookie) == 72);
91 CLASSERT ((int)sizeof(((kib_putreq_msg_t *)0)->ibprm_cookie) == 8);
93 /* Checks for struct kib_putack_msg_t */
94 CLASSERT ((int)sizeof(kib_putack_msg_t) == 32);
95 CLASSERT ((int)offsetof(kib_putack_msg_t, ibpam_src_cookie) == 0);
96 CLASSERT ((int)sizeof(((kib_putack_msg_t *)0)->ibpam_src_cookie) == 8);
97 CLASSERT ((int)offsetof(kib_putack_msg_t, ibpam_dst_cookie) == 8);
98 CLASSERT ((int)sizeof(((kib_putack_msg_t *)0)->ibpam_dst_cookie) == 8);
99 CLASSERT ((int)offsetof(kib_putack_msg_t, ibpam_rd) == 16);
100 CLASSERT ((int)sizeof(((kib_putack_msg_t *)0)->ibpam_rd) == 16);
102 /* Checks for struct kib_get_msg_t */
103 CLASSERT ((int)sizeof(kib_get_msg_t) == 96);
104 CLASSERT ((int)offsetof(kib_get_msg_t, ibgm_hdr) == 0);
105 CLASSERT ((int)sizeof(((kib_get_msg_t *)0)->ibgm_hdr) == 72);
106 CLASSERT ((int)offsetof(kib_get_msg_t, ibgm_cookie) == 72);
107 CLASSERT ((int)sizeof(((kib_get_msg_t *)0)->ibgm_cookie) == 8);
108 CLASSERT ((int)offsetof(kib_get_msg_t, ibgm_rd) == 80);
109 CLASSERT ((int)sizeof(((kib_get_msg_t *)0)->ibgm_rd) == 16);
111 /* Checks for struct kib_completion_msg_t */
112 CLASSERT ((int)sizeof(kib_completion_msg_t) == 12);
113 CLASSERT ((int)offsetof(kib_completion_msg_t, ibcm_cookie) == 0);
114 CLASSERT ((int)sizeof(((kib_completion_msg_t *)0)->ibcm_cookie) == 8);
115 CLASSERT ((int)offsetof(kib_completion_msg_t, ibcm_status) == 8);
116 CLASSERT ((int)sizeof(((kib_completion_msg_t *)0)->ibcm_status) == 4);
118 /* Checks for struct kib_msg_t */
119 CLASSERT ((int)sizeof(kib_msg_t) == 152);
120 CLASSERT ((int)offsetof(kib_msg_t, ibm_magic) == 0);
121 CLASSERT ((int)sizeof(((kib_msg_t *)0)->ibm_magic) == 4);
122 CLASSERT ((int)offsetof(kib_msg_t, ibm_version) == 4);
123 CLASSERT ((int)sizeof(((kib_msg_t *)0)->ibm_version) == 2);
124 CLASSERT ((int)offsetof(kib_msg_t, ibm_type) == 6);
125 CLASSERT ((int)sizeof(((kib_msg_t *)0)->ibm_type) == 1);
126 CLASSERT ((int)offsetof(kib_msg_t, ibm_credits) == 7);
127 CLASSERT ((int)sizeof(((kib_msg_t *)0)->ibm_credits) == 1);
128 CLASSERT ((int)offsetof(kib_msg_t, ibm_nob) == 8);
129 CLASSERT ((int)sizeof(((kib_msg_t *)0)->ibm_nob) == 4);
130 CLASSERT ((int)offsetof(kib_msg_t, ibm_cksum) == 12);
131 CLASSERT ((int)sizeof(((kib_msg_t *)0)->ibm_cksum) == 4);
132 CLASSERT ((int)offsetof(kib_msg_t, ibm_srcnid) == 16);
133 CLASSERT ((int)sizeof(((kib_msg_t *)0)->ibm_srcnid) == 8);
134 CLASSERT ((int)offsetof(kib_msg_t, ibm_srcstamp) == 24);
135 CLASSERT ((int)sizeof(((kib_msg_t *)0)->ibm_srcstamp) == 8);
136 CLASSERT ((int)offsetof(kib_msg_t, ibm_dstnid) == 32);
137 CLASSERT ((int)sizeof(((kib_msg_t *)0)->ibm_dstnid) == 8);
138 CLASSERT ((int)offsetof(kib_msg_t, ibm_dststamp) == 40);
139 CLASSERT ((int)sizeof(((kib_msg_t *)0)->ibm_dststamp) == 8);
140 CLASSERT ((int)offsetof(kib_msg_t, ibm_seq) == 48);
141 CLASSERT ((int)sizeof(((kib_msg_t *)0)->ibm_seq) == 8);
142 CLASSERT ((int)offsetof(kib_msg_t, ibm_u.connparams) == 56);
143 CLASSERT ((int)sizeof(((kib_msg_t *)0)->ibm_u.connparams) == 12);
144 CLASSERT ((int)offsetof(kib_msg_t, ibm_u.immediate) == 56);
145 CLASSERT ((int)sizeof(((kib_msg_t *)0)->ibm_u.immediate) == 72);
146 CLASSERT ((int)offsetof(kib_msg_t, ibm_u.putreq) == 56);
147 CLASSERT ((int)sizeof(((kib_msg_t *)0)->ibm_u.putreq) == 80);
148 CLASSERT ((int)offsetof(kib_msg_t, ibm_u.putack) == 56);
149 CLASSERT ((int)sizeof(((kib_msg_t *)0)->ibm_u.putack) == 32);
150 CLASSERT ((int)offsetof(kib_msg_t, ibm_u.get) == 56);
151 CLASSERT ((int)sizeof(((kib_msg_t *)0)->ibm_u.get) == 96);
152 CLASSERT ((int)offsetof(kib_msg_t, ibm_u.completion) == 56);
153 CLASSERT ((int)sizeof(((kib_msg_t *)0)->ibm_u.completion) == 12);
157 kibnal_cksum (void *ptr, int nob)
163 sum = ((sum << 1) | (sum >> 31)) + *c++;
165 /* ensure I don't return 0 (== no checksum) */
166 return (sum == 0) ? 1 : sum;
170 kibnal_init_msg(kib_msg_t *msg, int type, int body_nob)
172 msg->ibm_type = type;
173 msg->ibm_nob = offsetof(kib_msg_t, ibm_u) + body_nob;
177 kibnal_pack_msg(kib_msg_t *msg, __u32 version, int credits,
178 lnet_nid_t dstnid, __u64 dststamp, __u64 seq)
180 /* CAVEAT EMPTOR! all message fields not set here should have been
181 * initialised previously. */
182 msg->ibm_magic = IBNAL_MSG_MAGIC;
183 msg->ibm_version = version;
185 msg->ibm_credits = credits;
188 msg->ibm_srcnid = lnet_ptlcompat_srcnid(kibnal_data.kib_ni->ni_nid,
190 msg->ibm_srcstamp = kibnal_data.kib_incarnation;
191 msg->ibm_dstnid = dstnid;
192 msg->ibm_dststamp = dststamp;
195 if (*kibnal_tunables.kib_cksum) {
196 /* NB ibm_cksum zero while computing cksum */
197 msg->ibm_cksum = kibnal_cksum(msg, msg->ibm_nob);
202 kibnal_unpack_msg(kib_msg_t *msg, __u32 expected_version, int nob)
204 const int hdr_size = offsetof(kib_msg_t, ibm_u);
213 /* 6 bytes are enough to have received magic + version */
215 CERROR("Short message: %d\n", nob);
219 /* Future protocol version compatibility support!
220 * If the viblnd-specific protocol changes, or when LNET unifies
221 * protocols over all LNDs, the initial connection will negotiate a
222 * protocol version. If I find this, I avoid any console errors. If
223 * my is doing connection establishment, the reject will tell the peer
224 * which version I'm running. */
226 if (msg->ibm_magic == IBNAL_MSG_MAGIC) {
228 } else if (msg->ibm_magic == __swab32(IBNAL_MSG_MAGIC)) {
231 if (msg->ibm_magic == LNET_PROTO_MAGIC ||
232 msg->ibm_magic == __swab32(LNET_PROTO_MAGIC))
235 /* Completely out to lunch */
236 CERROR("Bad magic: %08x\n", msg->ibm_magic);
240 msg_version = flip ? __swab16(msg->ibm_version) : msg->ibm_version;
241 if (expected_version == 0) {
242 if (msg_version != IBNAL_MSG_VERSION_RDMAREPLYNOTRSRVD &&
243 msg_version != IBNAL_MSG_VERSION)
245 } else if (msg_version != expected_version) {
246 CERROR("Bad version: %x(%x expected)\n",
247 msg_version, expected_version);
251 if (nob < hdr_size) {
252 CERROR("Short message: %d\n", nob);
256 msg_nob = flip ? __swab32(msg->ibm_nob) : msg->ibm_nob;
258 CERROR("Short message: got %d, wanted %d\n", nob, msg_nob);
262 /* checksum must be computed with ibm_cksum zero and BEFORE anything
264 msg_cksum = flip ? __swab32(msg->ibm_cksum) : msg->ibm_cksum;
266 if (msg_cksum != 0 &&
267 msg_cksum != kibnal_cksum(msg, msg_nob)) {
268 CERROR("Bad checksum\n");
271 msg->ibm_cksum = msg_cksum;
274 /* leave magic unflipped as a clue to peer endianness */
275 msg->ibm_version = msg_version;
276 CLASSERT (sizeof(msg->ibm_type) == 1);
277 CLASSERT (sizeof(msg->ibm_credits) == 1);
278 msg->ibm_nob = msg_nob;
279 __swab64s(&msg->ibm_srcnid);
280 __swab64s(&msg->ibm_srcstamp);
281 __swab64s(&msg->ibm_dstnid);
282 __swab64s(&msg->ibm_dststamp);
283 __swab64s(&msg->ibm_seq);
286 if (msg->ibm_srcnid == LNET_NID_ANY) {
287 CERROR("Bad src nid: %s\n", libcfs_nid2str(msg->ibm_srcnid));
291 switch (msg->ibm_type) {
293 CERROR("Unknown message type %x\n", msg->ibm_type);
299 case IBNAL_MSG_IMMEDIATE:
300 if (msg_nob < offsetof(kib_msg_t, ibm_u.immediate.ibim_payload[0])) {
301 CERROR("Short IMMEDIATE: %d(%d)\n", msg_nob,
302 (int)offsetof(kib_msg_t, ibm_u.immediate.ibim_payload[0]));
307 case IBNAL_MSG_PUT_REQ:
308 if (msg_nob < hdr_size + sizeof(msg->ibm_u.putreq)) {
309 CERROR("Short PUT_REQ: %d(%d)\n", msg_nob,
310 (int)(hdr_size + sizeof(msg->ibm_u.putreq)));
315 case IBNAL_MSG_PUT_ACK:
316 if (msg_nob < hdr_size + sizeof(msg->ibm_u.putack)) {
317 CERROR("Short PUT_ACK: %d(%d)\n", msg_nob,
318 (int)(hdr_size + sizeof(msg->ibm_u.putack)));
323 __swab64s(&msg->ibm_u.putack.ibpam_rd.rd_addr);
324 __swab32s(&msg->ibm_u.putack.ibpam_rd.rd_nob);
325 __swab32s(&msg->ibm_u.putack.ibpam_rd.rd_key);
329 __swab32s(&msg->ibm_u.putack.ibpam_rd.rd_key);
330 __swab32s(&msg->ibm_u.putack.ibpam_rd.rd_nfrag);
333 n = msg->ibm_u.putack.ibpam_rd.rd_nfrag;
334 if (n <= 0 || n > IBNAL_MAX_RDMA_FRAGS) {
335 CERROR("Bad PUT_ACK nfrags: %d, should be 0 < n <= %d\n",
336 n, IBNAL_MAX_RDMA_FRAGS);
340 if (msg_nob < offsetof(kib_msg_t, ibm_u.putack.ibpam_rd.rd_frags[n])) {
341 CERROR("Short PUT_ACK: %d(%d)\n", msg_nob,
342 (int)offsetof(kib_msg_t, ibm_u.putack.ibpam_rd.rd_frags[n]));
347 for (i = 0; i < n; i++) {
348 __swab32s(&msg->ibm_u.putack.ibpam_rd.rd_frags[i].rf_nob);
349 __swab32s(&msg->ibm_u.putack.ibpam_rd.rd_frags[i].rf_addr_lo);
350 __swab32s(&msg->ibm_u.putack.ibpam_rd.rd_frags[i].rf_addr_hi);
356 case IBNAL_MSG_GET_REQ:
357 if (msg_nob < hdr_size + sizeof(msg->ibm_u.get)) {
358 CERROR("Short GET_REQ: %d(%d)\n", msg_nob,
359 (int)(hdr_size + sizeof(msg->ibm_u.get)));
364 __swab64s(&msg->ibm_u.get.ibgm_rd.rd_addr);
365 __swab32s(&msg->ibm_u.get.ibgm_rd.rd_nob);
366 __swab32s(&msg->ibm_u.get.ibgm_rd.rd_key);
370 __swab32s(&msg->ibm_u.get.ibgm_rd.rd_key);
371 __swab32s(&msg->ibm_u.get.ibgm_rd.rd_nfrag);
374 n = msg->ibm_u.get.ibgm_rd.rd_nfrag;
375 if (n <= 0 || n > IBNAL_MAX_RDMA_FRAGS) {
376 CERROR("Bad GET_REQ nfrags: %d, should be 0 < n <= %d\n",
377 n, IBNAL_MAX_RDMA_FRAGS);
381 if (msg_nob < offsetof(kib_msg_t, ibm_u.get.ibgm_rd.rd_frags[n])) {
382 CERROR("Short GET_REQ: %d(%d)\n", msg_nob,
383 (int)offsetof(kib_msg_t, ibm_u.get.ibgm_rd.rd_frags[n]));
388 for (i = 0; i < msg->ibm_u.get.ibgm_rd.rd_nfrag; i++) {
389 __swab32s(&msg->ibm_u.get.ibgm_rd.rd_frags[i].rf_nob);
390 __swab32s(&msg->ibm_u.get.ibgm_rd.rd_frags[i].rf_addr_lo);
391 __swab32s(&msg->ibm_u.get.ibgm_rd.rd_frags[i].rf_addr_hi);
396 case IBNAL_MSG_PUT_NAK:
397 case IBNAL_MSG_PUT_DONE:
398 case IBNAL_MSG_GET_DONE:
399 if (msg_nob < hdr_size + sizeof(msg->ibm_u.completion)) {
400 CERROR("Short RDMA completion: %d(%d)\n", msg_nob,
401 (int)(hdr_size + sizeof(msg->ibm_u.completion)));
405 __swab32s(&msg->ibm_u.completion.ibcm_status);
408 case IBNAL_MSG_CONNREQ:
409 case IBNAL_MSG_CONNACK:
410 if (msg_nob < hdr_size + sizeof(msg->ibm_u.connparams)) {
411 CERROR("Short connreq/ack: %d(%d)\n", msg_nob,
412 (int)(hdr_size + sizeof(msg->ibm_u.connparams)));
416 __swab32s(&msg->ibm_u.connparams.ibcp_queue_depth);
417 __swab32s(&msg->ibm_u.connparams.ibcp_max_msg_size);
418 __swab32s(&msg->ibm_u.connparams.ibcp_max_frags);
426 kibnal_start_listener (lnet_ni_t *ni)
428 static cm_listen_data_t info;
432 LASSERT (kibnal_data.kib_listen_handle == NULL);
434 kibnal_data.kib_listen_handle =
435 cm_create_cep(cm_cep_transp_rc);
436 if (kibnal_data.kib_listen_handle == NULL) {
437 CERROR ("Can't create listen CEP\n");
441 CDEBUG(D_NET, "Created CEP %p for listening\n",
442 kibnal_data.kib_listen_handle);
444 memset(&info, 0, sizeof(info));
445 info.listen_addr.end_pt.sid =
446 (__u64)(*kibnal_tunables.kib_service_number);
448 cmrc = cm_listen(kibnal_data.kib_listen_handle, &info,
449 kibnal_listen_callback, NULL);
450 if (cmrc == cm_stat_success)
453 CERROR ("cm_listen error: %d\n", cmrc);
455 cmrc = cm_destroy_cep(kibnal_data.kib_listen_handle);
456 LASSERT (cmrc == cm_stat_success);
458 kibnal_data.kib_listen_handle = NULL;
463 kibnal_stop_listener(lnet_ni_t *ni)
467 LASSERT (kibnal_data.kib_listen_handle != NULL);
469 cmrc = cm_cancel(kibnal_data.kib_listen_handle);
470 if (cmrc != cm_stat_success)
471 CERROR ("Error %d stopping listener\n", cmrc);
473 cfs_pause(cfs_time_seconds(1)/10); /* ensure no more callbacks */
475 cmrc = cm_destroy_cep(kibnal_data.kib_listen_handle);
476 if (cmrc != vv_return_ok)
477 CERROR ("Error %d destroying CEP\n", cmrc);
479 kibnal_data.kib_listen_handle = NULL;
483 kibnal_create_peer (kib_peer_t **peerp, lnet_nid_t nid)
489 LASSERT (nid != LNET_NID_ANY);
491 LIBCFS_ALLOC(peer, sizeof (*peer));
493 CERROR("Cannot allocate peer\n");
497 memset(peer, 0, sizeof(*peer)); /* zero flags etc */
500 atomic_set (&peer->ibp_refcount, 1); /* 1 ref for caller */
502 INIT_LIST_HEAD (&peer->ibp_list); /* not in the peer table yet */
503 INIT_LIST_HEAD (&peer->ibp_conns);
504 INIT_LIST_HEAD (&peer->ibp_tx_queue);
507 peer->ibp_last_alive = cfs_time_current();
508 peer->ibp_reconnect_interval = 0; /* OK to connect at any time */
510 write_lock_irqsave(&kibnal_data.kib_global_lock, flags);
512 if (atomic_read(&kibnal_data.kib_npeers) >=
513 *kibnal_tunables.kib_concurrent_peers) {
514 rc = -EOVERFLOW; /* !! but at least it distinguishes */
515 } else if (kibnal_data.kib_listen_handle == NULL) {
516 rc = -ESHUTDOWN; /* shutdown has started */
519 /* npeers only grows with the global lock held */
520 atomic_inc(&kibnal_data.kib_npeers);
523 write_unlock_irqrestore(&kibnal_data.kib_global_lock, flags);
526 CERROR("Can't create peer: %s\n",
527 (rc == -ESHUTDOWN) ? "shutting down" :
529 LIBCFS_FREE(peer, sizeof(*peer));
538 kibnal_destroy_peer (kib_peer_t *peer)
540 LASSERT (atomic_read (&peer->ibp_refcount) == 0);
541 LASSERT (peer->ibp_persistence == 0);
542 LASSERT (!kibnal_peer_active(peer));
543 LASSERT (peer->ibp_connecting == 0);
544 LASSERT (peer->ibp_accepting == 0);
545 LASSERT (list_empty (&peer->ibp_conns));
546 LASSERT (list_empty (&peer->ibp_tx_queue));
548 LIBCFS_FREE (peer, sizeof (*peer));
550 /* NB a peer's connections keep a reference on their peer until
551 * they are destroyed, so we can be assured that _all_ state to do
552 * with this peer has been cleaned up when its refcount drops to
554 atomic_dec(&kibnal_data.kib_npeers);
558 kibnal_find_peer_locked (lnet_nid_t nid)
560 /* the caller is responsible for accounting the additional reference
561 * that this creates */
562 struct list_head *peer_list = kibnal_nid2peerlist (nid);
563 struct list_head *tmp;
566 list_for_each (tmp, peer_list) {
568 peer = list_entry (tmp, kib_peer_t, ibp_list);
570 LASSERT (peer->ibp_persistence != 0 || /* persistent peer */
571 peer->ibp_connecting != 0 || /* creating conns */
572 peer->ibp_accepting != 0 ||
573 !list_empty (&peer->ibp_conns)); /* active conn */
575 if (peer->ibp_nid != nid)
578 CDEBUG(D_NET, "got peer [%p] -> %s (%d)\n",
579 peer, libcfs_nid2str(nid),
580 atomic_read (&peer->ibp_refcount));
587 kibnal_unlink_peer_locked (kib_peer_t *peer)
589 LASSERT (peer->ibp_persistence == 0);
590 LASSERT (list_empty(&peer->ibp_conns));
592 LASSERT (kibnal_peer_active(peer));
593 list_del_init (&peer->ibp_list);
594 /* lose peerlist's ref */
595 kibnal_peer_decref(peer);
599 kibnal_get_peer_info (int index, lnet_nid_t *nidp, __u32 *ipp,
603 struct list_head *ptmp;
607 read_lock_irqsave(&kibnal_data.kib_global_lock, flags);
609 for (i = 0; i < kibnal_data.kib_peer_hash_size; i++) {
611 list_for_each (ptmp, &kibnal_data.kib_peers[i]) {
613 peer = list_entry (ptmp, kib_peer_t, ibp_list);
614 LASSERT (peer->ibp_persistence != 0 ||
615 peer->ibp_connecting != 0 ||
616 peer->ibp_accepting != 0 ||
617 !list_empty (&peer->ibp_conns));
622 *nidp = peer->ibp_nid;
624 *persistencep = peer->ibp_persistence;
626 read_unlock_irqrestore(&kibnal_data.kib_global_lock,
632 read_unlock_irqrestore(&kibnal_data.kib_global_lock, flags);
637 kibnal_add_persistent_peer (lnet_nid_t nid, __u32 ip)
644 CDEBUG(D_NET, "%s at %u.%u.%u.%u\n",
645 libcfs_nid2str(nid), HIPQUAD(ip));
647 if (nid == LNET_NID_ANY)
650 rc = kibnal_create_peer(&peer, nid);
654 write_lock_irqsave(&kibnal_data.kib_global_lock, flags);
656 /* I'm always called with a reference on kibnal_data.kib_ni
657 * so shutdown can't have started */
658 LASSERT (kibnal_data.kib_listen_handle != NULL);
660 peer2 = kibnal_find_peer_locked (nid);
662 kibnal_peer_decref (peer);
665 /* peer table takes existing ref on peer */
666 list_add_tail (&peer->ibp_list,
667 kibnal_nid2peerlist (nid));
671 peer->ibp_persistence++;
673 write_unlock_irqrestore(&kibnal_data.kib_global_lock, flags);
678 kibnal_del_peer_locked (kib_peer_t *peer)
680 struct list_head *ctmp;
681 struct list_head *cnxt;
684 peer->ibp_persistence = 0;
686 if (list_empty(&peer->ibp_conns)) {
687 kibnal_unlink_peer_locked(peer);
689 list_for_each_safe (ctmp, cnxt, &peer->ibp_conns) {
690 conn = list_entry(ctmp, kib_conn_t, ibc_list);
692 kibnal_close_conn_locked (conn, 0);
694 /* NB peer is no longer persistent; closing its last conn
697 /* NB peer now unlinked; might even be freed if the peer table had the
702 kibnal_del_peer (lnet_nid_t nid)
704 CFS_LIST_HEAD (zombies);
705 struct list_head *ptmp;
706 struct list_head *pnxt;
714 write_lock_irqsave(&kibnal_data.kib_global_lock, flags);
716 if (nid != LNET_NID_ANY)
717 lo = hi = kibnal_nid2peerlist(nid) - kibnal_data.kib_peers;
720 hi = kibnal_data.kib_peer_hash_size - 1;
723 for (i = lo; i <= hi; i++) {
724 list_for_each_safe (ptmp, pnxt, &kibnal_data.kib_peers[i]) {
725 peer = list_entry (ptmp, kib_peer_t, ibp_list);
726 LASSERT (peer->ibp_persistence != 0 ||
727 peer->ibp_connecting != 0 ||
728 peer->ibp_accepting != 0 ||
729 !list_empty (&peer->ibp_conns));
731 if (!(nid == LNET_NID_ANY || peer->ibp_nid == nid))
734 if (!list_empty(&peer->ibp_tx_queue)) {
735 LASSERT (list_empty(&peer->ibp_conns));
737 list_splice_init(&peer->ibp_tx_queue, &zombies);
740 kibnal_del_peer_locked (peer);
741 rc = 0; /* matched something */
745 write_unlock_irqrestore(&kibnal_data.kib_global_lock, flags);
747 kibnal_txlist_done(&zombies, -EIO);
753 kibnal_get_conn_by_idx (int index)
756 struct list_head *ptmp;
758 struct list_head *ctmp;
762 read_lock_irqsave(&kibnal_data.kib_global_lock, flags);
764 for (i = 0; i < kibnal_data.kib_peer_hash_size; i++) {
765 list_for_each (ptmp, &kibnal_data.kib_peers[i]) {
767 peer = list_entry (ptmp, kib_peer_t, ibp_list);
768 LASSERT (peer->ibp_persistence > 0 ||
769 peer->ibp_connecting != 0 ||
770 peer->ibp_accepting != 0 ||
771 !list_empty (&peer->ibp_conns));
773 list_for_each (ctmp, &peer->ibp_conns) {
777 conn = list_entry (ctmp, kib_conn_t, ibc_list);
778 kibnal_conn_addref(conn);
779 read_unlock_irqrestore(&kibnal_data.kib_global_lock,
786 read_unlock_irqrestore(&kibnal_data.kib_global_lock, flags);
791 kibnal_debug_rx (kib_rx_t *rx)
793 CDEBUG(D_CONSOLE, " %p nob %d msg_type %x "
794 "cred %d seq "LPD64"\n",
795 rx, rx->rx_nob, rx->rx_msg->ibm_type,
796 rx->rx_msg->ibm_credits, rx->rx_msg->ibm_seq);
800 kibnal_debug_tx (kib_tx_t *tx)
802 CDEBUG(D_CONSOLE, " %p snd %d q %d w %d rc %d dl %lx "
803 "cookie "LPX64" msg %s%s type %x cred %d seq "LPD64"\n",
804 tx, tx->tx_sending, tx->tx_queued, tx->tx_waiting,
805 tx->tx_status, tx->tx_deadline, tx->tx_cookie,
806 tx->tx_lntmsg[0] == NULL ? "-" : "!",
807 tx->tx_lntmsg[1] == NULL ? "-" : "!",
808 tx->tx_msg->ibm_type, tx->tx_msg->ibm_credits,
809 tx->tx_msg->ibm_seq);
813 kibnal_debug_conn (kib_conn_t *conn)
815 struct list_head *tmp;
818 spin_lock(&conn->ibc_lock);
820 CDEBUG(D_CONSOLE, "conn[%d] %p -> %s: \n",
821 atomic_read(&conn->ibc_refcount), conn,
822 libcfs_nid2str(conn->ibc_peer->ibp_nid));
823 CDEBUG(D_CONSOLE, " txseq "LPD64" rxseq "LPD64" state %d \n",
824 conn->ibc_txseq, conn->ibc_rxseq, conn->ibc_state);
825 CDEBUG(D_CONSOLE, " nposted %d cred %d o_cred %d r_cred %d\n",
826 conn->ibc_nsends_posted, conn->ibc_credits,
827 conn->ibc_outstanding_credits, conn->ibc_reserved_credits);
828 CDEBUG(D_CONSOLE, " disc %d comms_err %d\n",
829 conn->ibc_disconnect, conn->ibc_comms_error);
831 CDEBUG(D_CONSOLE, " early_rxs:\n");
832 list_for_each(tmp, &conn->ibc_early_rxs)
833 kibnal_debug_rx(list_entry(tmp, kib_rx_t, rx_list));
835 CDEBUG(D_CONSOLE, " tx_queue_nocred:\n");
836 list_for_each(tmp, &conn->ibc_tx_queue_nocred)
837 kibnal_debug_tx(list_entry(tmp, kib_tx_t, tx_list));
839 CDEBUG(D_CONSOLE, " tx_queue_rsrvd:\n");
840 list_for_each(tmp, &conn->ibc_tx_queue_rsrvd)
841 kibnal_debug_tx(list_entry(tmp, kib_tx_t, tx_list));
843 CDEBUG(D_CONSOLE, " tx_queue:\n");
844 list_for_each(tmp, &conn->ibc_tx_queue)
845 kibnal_debug_tx(list_entry(tmp, kib_tx_t, tx_list));
847 CDEBUG(D_CONSOLE, " active_txs:\n");
848 list_for_each(tmp, &conn->ibc_active_txs)
849 kibnal_debug_tx(list_entry(tmp, kib_tx_t, tx_list));
851 CDEBUG(D_CONSOLE, " rxs:\n");
852 for (i = 0; i < IBNAL_RX_MSGS; i++)
853 kibnal_debug_rx(&conn->ibc_rxs[i]);
855 spin_unlock(&conn->ibc_lock);
859 kibnal_set_qp_state (kib_conn_t *conn, vv_qp_state_t new_state)
861 static vv_qp_attr_t attr;
863 kib_connvars_t *cv = conn->ibc_connvars;
866 /* Only called by connd => static OK */
867 LASSERT (!in_interrupt());
868 LASSERT (current == kibnal_data.kib_connd);
870 memset(&attr, 0, sizeof(attr));
876 case vv_qp_state_init: {
877 struct vv_qp_modify_init_st *init = &attr.modify.params.init;
879 init->p_key_indx = cv->cv_pkey_index;
880 init->phy_port_num = cv->cv_port;
881 init->q_key = IBNAL_QKEY; /* XXX but VV_QP_AT_Q_KEY not set! */
882 init->access_control = vv_acc_r_mem_read |
883 vv_acc_r_mem_write; /* XXX vv_acc_l_mem_write ? */
885 attr.modify.vv_qp_attr_mask = VV_QP_AT_P_KEY_IX |
886 VV_QP_AT_PHY_PORT_NUM |
887 VV_QP_AT_ACCESS_CON_F;
890 case vv_qp_state_rtr: {
891 struct vv_qp_modify_rtr_st *rtr = &attr.modify.params.rtr;
892 vv_add_vec_t *av = &rtr->remote_add_vec;
894 av->dlid = cv->cv_path.dlid;
895 av->grh_flag = (!IBNAL_LOCAL_SUB);
896 av->max_static_rate = IBNAL_R_2_STATIC_RATE(cv->cv_path.rate);
897 av->service_level = cv->cv_path.sl;
898 av->source_path_bit = IBNAL_SOURCE_PATH_BIT;
899 av->pmtu = cv->cv_path.mtu;
900 av->rnr_retry_count = cv->cv_rnr_count;
901 av->global_dest.traffic_class = cv->cv_path.traffic_class;
902 av->global_dest.hope_limit = cv->cv_path.hop_limut;
903 av->global_dest.flow_lable = cv->cv_path.flow_label;
904 av->global_dest.s_gid_index = cv->cv_sgid_index;
905 // XXX other av fields zero?
907 rtr->destanation_qp = cv->cv_remote_qpn;
908 rtr->receive_psn = cv->cv_rxpsn;
909 rtr->responder_rdma_r_atom_num = IBNAL_OUS_DST_RD;
910 rtr->opt_min_rnr_nak_timer = *kibnal_tunables.kib_rnr_nak_timer;
913 // XXX sdp sets VV_QP_AT_OP_F but no actual optional options
914 attr.modify.vv_qp_attr_mask = VV_QP_AT_ADD_VEC |
917 VV_QP_AT_MIN_RNR_NAK_T |
918 VV_QP_AT_RESP_RDMA_ATOM_OUT_NUM |
922 case vv_qp_state_rts: {
923 struct vv_qp_modify_rts_st *rts = &attr.modify.params.rts;
925 rts->send_psn = cv->cv_txpsn;
926 rts->local_ack_timeout = *kibnal_tunables.kib_local_ack_timeout;
927 rts->retry_num = *kibnal_tunables.kib_retry_cnt;
928 rts->rnr_num = *kibnal_tunables.kib_rnr_cnt;
929 rts->dest_out_rdma_r_atom_num = IBNAL_OUS_DST_RD;
931 attr.modify.vv_qp_attr_mask = VV_QP_AT_S_PSN |
935 VV_QP_AT_DEST_RDMA_ATOM_OUT_NUM;
938 case vv_qp_state_error:
939 case vv_qp_state_reset:
940 attr.modify.vv_qp_attr_mask = 0;
944 attr.modify.qp_modify_into_state = new_state;
945 attr.modify.vv_qp_attr_mask |= VV_QP_AT_STATE;
947 vvrc = vv_qp_modify(kibnal_data.kib_hca, conn->ibc_qp, &attr, NULL);
948 if (vvrc != vv_return_ok) {
949 CERROR("Can't modify qp -> %s state to %d: %d\n",
950 libcfs_nid2str(conn->ibc_peer->ibp_nid),
959 kibnal_create_conn (cm_cep_handle_t cep)
968 static vv_qp_attr_t reqattr;
969 static vv_qp_attr_t rspattr;
971 /* Only the connd creates conns => single threaded */
972 LASSERT(!in_interrupt());
973 LASSERT(current == kibnal_data.kib_connd);
975 LIBCFS_ALLOC(conn, sizeof (*conn));
977 CERROR ("Can't allocate connection\n");
981 /* zero flags, NULL pointers etc... */
982 memset (conn, 0, sizeof (*conn));
984 conn->ibc_version = IBNAL_MSG_VERSION; /* Use latest version at first */
986 INIT_LIST_HEAD (&conn->ibc_early_rxs);
987 INIT_LIST_HEAD (&conn->ibc_tx_queue_nocred);
988 INIT_LIST_HEAD (&conn->ibc_tx_queue);
989 INIT_LIST_HEAD (&conn->ibc_tx_queue_rsrvd);
990 INIT_LIST_HEAD (&conn->ibc_active_txs);
991 spin_lock_init (&conn->ibc_lock);
993 atomic_inc (&kibnal_data.kib_nconns);
994 /* well not really, but I call destroy() on failure, which decrements */
998 LIBCFS_ALLOC(conn->ibc_connvars, sizeof(*conn->ibc_connvars));
999 if (conn->ibc_connvars == NULL) {
1000 CERROR("Can't allocate in-progress connection state\n");
1003 memset (conn->ibc_connvars, 0, sizeof(*conn->ibc_connvars));
1004 /* Random seed for QP sequence number */
1005 get_random_bytes(&conn->ibc_connvars->cv_rxpsn,
1006 sizeof(conn->ibc_connvars->cv_rxpsn));
1008 LIBCFS_ALLOC(conn->ibc_rxs, IBNAL_RX_MSGS * sizeof (kib_rx_t));
1009 if (conn->ibc_rxs == NULL) {
1010 CERROR("Cannot allocate RX buffers\n");
1013 memset (conn->ibc_rxs, 0, IBNAL_RX_MSGS * sizeof(kib_rx_t));
1015 rc = kibnal_alloc_pages(&conn->ibc_rx_pages, IBNAL_RX_MSG_PAGES, 1);
1019 for (i = ipage = page_offset = 0; i < IBNAL_RX_MSGS; i++) {
1020 struct page *page = conn->ibc_rx_pages->ibp_pages[ipage];
1021 kib_rx_t *rx = &conn->ibc_rxs[i];
1022 vv_mem_reg_h_t mem_h;
1026 rx->rx_msg = (kib_msg_t *)(((char *)page_address(page)) +
1029 vvrc = vv_get_gen_mr_attrib(kibnal_data.kib_hca,
1035 LASSERT (vvrc == vv_return_ok);
1037 CDEBUG(D_NET, "Rx[%d] %p->%p[%x]\n", i, rx,
1038 rx->rx_msg, rx->rx_lkey);
1040 page_offset += IBNAL_MSG_SIZE;
1041 LASSERT (page_offset <= PAGE_SIZE);
1043 if (page_offset == PAGE_SIZE) {
1046 LASSERT (ipage <= IBNAL_RX_MSG_PAGES);
1050 memset(&reqattr, 0, sizeof(reqattr));
1052 reqattr.create.qp_type = vv_qp_type_r_conn;
1053 reqattr.create.cq_send_h = kibnal_data.kib_cq;
1054 reqattr.create.cq_receive_h = kibnal_data.kib_cq;
1055 reqattr.create.send_max_outstand_wr = (1 + IBNAL_MAX_RDMA_FRAGS) *
1056 (*kibnal_tunables.kib_concurrent_sends);
1057 reqattr.create.receive_max_outstand_wr = IBNAL_RX_MSGS;
1058 reqattr.create.max_scatgat_per_send_wr = 1;
1059 reqattr.create.max_scatgat_per_receive_wr = 1;
1060 reqattr.create.signaling_type = vv_selectable_signaling;
1061 reqattr.create.pd_h = kibnal_data.kib_pd;
1062 reqattr.create.recv_solicited_events = vv_selectable_signaling; // vv_signal_all;
1064 vvrc = vv_qp_create(kibnal_data.kib_hca, &reqattr, NULL,
1065 &conn->ibc_qp, &rspattr);
1066 if (vvrc != vv_return_ok) {
1067 CERROR ("Failed to create queue pair: %d\n", vvrc);
1071 /* Mark QP created */
1072 conn->ibc_state = IBNAL_CONN_INIT_QP;
1073 conn->ibc_connvars->cv_local_qpn = rspattr.create_return.qp_num;
1075 if (rspattr.create_return.receive_max_outstand_wr <
1077 rspattr.create_return.send_max_outstand_wr <
1078 (1 + IBNAL_MAX_RDMA_FRAGS) * (*kibnal_tunables.kib_concurrent_sends)) {
1079 CERROR("Insufficient rx/tx work items: wanted %d/%d got %d/%d\n",
1081 (1 + IBNAL_MAX_RDMA_FRAGS) *
1082 (*kibnal_tunables.kib_concurrent_sends),
1083 rspattr.create_return.receive_max_outstand_wr,
1084 rspattr.create_return.send_max_outstand_wr);
1088 /* Mark init complete */
1089 conn->ibc_state = IBNAL_CONN_INIT;
1091 /* 1 ref for caller */
1092 atomic_set (&conn->ibc_refcount, 1);
1096 kibnal_destroy_conn (conn);
1101 kibnal_destroy_conn (kib_conn_t *conn)
1105 /* Only the connd does this (i.e. single threaded) */
1106 LASSERT (!in_interrupt());
1107 LASSERT (current == kibnal_data.kib_connd);
1109 CDEBUG (D_NET, "connection %p\n", conn);
1111 LASSERT (atomic_read (&conn->ibc_refcount) == 0);
1112 LASSERT (list_empty(&conn->ibc_early_rxs));
1113 LASSERT (list_empty(&conn->ibc_tx_queue));
1114 LASSERT (list_empty(&conn->ibc_tx_queue_rsrvd));
1115 LASSERT (list_empty(&conn->ibc_tx_queue_nocred));
1116 LASSERT (list_empty(&conn->ibc_active_txs));
1117 LASSERT (conn->ibc_nsends_posted == 0);
1119 switch (conn->ibc_state) {
1121 /* conn must be completely disengaged from the network */
1124 case IBNAL_CONN_DISCONNECTED:
1125 /* connvars should have been freed already */
1126 LASSERT (conn->ibc_connvars == NULL);
1129 case IBNAL_CONN_INIT:
1130 vvrc = cm_destroy_cep(conn->ibc_cep);
1131 LASSERT (vvrc == vv_return_ok);
1134 case IBNAL_CONN_INIT_QP:
1135 kibnal_set_qp_state(conn, vv_qp_state_reset);
1136 vvrc = vv_qp_destroy(kibnal_data.kib_hca, conn->ibc_qp);
1137 if (vvrc != vv_return_ok)
1138 CERROR("Can't destroy QP: %d\n", vvrc);
1141 case IBNAL_CONN_INIT_NOTHING:
1145 if (conn->ibc_rx_pages != NULL)
1146 kibnal_free_pages(conn->ibc_rx_pages);
1148 if (conn->ibc_rxs != NULL)
1149 LIBCFS_FREE(conn->ibc_rxs,
1150 IBNAL_RX_MSGS * sizeof(kib_rx_t));
1152 if (conn->ibc_connvars != NULL)
1153 LIBCFS_FREE(conn->ibc_connvars, sizeof(*conn->ibc_connvars));
1155 if (conn->ibc_peer != NULL)
1156 kibnal_peer_decref(conn->ibc_peer);
1158 LIBCFS_FREE(conn, sizeof (*conn));
1160 atomic_dec(&kibnal_data.kib_nconns);
1164 kibnal_close_peer_conns_locked (kib_peer_t *peer, int why)
1167 struct list_head *ctmp;
1168 struct list_head *cnxt;
1171 list_for_each_safe (ctmp, cnxt, &peer->ibp_conns) {
1172 conn = list_entry (ctmp, kib_conn_t, ibc_list);
1175 kibnal_close_conn_locked (conn, why);
1182 kibnal_close_stale_conns_locked (kib_peer_t *peer, __u64 incarnation)
1185 struct list_head *ctmp;
1186 struct list_head *cnxt;
1189 list_for_each_safe (ctmp, cnxt, &peer->ibp_conns) {
1190 conn = list_entry (ctmp, kib_conn_t, ibc_list);
1192 if (conn->ibc_incarnation == incarnation)
1195 CDEBUG(D_NET, "Closing stale conn -> %s incarnation:"LPX64"("LPX64")\n",
1196 libcfs_nid2str(peer->ibp_nid),
1197 conn->ibc_incarnation, incarnation);
1200 kibnal_close_conn_locked (conn, -ESTALE);
1207 kibnal_close_matching_conns (lnet_nid_t nid)
1210 struct list_head *ptmp;
1211 struct list_head *pnxt;
1215 unsigned long flags;
1218 write_lock_irqsave(&kibnal_data.kib_global_lock, flags);
1220 if (nid != LNET_NID_ANY)
1221 lo = hi = kibnal_nid2peerlist(nid) - kibnal_data.kib_peers;
1224 hi = kibnal_data.kib_peer_hash_size - 1;
1227 for (i = lo; i <= hi; i++) {
1228 list_for_each_safe (ptmp, pnxt, &kibnal_data.kib_peers[i]) {
1230 peer = list_entry (ptmp, kib_peer_t, ibp_list);
1231 LASSERT (peer->ibp_persistence != 0 ||
1232 peer->ibp_connecting != 0 ||
1233 peer->ibp_accepting != 0 ||
1234 !list_empty (&peer->ibp_conns));
1236 if (!(nid == LNET_NID_ANY || nid == peer->ibp_nid))
1239 count += kibnal_close_peer_conns_locked (peer, 0);
1243 write_unlock_irqrestore(&kibnal_data.kib_global_lock, flags);
1245 /* wildcards always succeed */
1246 if (nid == LNET_NID_ANY)
1249 return (count == 0 ? -ENOENT : 0);
1253 kibnal_ctl(lnet_ni_t *ni, unsigned int cmd, void *arg)
1255 struct libcfs_ioctl_data *data = arg;
1258 LASSERT (ni == kibnal_data.kib_ni);
1261 case IOC_LIBCFS_GET_PEER: {
1264 int share_count = 0;
1266 rc = kibnal_get_peer_info(data->ioc_count,
1267 &nid, &ip, &share_count);
1268 data->ioc_nid = nid;
1269 data->ioc_count = share_count;
1270 data->ioc_u32[0] = ip;
1271 data->ioc_u32[1] = *kibnal_tunables.kib_service_number; /* port */
1274 case IOC_LIBCFS_ADD_PEER: {
1275 rc = kibnal_add_persistent_peer (data->ioc_nid,
1276 data->ioc_u32[0]); /* IP */
1279 case IOC_LIBCFS_DEL_PEER: {
1280 rc = kibnal_del_peer (data->ioc_nid);
1283 case IOC_LIBCFS_GET_CONN: {
1284 kib_conn_t *conn = kibnal_get_conn_by_idx (data->ioc_count);
1289 // kibnal_debug_conn(conn);
1291 data->ioc_nid = conn->ibc_peer->ibp_nid;
1292 kibnal_conn_decref(conn);
1296 case IOC_LIBCFS_CLOSE_CONNECTION: {
1297 rc = kibnal_close_matching_conns (data->ioc_nid);
1300 case IOC_LIBCFS_REGISTER_MYNID: {
1301 if (ni->ni_nid == data->ioc_nid) {
1304 CERROR("obsolete IOC_LIBCFS_REGISTER_MYNID: %s(%s)\n",
1305 libcfs_nid2str(data->ioc_nid),
1306 libcfs_nid2str(ni->ni_nid));
1317 kibnal_free_pages (kib_pages_t *p)
1319 int npages = p->ibp_npages;
1322 for (i = 0; i < npages; i++)
1323 if (p->ibp_pages[i] != NULL)
1324 __free_page(p->ibp_pages[i]);
1326 LIBCFS_FREE (p, offsetof(kib_pages_t, ibp_pages[npages]));
1330 kibnal_alloc_pages (kib_pages_t **pp, int npages, int allow_write)
1335 LIBCFS_ALLOC(p, offsetof(kib_pages_t, ibp_pages[npages]));
1337 CERROR ("Can't allocate buffer %d\n", npages);
1341 memset (p, 0, offsetof(kib_pages_t, ibp_pages[npages]));
1342 p->ibp_npages = npages;
1344 for (i = 0; i < npages; i++) {
1345 p->ibp_pages[i] = alloc_page (GFP_KERNEL);
1346 if (p->ibp_pages[i] == NULL) {
1347 CERROR ("Can't allocate page %d of %d\n", i, npages);
1348 kibnal_free_pages(p);
1358 kibnal_alloc_tx_descs (void)
1362 LIBCFS_ALLOC (kibnal_data.kib_tx_descs,
1363 IBNAL_TX_MSGS() * sizeof(kib_tx_t));
1364 if (kibnal_data.kib_tx_descs == NULL)
1367 memset(kibnal_data.kib_tx_descs, 0,
1368 IBNAL_TX_MSGS() * sizeof(kib_tx_t));
1370 for (i = 0; i < IBNAL_TX_MSGS(); i++) {
1371 kib_tx_t *tx = &kibnal_data.kib_tx_descs[i];
1374 LIBCFS_ALLOC(tx->tx_pages, LNET_MAX_IOV *
1375 sizeof(*tx->tx_pages));
1376 if (tx->tx_pages == NULL)
1379 LIBCFS_ALLOC(tx->tx_wrq,
1380 (1 + IBNAL_MAX_RDMA_FRAGS) *
1381 sizeof(*tx->tx_wrq));
1382 if (tx->tx_wrq == NULL)
1385 LIBCFS_ALLOC(tx->tx_gl,
1386 (1 + IBNAL_MAX_RDMA_FRAGS) *
1387 sizeof(*tx->tx_gl));
1388 if (tx->tx_gl == NULL)
1391 LIBCFS_ALLOC(tx->tx_rd,
1392 offsetof(kib_rdma_desc_t,
1393 rd_frags[IBNAL_MAX_RDMA_FRAGS]));
1394 if (tx->tx_rd == NULL)
1403 kibnal_free_tx_descs (void)
1407 if (kibnal_data.kib_tx_descs == NULL)
1410 for (i = 0; i < IBNAL_TX_MSGS(); i++) {
1411 kib_tx_t *tx = &kibnal_data.kib_tx_descs[i];
1414 if (tx->tx_pages != NULL)
1415 LIBCFS_FREE(tx->tx_pages, LNET_MAX_IOV *
1416 sizeof(*tx->tx_pages));
1418 if (tx->tx_wrq != NULL)
1419 LIBCFS_FREE(tx->tx_wrq,
1420 (1 + IBNAL_MAX_RDMA_FRAGS) *
1421 sizeof(*tx->tx_wrq));
1423 if (tx->tx_gl != NULL)
1424 LIBCFS_FREE(tx->tx_gl,
1425 (1 + IBNAL_MAX_RDMA_FRAGS) *
1426 sizeof(*tx->tx_gl));
1428 if (tx->tx_rd != NULL)
1429 LIBCFS_FREE(tx->tx_rd,
1430 offsetof(kib_rdma_desc_t,
1431 rd_frags[IBNAL_MAX_RDMA_FRAGS]));
1435 LIBCFS_FREE(kibnal_data.kib_tx_descs,
1436 IBNAL_TX_MSGS() * sizeof(kib_tx_t));
1441 kibnal_free_fmrs (int n)
1447 for (i = 0; i < n; i++) {
1448 tx = &kibnal_data.kib_tx_descs[i];
1450 vvrc = vv_free_fmr(kibnal_data.kib_hca,
1451 tx->tx_md.md_fmrhandle);
1452 if (vvrc != vv_return_ok)
1453 CWARN("vv_free_fmr[%d]: %d\n", i, vvrc);
1459 kibnal_setup_tx_descs (void)
1462 int page_offset = 0;
1465 vv_mem_reg_h_t mem_h;
1474 /* pre-mapped messages are not bigger than 1 page */
1475 CLASSERT (IBNAL_MSG_SIZE <= PAGE_SIZE);
1477 /* No fancy arithmetic when we do the buffer calculations */
1478 CLASSERT (PAGE_SIZE % IBNAL_MSG_SIZE == 0);
1480 rc = kibnal_alloc_pages(&kibnal_data.kib_tx_pages,
1481 IBNAL_TX_MSG_PAGES(), 0);
1485 for (i = 0; i < IBNAL_TX_MSGS(); i++) {
1486 page = kibnal_data.kib_tx_pages->ibp_pages[ipage];
1487 tx = &kibnal_data.kib_tx_descs[i];
1490 memset(&fmr_props, 0, sizeof(fmr_props));
1491 fmr_props.pd_hndl = kibnal_data.kib_pd;
1492 fmr_props.acl = (vv_acc_r_mem_write |
1493 vv_acc_l_mem_write);
1494 fmr_props.max_pages = LNET_MAX_IOV;
1495 fmr_props.log2_page_sz = PAGE_SHIFT;
1496 fmr_props.max_outstanding_maps = *kibnal_tunables.kib_fmr_remaps;
1498 vvrc = vv_alloc_fmr(kibnal_data.kib_hca,
1500 &tx->tx_md.md_fmrhandle);
1501 if (vvrc != vv_return_ok) {
1502 CERROR("Can't allocate fmr %d: %d\n", i, vvrc);
1503 kibnal_free_fmrs(i);
1504 kibnal_free_pages (kibnal_data.kib_tx_pages);
1508 tx->tx_md.md_fmrcount = *kibnal_tunables.kib_fmr_remaps;
1509 tx->tx_md.md_active = 0;
1511 tx->tx_msg = (kib_msg_t *)(((char *)page_address(page)) +
1514 vvrc = vv_get_gen_mr_attrib(kibnal_data.kib_hca,
1520 LASSERT (vvrc == vv_return_ok);
1522 CDEBUG(D_NET, "Tx[%d] %p->%p[%x]\n", i, tx,
1523 tx->tx_msg, tx->tx_lkey);
1525 list_add (&tx->tx_list, &kibnal_data.kib_idle_txs);
1527 page_offset += IBNAL_MSG_SIZE;
1528 LASSERT (page_offset <= PAGE_SIZE);
1530 if (page_offset == PAGE_SIZE) {
1533 LASSERT (ipage <= IBNAL_TX_MSG_PAGES());
1541 kibnal_shutdown (lnet_ni_t *ni)
1546 LASSERT (ni == kibnal_data.kib_ni);
1547 LASSERT (ni->ni_data == &kibnal_data);
1549 CDEBUG(D_MALLOC, "before NAL cleanup: kmem %d\n",
1550 atomic_read (&libcfs_kmemory));
1552 switch (kibnal_data.kib_init) {
1554 case IBNAL_INIT_ALL:
1555 /* stop accepting connections and prevent new peers */
1556 kibnal_stop_listener(ni);
1558 /* nuke all existing peers */
1559 kibnal_del_peer(LNET_NID_ANY);
1561 /* Wait for all peer state to clean up */
1563 while (atomic_read(&kibnal_data.kib_npeers) != 0) {
1565 CDEBUG(((i & (-i)) == i) ? D_WARNING : D_NET, /* 2**n? */
1566 "waiting for %d peers to disconnect\n",
1567 atomic_read(&kibnal_data.kib_npeers));
1568 cfs_pause(cfs_time_seconds(1));
1573 vvrc = vv_cq_destroy(kibnal_data.kib_hca, kibnal_data.kib_cq);
1574 if (vvrc != vv_return_ok)
1575 CERROR ("Destroy CQ error: %d\n", vvrc);
1578 case IBNAL_INIT_TXD:
1579 kibnal_free_pages (kibnal_data.kib_tx_pages);
1581 kibnal_free_fmrs(IBNAL_TX_MSGS());
1587 /* Only deallocate a PD if we actually allocated one */
1588 vvrc = vv_pd_deallocate(kibnal_data.kib_hca,
1589 kibnal_data.kib_pd);
1590 if (vvrc != vv_return_ok)
1591 CERROR ("Destroy PD error: %d\n", vvrc);
1595 case IBNAL_INIT_ASYNC:
1596 vvrc = vv_dell_async_event_cb (kibnal_data.kib_hca,
1597 kibnal_async_callback);
1598 if (vvrc != vv_return_ok)
1599 CERROR("vv_dell_async_event_cb error: %d\n", vvrc);
1603 case IBNAL_INIT_HCA:
1604 vvrc = vv_hca_close(kibnal_data.kib_hca);
1605 if (vvrc != vv_return_ok)
1606 CERROR ("Close HCA error: %d\n", vvrc);
1609 case IBNAL_INIT_DATA:
1610 LASSERT (atomic_read(&kibnal_data.kib_npeers) == 0);
1611 LASSERT (kibnal_data.kib_peers != NULL);
1612 for (i = 0; i < kibnal_data.kib_peer_hash_size; i++) {
1613 LASSERT (list_empty (&kibnal_data.kib_peers[i]));
1615 LASSERT (atomic_read (&kibnal_data.kib_nconns) == 0);
1616 LASSERT (list_empty (&kibnal_data.kib_connd_zombies));
1617 LASSERT (list_empty (&kibnal_data.kib_connd_conns));
1618 LASSERT (list_empty (&kibnal_data.kib_connd_pcreqs));
1619 LASSERT (list_empty (&kibnal_data.kib_connd_peers));
1621 /* flag threads to terminate; wake and wait for them to die */
1622 kibnal_data.kib_shutdown = 1;
1623 wake_up_all (&kibnal_data.kib_sched_waitq);
1624 wake_up_all (&kibnal_data.kib_connd_waitq);
1627 while (atomic_read (&kibnal_data.kib_nthreads) != 0) {
1629 CDEBUG(((i & (-i)) == i) ? D_WARNING : D_NET, /* power of 2? */
1630 "Waiting for %d threads to terminate\n",
1631 atomic_read (&kibnal_data.kib_nthreads));
1632 cfs_pause(cfs_time_seconds(1));
1636 case IBNAL_INIT_NOTHING:
1640 kibnal_free_tx_descs();
1642 if (kibnal_data.kib_peers != NULL)
1643 LIBCFS_FREE (kibnal_data.kib_peers,
1644 sizeof (struct list_head) *
1645 kibnal_data.kib_peer_hash_size);
1647 CDEBUG(D_MALLOC, "after NAL cleanup: kmem %d\n",
1648 atomic_read (&libcfs_kmemory));
1650 kibnal_data.kib_init = IBNAL_INIT_NOTHING;
1651 PORTAL_MODULE_UNUSE;
1655 kibnal_startup (lnet_ni_t *ni)
1668 vv_request_event_record_t req_er;
1671 LASSERT (ni->ni_lnd == &the_kiblnd);
1673 /* Only 1 instance supported */
1674 if (kibnal_data.kib_init != IBNAL_INIT_NOTHING) {
1675 CERROR ("Only 1 instance supported\n");
1679 if (*kibnal_tunables.kib_credits > *kibnal_tunables.kib_ntx) {
1680 CERROR ("Can't set credits(%d) > ntx(%d)\n",
1681 *kibnal_tunables.kib_credits,
1682 *kibnal_tunables.kib_ntx);
1686 ni->ni_maxtxcredits = *kibnal_tunables.kib_credits;
1687 ni->ni_peertxcredits = *kibnal_tunables.kib_peercredits;
1689 CLASSERT (LNET_MAX_INTERFACES > 1);
1691 if (ni->ni_interfaces[0] != NULL) {
1692 /* Use the HCA specified in 'networks=' */
1694 if (ni->ni_interfaces[1] != NULL) {
1695 CERROR("Multiple interfaces not supported\n");
1699 /* Parse <hca base name><number> */
1700 hca_name = ni->ni_interfaces[0];
1701 nob = strlen(*kibnal_tunables.kib_hca_basename);
1703 if (strncmp(hca_name, *kibnal_tunables.kib_hca_basename, nob) ||
1704 sscanf(hca_name + nob, "%d%n", &devno, &nob) < 1) {
1705 CERROR("Unrecognised HCA %s\n", hca_name);
1710 /* Use <hca base name>0 */
1714 snprintf(hca_name, sizeof(scratch), "%s%d",
1715 *kibnal_tunables.kib_hca_basename, devno);
1716 if (strlen(hca_name) == sizeof(scratch) - 1) {
1717 CERROR("HCA name %s truncated\n", hca_name);
1722 /* Find IP address from <ipif base name><hca number> */
1723 snprintf(ipif_name, sizeof(ipif_name), "%s%d",
1724 *kibnal_tunables.kib_ipif_basename, devno);
1725 if (strlen(ipif_name) == sizeof(ipif_name) - 1) {
1726 CERROR("IPoIB interface name %s truncated\n", ipif_name);
1730 rc = libcfs_ipif_query(ipif_name, &up, &ip, &netmask);
1732 CERROR("Can't query IPoIB interface %s: %d\n", ipif_name, rc);
1737 CERROR("Can't query IPoIB interface %s: it's down\n", ipif_name);
1741 ni->ni_nid = LNET_MKNID(LNET_NIDNET(ni->ni_nid), ip);
1744 memset (&kibnal_data, 0, sizeof (kibnal_data)); /* zero pointers, flags etc */
1746 kibnal_data.kib_ni = ni;
1747 ni->ni_data = &kibnal_data;
1749 do_gettimeofday(&tv);
1750 kibnal_data.kib_incarnation = (((__u64)tv.tv_sec) * 1000000) + tv.tv_usec;
1752 rwlock_init(&kibnal_data.kib_global_lock);
1754 kibnal_data.kib_peer_hash_size = IBNAL_PEER_HASH_SIZE;
1755 LIBCFS_ALLOC (kibnal_data.kib_peers,
1756 sizeof (struct list_head) * kibnal_data.kib_peer_hash_size);
1757 if (kibnal_data.kib_peers == NULL) {
1760 for (i = 0; i < kibnal_data.kib_peer_hash_size; i++)
1761 INIT_LIST_HEAD(&kibnal_data.kib_peers[i]);
1763 spin_lock_init (&kibnal_data.kib_connd_lock);
1764 INIT_LIST_HEAD (&kibnal_data.kib_connd_peers);
1765 INIT_LIST_HEAD (&kibnal_data.kib_connd_pcreqs);
1766 INIT_LIST_HEAD (&kibnal_data.kib_connd_conns);
1767 INIT_LIST_HEAD (&kibnal_data.kib_connd_zombies);
1768 init_waitqueue_head (&kibnal_data.kib_connd_waitq);
1770 spin_lock_init (&kibnal_data.kib_sched_lock);
1771 init_waitqueue_head (&kibnal_data.kib_sched_waitq);
1773 spin_lock_init (&kibnal_data.kib_tx_lock);
1774 INIT_LIST_HEAD (&kibnal_data.kib_idle_txs);
1776 rc = kibnal_alloc_tx_descs();
1778 CERROR("Can't allocate tx descs\n");
1782 /* lists/ptrs/locks initialised */
1783 kibnal_data.kib_init = IBNAL_INIT_DATA;
1784 /*****************************************************/
1786 for (i = 0; i < IBNAL_N_SCHED; i++) {
1787 rc = kibnal_thread_start (kibnal_scheduler, (void *)((long)i));
1789 CERROR("Can't spawn vibnal scheduler[%d]: %d\n",
1795 rc = kibnal_thread_start (kibnal_connd, NULL);
1797 CERROR ("Can't spawn vibnal connd: %d\n", rc);
1801 vvrc = vv_hca_open(hca_name, NULL, &kibnal_data.kib_hca);
1802 if (vvrc != vv_return_ok) {
1803 CERROR ("Can't open HCA %s: %d\n", hca_name, vvrc);
1807 /* Channel Adapter opened */
1808 kibnal_data.kib_init = IBNAL_INIT_HCA;
1810 /* register to get HCA's asynchronous events. */
1811 req_er.req_event_type = VV_ASYNC_EVENT_ALL_MASK;
1812 vvrc = vv_set_async_event_cb (kibnal_data.kib_hca, req_er,
1813 kibnal_async_callback);
1814 if (vvrc != vv_return_ok) {
1815 CERROR ("Can't set HCA %s callback: %d\n", hca_name, vvrc);
1819 kibnal_data.kib_init = IBNAL_INIT_ASYNC;
1821 /*****************************************************/
1823 vvrc = vv_hca_query(kibnal_data.kib_hca, &kibnal_data.kib_hca_attrs);
1824 if (vvrc != vv_return_ok) {
1825 CERROR ("Can't size port attrs for %s: %d\n", hca_name, vvrc);
1829 kibnal_data.kib_port = -1;
1831 for (i = 0; i<kibnal_data.kib_hca_attrs.port_num; i++) {
1834 u_int32_t tbl_count;
1835 vv_port_attrib_t *pattr = &kibnal_data.kib_port_attr;
1837 vvrc = vv_port_query(kibnal_data.kib_hca, port_num, pattr);
1838 if (vvrc != vv_return_ok) {
1839 CERROR("vv_port_query failed for %s port %d: %d\n",
1840 hca_name, port_num, vvrc);
1844 switch (pattr->port_state) {
1845 case vv_state_linkDoun:
1846 CDEBUG(D_NET, "port[%d] Down\n", port_num);
1848 case vv_state_linkInit:
1849 CDEBUG(D_NET, "port[%d] Init\n", port_num);
1851 case vv_state_linkArm:
1852 CDEBUG(D_NET, "port[%d] Armed\n", port_num);
1854 case vv_state_linkActive:
1855 CDEBUG(D_NET, "port[%d] Active\n", port_num);
1857 /* Found a suitable port. Get its GUID and PKEY. */
1859 vvrc = vv_get_port_gid_tbl(kibnal_data.kib_hca,
1860 port_num, &tbl_count,
1861 &kibnal_data.kib_port_gid);
1862 if (vvrc != vv_return_ok) {
1863 CERROR("vv_get_port_gid_tbl failed "
1864 "for %s port %d: %d\n",
1865 hca_name, port_num, vvrc);
1870 vvrc = vv_get_port_partition_tbl(kibnal_data.kib_hca,
1871 port_num, &tbl_count,
1872 &kibnal_data.kib_port_pkey);
1873 if (vvrc != vv_return_ok) {
1874 CERROR("vv_get_port_partition_tbl failed "
1875 "for %s port %d: %d\n",
1876 hca_name, port_num, vvrc);
1880 kibnal_data.kib_port = port_num;
1883 case vv_state_linkActDefer: /* TODO: correct? */
1884 case vv_state_linkNoChange:
1885 CERROR("Unexpected %s port[%d] state %d\n",
1886 hca_name, i, pattr->port_state);
1892 if (kibnal_data.kib_port == -1) {
1893 CERROR ("Can't find an active port on %s\n", hca_name);
1897 CDEBUG(D_NET, "Using %s port %d - GID="LPX64":"LPX64"\n",
1898 hca_name, kibnal_data.kib_port,
1899 kibnal_data.kib_port_gid.scope.g.subnet,
1900 kibnal_data.kib_port_gid.scope.g.eui64);
1902 /*****************************************************/
1905 /* We use a pre-allocated PD */
1906 vvrc = vv_get_gen_pd_h(kibnal_data.kib_hca, &kibnal_data.kib_pd);
1908 vvrc = vv_pd_allocate(kibnal_data.kib_hca, &kibnal_data.kib_pd);
1910 if (vvrc != vv_return_ok) {
1911 CERROR ("Can't init PD: %d\n", vvrc);
1915 /* flag PD initialised */
1916 kibnal_data.kib_init = IBNAL_INIT_PD;
1917 /*****************************************************/
1919 rc = kibnal_setup_tx_descs();
1921 CERROR ("Can't register tx descs: %d\n", rc);
1925 /* flag TX descs initialised */
1926 kibnal_data.kib_init = IBNAL_INIT_TXD;
1927 /*****************************************************/
1932 vvrc = vv_cq_create(kibnal_data.kib_hca, IBNAL_CQ_ENTRIES(),
1935 &kibnal_data.kib_cq, &nentries);
1937 CERROR ("Can't create RX CQ: %d\n", vvrc);
1941 /* flag CQ initialised */
1942 kibnal_data.kib_init = IBNAL_INIT_CQ;
1944 if (nentries < IBNAL_CQ_ENTRIES()) {
1945 CERROR ("CQ only has %d entries, need %d\n",
1946 nentries, IBNAL_CQ_ENTRIES());
1950 vvrc = vv_request_completion_notification(kibnal_data.kib_hca,
1952 vv_next_solicit_unsolicit_event);
1954 CERROR ("Failed to re-arm completion queue: %d\n", rc);
1959 rc = kibnal_start_listener(ni);
1961 CERROR("Can't start listener: %d\n", rc);
1965 /* flag everything initialised */
1966 kibnal_data.kib_init = IBNAL_INIT_ALL;
1967 /*****************************************************/
1972 CDEBUG(D_NET, "kibnal_startup failed\n");
1973 kibnal_shutdown (ni);
1978 kibnal_module_fini (void)
1980 lnet_unregister_lnd(&the_kiblnd);
1981 kibnal_tunables_fini();
1985 kibnal_module_init (void)
1989 vibnal_assert_wire_constants();
1991 CLASSERT (offsetof(kib_msg_t, ibm_u) + sizeof(kib_connparams_t)
1992 <= cm_REQ_priv_data_len);
1993 CLASSERT (offsetof(kib_msg_t, ibm_u) + sizeof(kib_connparams_t)
1994 <= cm_REP_priv_data_len);
1995 CLASSERT (sizeof(kib_msg_t) <= IBNAL_MSG_SIZE);
1997 CLASSERT (offsetof(kib_msg_t, ibm_u.get.ibgm_rd.rd_frags[IBNAL_MAX_RDMA_FRAGS])
1999 CLASSERT (offsetof(kib_msg_t, ibm_u.putack.ibpam_rd.rd_frags[IBNAL_MAX_RDMA_FRAGS])
2002 rc = kibnal_tunables_init();
2006 lnet_register_lnd(&the_kiblnd);
2011 MODULE_AUTHOR("Cluster File Systems, Inc. <info@clusterfs.com>");
2012 MODULE_DESCRIPTION("Kernel Voltaire IB LND v1.00");
2013 MODULE_LICENSE("GPL");
2015 module_init(kibnal_module_init);
2016 module_exit(kibnal_module_fini);