Whamcloud - gitweb
LU-1714 lnet: Properly initialize sg_magic value
[fs/lustre-release.git] / lnet / klnds / o2iblnd / o2iblnd.c
1 /*
2  * GPL HEADER START
3  *
4  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
5  *
6  * This program is free software; you can redistribute it and/or modify
7  * it under the terms of the GNU General Public License version 2 only,
8  * as published by the Free Software Foundation.
9  *
10  * This program is distributed in the hope that it will be useful, but
11  * WITHOUT ANY WARRANTY; without even the implied warranty of
12  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
13  * General Public License version 2 for more details (a copy is included
14  * in the LICENSE file that accompanied this code).
15  *
16  * You should have received a copy of the GNU General Public License
17  * version 2 along with this program; If not, see
18  * http://www.sun.com/software/products/lustre/docs/GPLv2.pdf
19  *
20  * Please contact Sun Microsystems, Inc., 4150 Network Circle, Santa Clara,
21  * CA 95054 USA or visit www.sun.com if you need additional information or
22  * have any questions.
23  *
24  * GPL HEADER END
25  */
26 /*
27  * Copyright (c) 2007, 2010, Oracle and/or its affiliates. All rights reserved.
28  * Use is subject to license terms.
29  *
30  * Copyright (c) 2011, Whamcloud, Inc.
31  */
32 /*
33  * This file is part of Lustre, http://www.lustre.org/
34  * Lustre is a trademark of Sun Microsystems, Inc.
35  *
36  * lnet/klnds/o2iblnd/o2iblnd.c
37  *
38  * Author: Eric Barton <eric@bartonsoftware.com>
39  */
40
41 #include "o2iblnd.h"
42
43 lnd_t the_o2iblnd = {
44         .lnd_type       = O2IBLND,
45         .lnd_startup    = kiblnd_startup,
46         .lnd_shutdown   = kiblnd_shutdown,
47         .lnd_ctl        = kiblnd_ctl,
48         .lnd_query      = kiblnd_query,
49         .lnd_send       = kiblnd_send,
50         .lnd_recv       = kiblnd_recv,
51 };
52
53 kib_data_t              kiblnd_data;
54
55 __u32
56 kiblnd_cksum (void *ptr, int nob)
57 {
58         char  *c  = ptr;
59         __u32  sum = 0;
60
61         while (nob-- > 0)
62                 sum = ((sum << 1) | (sum >> 31)) + *c++;
63
64         /* ensure I don't return 0 (== no checksum) */
65         return (sum == 0) ? 1 : sum;
66 }
67
68 static char *
69 kiblnd_msgtype2str(int type)
70 {
71         switch (type) {
72         case IBLND_MSG_CONNREQ:
73                 return "CONNREQ";
74
75         case IBLND_MSG_CONNACK:
76                 return "CONNACK";
77
78         case IBLND_MSG_NOOP:
79                 return "NOOP";
80
81         case IBLND_MSG_IMMEDIATE:
82                 return "IMMEDIATE";
83
84         case IBLND_MSG_PUT_REQ:
85                 return "PUT_REQ";
86
87         case IBLND_MSG_PUT_NAK:
88                 return "PUT_NAK";
89
90         case IBLND_MSG_PUT_ACK:
91                 return "PUT_ACK";
92
93         case IBLND_MSG_PUT_DONE:
94                 return "PUT_DONE";
95
96         case IBLND_MSG_GET_REQ:
97                 return "GET_REQ";
98
99         case IBLND_MSG_GET_DONE:
100                 return "GET_DONE";
101
102         default:
103                 return "???";
104         }
105 }
106
107 static int
108 kiblnd_msgtype2size(int type)
109 {
110         const int hdr_size = offsetof(kib_msg_t, ibm_u);
111
112         switch (type) {
113         case IBLND_MSG_CONNREQ:
114         case IBLND_MSG_CONNACK:
115                 return hdr_size + sizeof(kib_connparams_t);
116
117         case IBLND_MSG_NOOP:
118                 return hdr_size;
119
120         case IBLND_MSG_IMMEDIATE:
121                 return offsetof(kib_msg_t, ibm_u.immediate.ibim_payload[0]);
122
123         case IBLND_MSG_PUT_REQ:
124                 return hdr_size + sizeof(kib_putreq_msg_t);
125
126         case IBLND_MSG_PUT_ACK:
127                 return hdr_size + sizeof(kib_putack_msg_t);
128
129         case IBLND_MSG_GET_REQ:
130                 return hdr_size + sizeof(kib_get_msg_t);
131
132         case IBLND_MSG_PUT_NAK:
133         case IBLND_MSG_PUT_DONE:
134         case IBLND_MSG_GET_DONE:
135                 return hdr_size + sizeof(kib_completion_msg_t);
136         default:
137                 return -1;
138         }
139 }
140
141 static int
142 kiblnd_unpack_rd(kib_msg_t *msg, int flip)
143 {
144         kib_rdma_desc_t   *rd;
145         int                nob;
146         int                n;
147         int                i;
148
149         LASSERT (msg->ibm_type == IBLND_MSG_GET_REQ ||
150                  msg->ibm_type == IBLND_MSG_PUT_ACK);
151
152         rd = msg->ibm_type == IBLND_MSG_GET_REQ ?
153                               &msg->ibm_u.get.ibgm_rd :
154                               &msg->ibm_u.putack.ibpam_rd;
155
156         if (flip) {
157                 __swab32s(&rd->rd_key);
158                 __swab32s(&rd->rd_nfrags);
159         }
160
161         n = rd->rd_nfrags;
162
163         if (n <= 0 || n > IBLND_MAX_RDMA_FRAGS) {
164                 CERROR("Bad nfrags: %d, should be 0 < n <= %d\n",
165                        n, IBLND_MAX_RDMA_FRAGS);
166                 return 1;
167         }
168
169         nob = offsetof (kib_msg_t, ibm_u) +
170               kiblnd_rd_msg_size(rd, msg->ibm_type, n);
171
172         if (msg->ibm_nob < nob) {
173                 CERROR("Short %s: %d(%d)\n",
174                        kiblnd_msgtype2str(msg->ibm_type), msg->ibm_nob, nob);
175                 return 1;
176         }
177
178         if (!flip)
179                 return 0;
180
181         for (i = 0; i < n; i++) {
182                 __swab32s(&rd->rd_frags[i].rf_nob);
183                 __swab64s(&rd->rd_frags[i].rf_addr);
184         }
185
186         return 0;
187 }
188
189 void
190 kiblnd_pack_msg (lnet_ni_t *ni, kib_msg_t *msg, int version,
191                  int credits, lnet_nid_t dstnid, __u64 dststamp)
192 {
193         kib_net_t *net = ni->ni_data;
194
195         /* CAVEAT EMPTOR! all message fields not set here should have been
196          * initialised previously. */
197         msg->ibm_magic    = IBLND_MSG_MAGIC;
198         msg->ibm_version  = version;
199         /*   ibm_type */
200         msg->ibm_credits  = credits;
201         /*   ibm_nob */
202         msg->ibm_cksum    = 0;
203         msg->ibm_srcnid   = ni->ni_nid;
204         msg->ibm_srcstamp = net->ibn_incarnation;
205         msg->ibm_dstnid   = dstnid;
206         msg->ibm_dststamp = dststamp;
207
208         if (*kiblnd_tunables.kib_cksum) {
209                 /* NB ibm_cksum zero while computing cksum */
210                 msg->ibm_cksum = kiblnd_cksum(msg, msg->ibm_nob);
211         }
212 }
213
214 int
215 kiblnd_unpack_msg(kib_msg_t *msg, int nob)
216 {
217         const int hdr_size = offsetof(kib_msg_t, ibm_u);
218         __u32     msg_cksum;
219         __u16     version;
220         int       msg_nob;
221         int       flip;
222
223         /* 6 bytes are enough to have received magic + version */
224         if (nob < 6) {
225                 CERROR("Short message: %d\n", nob);
226                 return -EPROTO;
227         }
228
229         if (msg->ibm_magic == IBLND_MSG_MAGIC) {
230                 flip = 0;
231         } else if (msg->ibm_magic == __swab32(IBLND_MSG_MAGIC)) {
232                 flip = 1;
233         } else {
234                 CERROR("Bad magic: %08x\n", msg->ibm_magic);
235                 return -EPROTO;
236         }
237
238         version = flip ? __swab16(msg->ibm_version) : msg->ibm_version;
239         if (version != IBLND_MSG_VERSION &&
240             version != IBLND_MSG_VERSION_1) {
241                 CERROR("Bad version: %x\n", version);
242                 return -EPROTO;
243         }
244
245         if (nob < hdr_size) {
246                 CERROR("Short message: %d\n", nob);
247                 return -EPROTO;
248         }
249
250         msg_nob = flip ? __swab32(msg->ibm_nob) : msg->ibm_nob;
251         if (msg_nob > nob) {
252                 CERROR("Short message: got %d, wanted %d\n", nob, msg_nob);
253                 return -EPROTO;
254         }
255
256         /* checksum must be computed with ibm_cksum zero and BEFORE anything
257          * gets flipped */
258         msg_cksum = flip ? __swab32(msg->ibm_cksum) : msg->ibm_cksum;
259         msg->ibm_cksum = 0;
260         if (msg_cksum != 0 &&
261             msg_cksum != kiblnd_cksum(msg, msg_nob)) {
262                 CERROR("Bad checksum\n");
263                 return -EPROTO;
264         }
265
266         msg->ibm_cksum = msg_cksum;
267
268         if (flip) {
269                 /* leave magic unflipped as a clue to peer endianness */
270                 msg->ibm_version = version;
271                 CLASSERT (sizeof(msg->ibm_type) == 1);
272                 CLASSERT (sizeof(msg->ibm_credits) == 1);
273                 msg->ibm_nob     = msg_nob;
274                 __swab64s(&msg->ibm_srcnid);
275                 __swab64s(&msg->ibm_srcstamp);
276                 __swab64s(&msg->ibm_dstnid);
277                 __swab64s(&msg->ibm_dststamp);
278         }
279
280         if (msg->ibm_srcnid == LNET_NID_ANY) {
281                 CERROR("Bad src nid: %s\n", libcfs_nid2str(msg->ibm_srcnid));
282                 return -EPROTO;
283         }
284
285         if (msg_nob < kiblnd_msgtype2size(msg->ibm_type)) {
286                 CERROR("Short %s: %d(%d)\n", kiblnd_msgtype2str(msg->ibm_type),
287                        msg_nob, kiblnd_msgtype2size(msg->ibm_type));
288                 return -EPROTO;
289         }
290
291         switch (msg->ibm_type) {
292         default:
293                 CERROR("Unknown message type %x\n", msg->ibm_type);
294                 return -EPROTO;
295
296         case IBLND_MSG_NOOP:
297         case IBLND_MSG_IMMEDIATE:
298         case IBLND_MSG_PUT_REQ:
299                 break;
300
301         case IBLND_MSG_PUT_ACK:
302         case IBLND_MSG_GET_REQ:
303                 if (kiblnd_unpack_rd(msg, flip))
304                         return -EPROTO;
305                 break;
306
307         case IBLND_MSG_PUT_NAK:
308         case IBLND_MSG_PUT_DONE:
309         case IBLND_MSG_GET_DONE:
310                 if (flip)
311                         __swab32s(&msg->ibm_u.completion.ibcm_status);
312                 break;
313
314         case IBLND_MSG_CONNREQ:
315         case IBLND_MSG_CONNACK:
316                 if (flip) {
317                         __swab16s(&msg->ibm_u.connparams.ibcp_queue_depth);
318                         __swab16s(&msg->ibm_u.connparams.ibcp_max_frags);
319                         __swab32s(&msg->ibm_u.connparams.ibcp_max_msg_size);
320                 }
321                 break;
322         }
323         return 0;
324 }
325
326 int
327 kiblnd_create_peer(lnet_ni_t *ni, kib_peer_t **peerp, lnet_nid_t nid)
328 {
329         kib_peer_t      *peer;
330         kib_net_t       *net = ni->ni_data;
331         int             cpt = lnet_cpt_of_nid(nid);
332         unsigned long   flags;
333
334         LASSERT(net != NULL);
335         LASSERT(nid != LNET_NID_ANY);
336
337         LIBCFS_CPT_ALLOC(peer, lnet_cpt_table(), cpt, sizeof(*peer));
338         if (peer == NULL) {
339                 CERROR("Cannot allocate peer\n");
340                 return -ENOMEM;
341         }
342
343         memset(peer, 0, sizeof(*peer));         /* zero flags etc */
344
345         peer->ibp_ni = ni;
346         peer->ibp_nid = nid;
347         peer->ibp_error = 0;
348         peer->ibp_last_alive = 0;
349         cfs_atomic_set(&peer->ibp_refcount, 1);  /* 1 ref for caller */
350
351         CFS_INIT_LIST_HEAD(&peer->ibp_list);     /* not in the peer table yet */
352         CFS_INIT_LIST_HEAD(&peer->ibp_conns);
353         CFS_INIT_LIST_HEAD(&peer->ibp_tx_queue);
354
355         cfs_write_lock_irqsave(&kiblnd_data.kib_global_lock, flags);
356
357         /* always called with a ref on ni, which prevents ni being shutdown */
358         LASSERT (net->ibn_shutdown == 0);
359
360         /* npeers only grows with the global lock held */
361         cfs_atomic_inc(&net->ibn_npeers);
362
363         cfs_write_unlock_irqrestore(&kiblnd_data.kib_global_lock, flags);
364
365         *peerp = peer;
366         return 0;
367 }
368
369 void
370 kiblnd_destroy_peer (kib_peer_t *peer)
371 {
372         kib_net_t *net = peer->ibp_ni->ni_data;
373
374         LASSERT (net != NULL);
375         LASSERT (cfs_atomic_read(&peer->ibp_refcount) == 0);
376         LASSERT (!kiblnd_peer_active(peer));
377         LASSERT (peer->ibp_connecting == 0);
378         LASSERT (peer->ibp_accepting == 0);
379         LASSERT (cfs_list_empty(&peer->ibp_conns));
380         LASSERT (cfs_list_empty(&peer->ibp_tx_queue));
381
382         LIBCFS_FREE(peer, sizeof(*peer));
383
384         /* NB a peer's connections keep a reference on their peer until
385          * they are destroyed, so we can be assured that _all_ state to do
386          * with this peer has been cleaned up when its refcount drops to
387          * zero. */
388         cfs_atomic_dec(&net->ibn_npeers);
389 }
390
391 kib_peer_t *
392 kiblnd_find_peer_locked (lnet_nid_t nid)
393 {
394         /* the caller is responsible for accounting the additional reference
395          * that this creates */
396         cfs_list_t       *peer_list = kiblnd_nid2peerlist(nid);
397         cfs_list_t       *tmp;
398         kib_peer_t       *peer;
399
400         cfs_list_for_each (tmp, peer_list) {
401
402                 peer = cfs_list_entry(tmp, kib_peer_t, ibp_list);
403
404                 LASSERT (peer->ibp_connecting > 0 || /* creating conns */
405                          peer->ibp_accepting > 0 ||
406                          !cfs_list_empty(&peer->ibp_conns));  /* active conn */
407
408                 if (peer->ibp_nid != nid)
409                         continue;
410
411                 CDEBUG(D_NET, "got peer [%p] -> %s (%d) version: %x\n",
412                        peer, libcfs_nid2str(nid),
413                        cfs_atomic_read(&peer->ibp_refcount),
414                        peer->ibp_version);
415                 return peer;
416         }
417         return NULL;
418 }
419
420 void
421 kiblnd_unlink_peer_locked (kib_peer_t *peer)
422 {
423         LASSERT (cfs_list_empty(&peer->ibp_conns));
424
425         LASSERT (kiblnd_peer_active(peer));
426         cfs_list_del_init(&peer->ibp_list);
427         /* lose peerlist's ref */
428         kiblnd_peer_decref(peer);
429 }
430
431 int
432 kiblnd_get_peer_info (lnet_ni_t *ni, int index, 
433                       lnet_nid_t *nidp, int *count)
434 {
435         kib_peer_t            *peer;
436         cfs_list_t            *ptmp;
437         int                    i;
438         unsigned long          flags;
439
440         cfs_read_lock_irqsave(&kiblnd_data.kib_global_lock, flags);
441
442         for (i = 0; i < kiblnd_data.kib_peer_hash_size; i++) {
443
444                 cfs_list_for_each (ptmp, &kiblnd_data.kib_peers[i]) {
445
446                         peer = cfs_list_entry(ptmp, kib_peer_t, ibp_list);
447                         LASSERT (peer->ibp_connecting > 0 ||
448                                  peer->ibp_accepting > 0 ||
449                                  !cfs_list_empty(&peer->ibp_conns));
450
451                         if (peer->ibp_ni != ni)
452                                 continue;
453
454                         if (index-- > 0)
455                                 continue;
456
457                         *nidp = peer->ibp_nid;
458                         *count = cfs_atomic_read(&peer->ibp_refcount);
459
460                         cfs_read_unlock_irqrestore(&kiblnd_data.kib_global_lock,
461                                                    flags);
462                         return 0;
463                 }
464         }
465
466         cfs_read_unlock_irqrestore(&kiblnd_data.kib_global_lock, flags);
467         return -ENOENT;
468 }
469
470 void
471 kiblnd_del_peer_locked (kib_peer_t *peer)
472 {
473         cfs_list_t           *ctmp;
474         cfs_list_t           *cnxt;
475         kib_conn_t           *conn;
476
477         if (cfs_list_empty(&peer->ibp_conns)) {
478                 kiblnd_unlink_peer_locked(peer);
479         } else {
480                 cfs_list_for_each_safe (ctmp, cnxt, &peer->ibp_conns) {
481                         conn = cfs_list_entry(ctmp, kib_conn_t, ibc_list);
482
483                         kiblnd_close_conn_locked(conn, 0);
484                 }
485                 /* NB closing peer's last conn unlinked it. */
486         }
487         /* NB peer now unlinked; might even be freed if the peer table had the
488          * last ref on it. */
489 }
490
491 int
492 kiblnd_del_peer (lnet_ni_t *ni, lnet_nid_t nid)
493 {
494         CFS_LIST_HEAD         (zombies);
495         cfs_list_t            *ptmp;
496         cfs_list_t            *pnxt;
497         kib_peer_t            *peer;
498         int                    lo;
499         int                    hi;
500         int                    i;
501         unsigned long          flags;
502         int                    rc = -ENOENT;
503
504         cfs_write_lock_irqsave(&kiblnd_data.kib_global_lock, flags);
505
506         if (nid != LNET_NID_ANY) {
507                 lo = hi = kiblnd_nid2peerlist(nid) - kiblnd_data.kib_peers;
508         } else {
509                 lo = 0;
510                 hi = kiblnd_data.kib_peer_hash_size - 1;
511         }
512
513         for (i = lo; i <= hi; i++) {
514                 cfs_list_for_each_safe (ptmp, pnxt, &kiblnd_data.kib_peers[i]) {
515                         peer = cfs_list_entry(ptmp, kib_peer_t, ibp_list);
516                         LASSERT (peer->ibp_connecting > 0 ||
517                                  peer->ibp_accepting > 0 ||
518                                  !cfs_list_empty(&peer->ibp_conns));
519
520                         if (peer->ibp_ni != ni)
521                                 continue;
522
523                         if (!(nid == LNET_NID_ANY || peer->ibp_nid == nid))
524                                 continue;
525
526                         if (!cfs_list_empty(&peer->ibp_tx_queue)) {
527                                 LASSERT (cfs_list_empty(&peer->ibp_conns));
528
529                                 cfs_list_splice_init(&peer->ibp_tx_queue,
530                                                      &zombies);
531                         }
532
533                         kiblnd_del_peer_locked(peer);
534                         rc = 0;         /* matched something */
535                 }
536         }
537
538         cfs_write_unlock_irqrestore(&kiblnd_data.kib_global_lock, flags);
539
540         kiblnd_txlist_done(ni, &zombies, -EIO);
541
542         return rc;
543 }
544
545 kib_conn_t *
546 kiblnd_get_conn_by_idx (lnet_ni_t *ni, int index)
547 {
548         kib_peer_t            *peer;
549         cfs_list_t            *ptmp;
550         kib_conn_t            *conn;
551         cfs_list_t            *ctmp;
552         int                    i;
553         unsigned long          flags;
554
555         cfs_read_lock_irqsave(&kiblnd_data.kib_global_lock, flags);
556
557         for (i = 0; i < kiblnd_data.kib_peer_hash_size; i++) {
558                 cfs_list_for_each (ptmp, &kiblnd_data.kib_peers[i]) {
559
560                         peer = cfs_list_entry(ptmp, kib_peer_t, ibp_list);
561                         LASSERT (peer->ibp_connecting > 0 ||
562                                  peer->ibp_accepting > 0 ||
563                                  !cfs_list_empty(&peer->ibp_conns));
564
565                         if (peer->ibp_ni != ni)
566                                 continue;
567
568                         cfs_list_for_each (ctmp, &peer->ibp_conns) {
569                                 if (index-- > 0)
570                                         continue;
571
572                                 conn = cfs_list_entry(ctmp, kib_conn_t,
573                                                       ibc_list);
574                                 kiblnd_conn_addref(conn);
575                                 cfs_read_unlock_irqrestore(&kiblnd_data.kib_global_lock,
576                                                            flags);
577                                 return conn;
578                         }
579                 }
580         }
581
582         cfs_read_unlock_irqrestore(&kiblnd_data.kib_global_lock, flags);
583         return NULL;
584 }
585
586 void
587 kiblnd_debug_rx (kib_rx_t *rx)
588 {
589         CDEBUG(D_CONSOLE, "      %p status %d msg_type %x cred %d\n",
590                rx, rx->rx_status, rx->rx_msg->ibm_type,
591                rx->rx_msg->ibm_credits);
592 }
593
594 void
595 kiblnd_debug_tx (kib_tx_t *tx)
596 {
597         CDEBUG(D_CONSOLE, "      %p snd %d q %d w %d rc %d dl %lx "
598                "cookie "LPX64" msg %s%s type %x cred %d\n",
599                tx, tx->tx_sending, tx->tx_queued, tx->tx_waiting,
600                tx->tx_status, tx->tx_deadline, tx->tx_cookie,
601                tx->tx_lntmsg[0] == NULL ? "-" : "!",
602                tx->tx_lntmsg[1] == NULL ? "-" : "!",
603                tx->tx_msg->ibm_type, tx->tx_msg->ibm_credits);
604 }
605
606 void
607 kiblnd_debug_conn (kib_conn_t *conn)
608 {
609         cfs_list_t           *tmp;
610         int                   i;
611
612         cfs_spin_lock(&conn->ibc_lock);
613
614         CDEBUG(D_CONSOLE, "conn[%d] %p [version %x] -> %s: \n",
615                cfs_atomic_read(&conn->ibc_refcount), conn,
616                conn->ibc_version, libcfs_nid2str(conn->ibc_peer->ibp_nid));
617         CDEBUG(D_CONSOLE, "   state %d nposted %d/%d cred %d o_cred %d r_cred %d\n",
618                conn->ibc_state, conn->ibc_noops_posted,
619                conn->ibc_nsends_posted, conn->ibc_credits,
620                conn->ibc_outstanding_credits, conn->ibc_reserved_credits);
621         CDEBUG(D_CONSOLE, "   comms_err %d\n", conn->ibc_comms_error);
622
623         CDEBUG(D_CONSOLE, "   early_rxs:\n");
624         cfs_list_for_each(tmp, &conn->ibc_early_rxs)
625                 kiblnd_debug_rx(cfs_list_entry(tmp, kib_rx_t, rx_list));
626
627         CDEBUG(D_CONSOLE, "   tx_noops:\n");
628         cfs_list_for_each(tmp, &conn->ibc_tx_noops)
629                 kiblnd_debug_tx(cfs_list_entry(tmp, kib_tx_t, tx_list));
630
631         CDEBUG(D_CONSOLE, "   tx_queue_nocred:\n");
632         cfs_list_for_each(tmp, &conn->ibc_tx_queue_nocred)
633                 kiblnd_debug_tx(cfs_list_entry(tmp, kib_tx_t, tx_list));
634
635         CDEBUG(D_CONSOLE, "   tx_queue_rsrvd:\n");
636         cfs_list_for_each(tmp, &conn->ibc_tx_queue_rsrvd)
637                 kiblnd_debug_tx(cfs_list_entry(tmp, kib_tx_t, tx_list));
638
639         CDEBUG(D_CONSOLE, "   tx_queue:\n");
640         cfs_list_for_each(tmp, &conn->ibc_tx_queue)
641                 kiblnd_debug_tx(cfs_list_entry(tmp, kib_tx_t, tx_list));
642
643         CDEBUG(D_CONSOLE, "   active_txs:\n");
644         cfs_list_for_each(tmp, &conn->ibc_active_txs)
645                 kiblnd_debug_tx(cfs_list_entry(tmp, kib_tx_t, tx_list));
646
647         CDEBUG(D_CONSOLE, "   rxs:\n");
648         for (i = 0; i < IBLND_RX_MSGS(conn->ibc_version); i++)
649                 kiblnd_debug_rx(&conn->ibc_rxs[i]);
650
651         cfs_spin_unlock(&conn->ibc_lock);
652 }
653
654 int
655 kiblnd_translate_mtu(int value)
656 {
657         switch (value) {
658         default:
659                 return -1;
660         case 0:
661                 return 0;
662         case 256:
663                 return IB_MTU_256;
664         case 512:
665                 return IB_MTU_512;
666         case 1024:
667                 return IB_MTU_1024;
668         case 2048:
669                 return IB_MTU_2048;
670         case 4096:
671                 return IB_MTU_4096;
672         }
673 }
674
675 static void
676 kiblnd_setup_mtu_locked(struct rdma_cm_id *cmid)
677 {
678         int           mtu;
679
680         /* XXX There is no path record for iWARP, set by netdev->change_mtu? */
681         if (cmid->route.path_rec == NULL)
682                 return;
683
684         mtu = kiblnd_translate_mtu(*kiblnd_tunables.kib_ib_mtu);
685         LASSERT (mtu >= 0);
686         if (mtu != 0)
687                 cmid->route.path_rec->mtu = mtu;
688 }
689
690 #ifdef HAVE_OFED_IB_COMP_VECTOR
691 static int
692 kiblnd_get_completion_vector(kib_conn_t *conn, int cpt)
693 {
694         cpumask_t       *mask;
695         int             vectors;
696         int             off;
697         int             i;
698
699         vectors = conn->ibc_cmid->device->num_comp_vectors;
700         if (vectors <= 1)
701                 return 0;
702
703         mask = cfs_cpt_cpumask(lnet_cpt_table(), cpt);
704
705         /* hash NID to CPU id in this partition... */
706         off = conn->ibc_peer->ibp_nid % cpus_weight(*mask);
707         for_each_cpu_mask(i, *mask) {
708                 if (off-- == 0)
709                         return i % vectors;
710         }
711
712         LBUG();
713         return 1;
714 }
715 #endif
716
717 kib_conn_t *
718 kiblnd_create_conn(kib_peer_t *peer, struct rdma_cm_id *cmid,
719                    int state, int version)
720 {
721         /* CAVEAT EMPTOR:
722          * If the new conn is created successfully it takes over the caller's
723          * ref on 'peer'.  It also "owns" 'cmid' and destroys it when it itself
724          * is destroyed.  On failure, the caller's ref on 'peer' remains and
725          * she must dispose of 'cmid'.  (Actually I'd block forever if I tried
726          * to destroy 'cmid' here since I'm called from the CM which still has
727          * its ref on 'cmid'). */
728         cfs_rwlock_t           *glock = &kiblnd_data.kib_global_lock;
729         kib_net_t              *net = peer->ibp_ni->ni_data;
730         kib_dev_t              *dev = net->ibn_dev;
731         struct ib_qp_init_attr *init_qp_attr;
732         struct kib_sched_info   *sched;
733         kib_conn_t              *conn;
734         struct ib_cq            *cq;
735         unsigned long           flags;
736         int                     cpt;
737         int                     rc;
738         int                     i;
739
740         LASSERT(net != NULL);
741         LASSERT(!cfs_in_interrupt());
742
743         cpt = lnet_cpt_of_nid(peer->ibp_nid);
744         sched = kiblnd_data.kib_scheds[cpt];
745
746         LASSERT(sched->ibs_nthreads > 0);
747
748         LIBCFS_CPT_ALLOC(init_qp_attr, lnet_cpt_table(), cpt,
749                          sizeof(*init_qp_attr));
750         if (init_qp_attr == NULL) {
751                 CERROR("Can't allocate qp_attr for %s\n",
752                        libcfs_nid2str(peer->ibp_nid));
753                 goto failed_0;
754         }
755
756         LIBCFS_CPT_ALLOC(conn, lnet_cpt_table(), cpt, sizeof(*conn));
757         if (conn == NULL) {
758                 CERROR("Can't allocate connection for %s\n",
759                        libcfs_nid2str(peer->ibp_nid));
760                 goto failed_1;
761         }
762
763         conn->ibc_state = IBLND_CONN_INIT;
764         conn->ibc_version = version;
765         conn->ibc_peer = peer;                  /* I take the caller's ref */
766         cmid->context = conn;                   /* for future CM callbacks */
767         conn->ibc_cmid = cmid;
768
769         CFS_INIT_LIST_HEAD(&conn->ibc_early_rxs);
770         CFS_INIT_LIST_HEAD(&conn->ibc_tx_noops);
771         CFS_INIT_LIST_HEAD(&conn->ibc_tx_queue);
772         CFS_INIT_LIST_HEAD(&conn->ibc_tx_queue_rsrvd);
773         CFS_INIT_LIST_HEAD(&conn->ibc_tx_queue_nocred);
774         CFS_INIT_LIST_HEAD(&conn->ibc_active_txs);
775         cfs_spin_lock_init(&conn->ibc_lock);
776
777         LIBCFS_CPT_ALLOC(conn->ibc_connvars, lnet_cpt_table(), cpt,
778                          sizeof(*conn->ibc_connvars));
779         if (conn->ibc_connvars == NULL) {
780                 CERROR("Can't allocate in-progress connection state\n");
781                 goto failed_2;
782         }
783
784         cfs_write_lock_irqsave(glock, flags);
785         if (dev->ibd_failover) {
786                 cfs_write_unlock_irqrestore(glock, flags);
787                 CERROR("%s: failover in progress\n", dev->ibd_ifname);
788                 goto failed_2;
789         }
790
791         if (dev->ibd_hdev->ibh_ibdev != cmid->device) {
792                 /* wakeup failover thread and teardown connection */
793                 if (kiblnd_dev_can_failover(dev)) {
794                         cfs_list_add_tail(&dev->ibd_fail_list,
795                                       &kiblnd_data.kib_failed_devs);
796                         cfs_waitq_signal(&kiblnd_data.kib_failover_waitq);
797                 }
798
799                 cfs_write_unlock_irqrestore(glock, flags);
800                 CERROR("cmid HCA(%s), kib_dev(%s) need failover\n",
801                        cmid->device->name, dev->ibd_ifname);
802                 goto failed_2;
803         }
804
805         kiblnd_hdev_addref_locked(dev->ibd_hdev);
806         conn->ibc_hdev = dev->ibd_hdev;
807
808         kiblnd_setup_mtu_locked(cmid);
809
810         cfs_write_unlock_irqrestore(glock, flags);
811
812         LIBCFS_CPT_ALLOC(conn->ibc_rxs, lnet_cpt_table(), cpt,
813                          IBLND_RX_MSGS(version) * sizeof(kib_rx_t));
814         if (conn->ibc_rxs == NULL) {
815                 CERROR("Cannot allocate RX buffers\n");
816                 goto failed_2;
817         }
818
819         rc = kiblnd_alloc_pages(&conn->ibc_rx_pages, cpt,
820                                 IBLND_RX_MSG_PAGES(version));
821         if (rc != 0)
822                 goto failed_2;
823
824         kiblnd_map_rx_descs(conn);
825
826 #ifdef HAVE_OFED_IB_COMP_VECTOR
827         cq = ib_create_cq(cmid->device,
828                           kiblnd_cq_completion, kiblnd_cq_event, conn,
829                           IBLND_CQ_ENTRIES(version),
830                           kiblnd_get_completion_vector(conn, cpt));
831 #else
832         cq = ib_create_cq(cmid->device,
833                           kiblnd_cq_completion, kiblnd_cq_event, conn,
834                           IBLND_CQ_ENTRIES(version));
835 #endif
836         if (IS_ERR(cq)) {
837                 CERROR("Can't create CQ: %ld, cqe: %d\n",
838                        PTR_ERR(cq), IBLND_CQ_ENTRIES(version));
839                 goto failed_2;
840         }
841
842         conn->ibc_cq = cq;
843
844         rc = ib_req_notify_cq(cq, IB_CQ_NEXT_COMP);
845         if (rc != 0) {
846                 CERROR("Can't request completion notificiation: %d\n", rc);
847                 goto failed_2;
848         }
849
850         init_qp_attr->event_handler = kiblnd_qp_event;
851         init_qp_attr->qp_context = conn;
852         init_qp_attr->cap.max_send_wr = IBLND_SEND_WRS(version);
853         init_qp_attr->cap.max_recv_wr = IBLND_RECV_WRS(version);
854         init_qp_attr->cap.max_send_sge = 1;
855         init_qp_attr->cap.max_recv_sge = 1;
856         init_qp_attr->sq_sig_type = IB_SIGNAL_REQ_WR;
857         init_qp_attr->qp_type = IB_QPT_RC;
858         init_qp_attr->send_cq = cq;
859         init_qp_attr->recv_cq = cq;
860
861         conn->ibc_sched = sched;
862
863         rc = rdma_create_qp(cmid, conn->ibc_hdev->ibh_pd, init_qp_attr);
864         if (rc != 0) {
865                 CERROR("Can't create QP: %d, send_wr: %d, recv_wr: %d\n",
866                        rc, init_qp_attr->cap.max_send_wr,
867                        init_qp_attr->cap.max_recv_wr);
868                 goto failed_2;
869         }
870
871         LIBCFS_FREE(init_qp_attr, sizeof(*init_qp_attr));
872
873         /* 1 ref for caller and each rxmsg */
874         cfs_atomic_set(&conn->ibc_refcount, 1 + IBLND_RX_MSGS(version));
875         conn->ibc_nrx = IBLND_RX_MSGS(version);
876
877         /* post receives */
878         for (i = 0; i < IBLND_RX_MSGS(version); i++) {
879                 rc = kiblnd_post_rx(&conn->ibc_rxs[i],
880                                     IBLND_POSTRX_NO_CREDIT);
881                 if (rc != 0) {
882                         CERROR("Can't post rxmsg: %d\n", rc);
883
884                         /* Make posted receives complete */
885                         kiblnd_abort_receives(conn);
886
887                         /* correct # of posted buffers
888                          * NB locking needed now I'm racing with completion */
889                         cfs_spin_lock_irqsave(&sched->ibs_lock, flags);
890                         conn->ibc_nrx -= IBLND_RX_MSGS(version) - i;
891                         cfs_spin_unlock_irqrestore(&sched->ibs_lock, flags);
892
893                         /* cmid will be destroyed by CM(ofed) after cm_callback
894                          * returned, so we can't refer it anymore
895                          * (by kiblnd_connd()->kiblnd_destroy_conn) */
896                         rdma_destroy_qp(conn->ibc_cmid);
897                         conn->ibc_cmid = NULL;
898
899                         /* Drop my own and unused rxbuffer refcounts */
900                         while (i++ <= IBLND_RX_MSGS(version))
901                                 kiblnd_conn_decref(conn);
902
903                         return NULL;
904                 }
905         }
906
907         /* Init successful! */
908         LASSERT (state == IBLND_CONN_ACTIVE_CONNECT ||
909                  state == IBLND_CONN_PASSIVE_WAIT);
910         conn->ibc_state = state;
911
912         /* 1 more conn */
913         cfs_atomic_inc(&net->ibn_nconns);
914         return conn;
915
916  failed_2:
917         kiblnd_destroy_conn(conn);
918  failed_1:
919         LIBCFS_FREE(init_qp_attr, sizeof(*init_qp_attr));
920  failed_0:
921         return NULL;
922 }
923
924 void
925 kiblnd_destroy_conn (kib_conn_t *conn)
926 {
927         struct rdma_cm_id *cmid = conn->ibc_cmid;
928         kib_peer_t        *peer = conn->ibc_peer;
929         int                rc;
930
931         LASSERT (!cfs_in_interrupt());
932         LASSERT (cfs_atomic_read(&conn->ibc_refcount) == 0);
933         LASSERT (cfs_list_empty(&conn->ibc_early_rxs));
934         LASSERT (cfs_list_empty(&conn->ibc_tx_noops));
935         LASSERT (cfs_list_empty(&conn->ibc_tx_queue));
936         LASSERT (cfs_list_empty(&conn->ibc_tx_queue_rsrvd));
937         LASSERT (cfs_list_empty(&conn->ibc_tx_queue_nocred));
938         LASSERT (cfs_list_empty(&conn->ibc_active_txs));
939         LASSERT (conn->ibc_noops_posted == 0);
940         LASSERT (conn->ibc_nsends_posted == 0);
941
942         switch (conn->ibc_state) {
943         default:
944                 /* conn must be completely disengaged from the network */
945                 LBUG();
946
947         case IBLND_CONN_DISCONNECTED:
948                 /* connvars should have been freed already */
949                 LASSERT (conn->ibc_connvars == NULL);
950                 break;
951
952         case IBLND_CONN_INIT:
953                 break;
954         }
955
956         /* conn->ibc_cmid might be destroyed by CM already */
957         if (cmid != NULL && cmid->qp != NULL)
958                 rdma_destroy_qp(cmid);
959
960         if (conn->ibc_cq != NULL) {
961                 rc = ib_destroy_cq(conn->ibc_cq);
962                 if (rc != 0)
963                         CWARN("Error destroying CQ: %d\n", rc);
964         }
965
966         if (conn->ibc_rx_pages != NULL)
967                 kiblnd_unmap_rx_descs(conn);
968
969         if (conn->ibc_rxs != NULL) {
970                 LIBCFS_FREE(conn->ibc_rxs,
971                             IBLND_RX_MSGS(conn->ibc_version) * sizeof(kib_rx_t));
972         }
973
974         if (conn->ibc_connvars != NULL)
975                 LIBCFS_FREE(conn->ibc_connvars, sizeof(*conn->ibc_connvars));
976
977         if (conn->ibc_hdev != NULL)
978                 kiblnd_hdev_decref(conn->ibc_hdev);
979
980         /* See CAVEAT EMPTOR above in kiblnd_create_conn */
981         if (conn->ibc_state != IBLND_CONN_INIT) {
982                 kib_net_t *net = peer->ibp_ni->ni_data;
983
984                 kiblnd_peer_decref(peer);
985                 rdma_destroy_id(cmid);
986                 cfs_atomic_dec(&net->ibn_nconns);
987         }
988
989         LIBCFS_FREE(conn, sizeof(*conn));
990 }
991
992 int
993 kiblnd_close_peer_conns_locked (kib_peer_t *peer, int why)
994 {
995         kib_conn_t             *conn;
996         cfs_list_t             *ctmp;
997         cfs_list_t             *cnxt;
998         int                     count = 0;
999
1000         cfs_list_for_each_safe (ctmp, cnxt, &peer->ibp_conns) {
1001                 conn = cfs_list_entry(ctmp, kib_conn_t, ibc_list);
1002
1003                 CDEBUG(D_NET, "Closing conn -> %s, "
1004                               "version: %x, reason: %d\n",
1005                        libcfs_nid2str(peer->ibp_nid),
1006                        conn->ibc_version, why);
1007
1008                 kiblnd_close_conn_locked(conn, why);
1009                 count++;
1010         }
1011
1012         return count;
1013 }
1014
1015 int
1016 kiblnd_close_stale_conns_locked (kib_peer_t *peer,
1017                                  int version, __u64 incarnation)
1018 {
1019         kib_conn_t             *conn;
1020         cfs_list_t             *ctmp;
1021         cfs_list_t             *cnxt;
1022         int                     count = 0;
1023
1024         cfs_list_for_each_safe (ctmp, cnxt, &peer->ibp_conns) {
1025                 conn = cfs_list_entry(ctmp, kib_conn_t, ibc_list);
1026
1027                 if (conn->ibc_version     == version &&
1028                     conn->ibc_incarnation == incarnation)
1029                         continue;
1030
1031                 CDEBUG(D_NET, "Closing stale conn -> %s version: %x, "
1032                               "incarnation:"LPX64"(%x, "LPX64")\n",
1033                        libcfs_nid2str(peer->ibp_nid),
1034                        conn->ibc_version, conn->ibc_incarnation,
1035                        version, incarnation);
1036
1037                 kiblnd_close_conn_locked(conn, -ESTALE);
1038                 count++;
1039         }
1040
1041         return count;
1042 }
1043
1044 int
1045 kiblnd_close_matching_conns (lnet_ni_t *ni, lnet_nid_t nid)
1046 {
1047         kib_peer_t             *peer;
1048         cfs_list_t             *ptmp;
1049         cfs_list_t             *pnxt;
1050         int                     lo;
1051         int                     hi;
1052         int                     i;
1053         unsigned long           flags;
1054         int                     count = 0;
1055
1056         cfs_write_lock_irqsave(&kiblnd_data.kib_global_lock, flags);
1057
1058         if (nid != LNET_NID_ANY)
1059                 lo = hi = kiblnd_nid2peerlist(nid) - kiblnd_data.kib_peers;
1060         else {
1061                 lo = 0;
1062                 hi = kiblnd_data.kib_peer_hash_size - 1;
1063         }
1064
1065         for (i = lo; i <= hi; i++) {
1066                 cfs_list_for_each_safe (ptmp, pnxt, &kiblnd_data.kib_peers[i]) {
1067
1068                         peer = cfs_list_entry(ptmp, kib_peer_t, ibp_list);
1069                         LASSERT (peer->ibp_connecting > 0 ||
1070                                  peer->ibp_accepting > 0 ||
1071                                  !cfs_list_empty(&peer->ibp_conns));
1072
1073                         if (peer->ibp_ni != ni)
1074                                 continue;
1075
1076                         if (!(nid == LNET_NID_ANY || nid == peer->ibp_nid))
1077                                 continue;
1078
1079                         count += kiblnd_close_peer_conns_locked(peer, 0);
1080                 }
1081         }
1082
1083         cfs_write_unlock_irqrestore(&kiblnd_data.kib_global_lock, flags);
1084
1085         /* wildcards always succeed */
1086         if (nid == LNET_NID_ANY)
1087                 return 0;
1088
1089         return (count == 0) ? -ENOENT : 0;
1090 }
1091
1092 int
1093 kiblnd_ctl(lnet_ni_t *ni, unsigned int cmd, void *arg)
1094 {
1095         struct libcfs_ioctl_data *data = arg;
1096         int                       rc = -EINVAL;
1097
1098         switch(cmd) {
1099         case IOC_LIBCFS_GET_PEER: {
1100                 lnet_nid_t   nid = 0;
1101                 int          count = 0;
1102
1103                 rc = kiblnd_get_peer_info(ni, data->ioc_count,
1104                                           &nid, &count);
1105                 data->ioc_nid    = nid;
1106                 data->ioc_count  = count;
1107                 break;
1108         }
1109
1110         case IOC_LIBCFS_DEL_PEER: {
1111                 rc = kiblnd_del_peer(ni, data->ioc_nid);
1112                 break;
1113         }
1114         case IOC_LIBCFS_GET_CONN: {
1115                 kib_conn_t *conn;
1116
1117                 rc = 0;
1118                 conn = kiblnd_get_conn_by_idx(ni, data->ioc_count);
1119                 if (conn == NULL) {
1120                         rc = -ENOENT;
1121                         break;
1122                 }
1123
1124                 LASSERT (conn->ibc_cmid != NULL);
1125                 data->ioc_nid = conn->ibc_peer->ibp_nid;
1126                 if (conn->ibc_cmid->route.path_rec == NULL)
1127                         data->ioc_u32[0] = 0; /* iWarp has no path MTU */
1128                 else
1129                         data->ioc_u32[0] =
1130                         ib_mtu_enum_to_int(conn->ibc_cmid->route.path_rec->mtu);
1131                 kiblnd_conn_decref(conn);
1132                 break;
1133         }
1134         case IOC_LIBCFS_CLOSE_CONNECTION: {
1135                 rc = kiblnd_close_matching_conns(ni, data->ioc_nid);
1136                 break;
1137         }
1138
1139         default:
1140                 break;
1141         }
1142
1143         return rc;
1144 }
1145
1146 void
1147 kiblnd_query (lnet_ni_t *ni, lnet_nid_t nid, cfs_time_t *when)
1148 {
1149         cfs_time_t         last_alive = 0;
1150         cfs_time_t         now = cfs_time_current();
1151         cfs_rwlock_t      *glock = &kiblnd_data.kib_global_lock;
1152         kib_peer_t        *peer;
1153         unsigned long      flags;
1154
1155         cfs_read_lock_irqsave(glock, flags);
1156
1157         peer = kiblnd_find_peer_locked(nid);
1158         if (peer != NULL) {
1159                 LASSERT (peer->ibp_connecting > 0 || /* creating conns */
1160                          peer->ibp_accepting > 0 ||
1161                          !cfs_list_empty(&peer->ibp_conns));  /* active conn */
1162                 last_alive = peer->ibp_last_alive;
1163         }
1164
1165         cfs_read_unlock_irqrestore(glock, flags);
1166
1167         if (last_alive != 0)
1168                 *when = last_alive;
1169
1170         /* peer is not persistent in hash, trigger peer creation
1171          * and connection establishment with a NULL tx */
1172         if (peer == NULL)
1173                 kiblnd_launch_tx(ni, NULL, nid);
1174
1175         CDEBUG(D_NET, "Peer %s %p, alive %ld secs ago\n",
1176                libcfs_nid2str(nid), peer,
1177                last_alive ? cfs_duration_sec(now - last_alive) : -1);
1178         return;
1179 }
1180
1181 void
1182 kiblnd_free_pages(kib_pages_t *p)
1183 {
1184         int     npages = p->ibp_npages;
1185         int     i;
1186
1187         for (i = 0; i < npages; i++) {
1188                 if (p->ibp_pages[i] != NULL)
1189                         cfs_free_page(p->ibp_pages[i]);
1190         }
1191
1192         LIBCFS_FREE(p, offsetof(kib_pages_t, ibp_pages[npages]));
1193 }
1194
1195 int
1196 kiblnd_alloc_pages(kib_pages_t **pp, int cpt, int npages)
1197 {
1198         kib_pages_t     *p;
1199         int             i;
1200
1201         LIBCFS_CPT_ALLOC(p, lnet_cpt_table(), cpt,
1202                          offsetof(kib_pages_t, ibp_pages[npages]));
1203         if (p == NULL) {
1204                 CERROR("Can't allocate descriptor for %d pages\n", npages);
1205                 return -ENOMEM;
1206         }
1207
1208         memset(p, 0, offsetof(kib_pages_t, ibp_pages[npages]));
1209         p->ibp_npages = npages;
1210
1211         for (i = 0; i < npages; i++) {
1212                 p->ibp_pages[i] = cfs_page_cpt_alloc(lnet_cpt_table(), cpt,
1213                                                      CFS_ALLOC_IO);
1214                 if (p->ibp_pages[i] == NULL) {
1215                         CERROR("Can't allocate page %d of %d\n", i, npages);
1216                         kiblnd_free_pages(p);
1217                         return -ENOMEM;
1218                 }
1219         }
1220
1221         *pp = p;
1222         return 0;
1223 }
1224
1225 void
1226 kiblnd_unmap_rx_descs(kib_conn_t *conn)
1227 {
1228         kib_rx_t *rx;
1229         int       i;
1230
1231         LASSERT (conn->ibc_rxs != NULL);
1232         LASSERT (conn->ibc_hdev != NULL);
1233
1234         for (i = 0; i < IBLND_RX_MSGS(conn->ibc_version); i++) {
1235                 rx = &conn->ibc_rxs[i];
1236
1237                 LASSERT (rx->rx_nob >= 0); /* not posted */
1238
1239                 kiblnd_dma_unmap_single(conn->ibc_hdev->ibh_ibdev,
1240                                         KIBLND_UNMAP_ADDR(rx, rx_msgunmap,
1241                                                           rx->rx_msgaddr),
1242                                         IBLND_MSG_SIZE, DMA_FROM_DEVICE);
1243         }
1244
1245         kiblnd_free_pages(conn->ibc_rx_pages);
1246
1247         conn->ibc_rx_pages = NULL;
1248 }
1249
1250 void
1251 kiblnd_map_rx_descs(kib_conn_t *conn)
1252 {
1253         kib_rx_t       *rx;
1254         struct page    *pg;
1255         int             pg_off;
1256         int             ipg;
1257         int             i;
1258
1259         for (pg_off = ipg = i = 0;
1260              i < IBLND_RX_MSGS(conn->ibc_version); i++) {
1261                 pg = conn->ibc_rx_pages->ibp_pages[ipg];
1262                 rx = &conn->ibc_rxs[i];
1263
1264                 rx->rx_conn = conn;
1265                 rx->rx_msg = (kib_msg_t *)(((char *)page_address(pg)) + pg_off);
1266
1267                 rx->rx_msgaddr = kiblnd_dma_map_single(conn->ibc_hdev->ibh_ibdev,
1268                                                        rx->rx_msg, IBLND_MSG_SIZE,
1269                                                        DMA_FROM_DEVICE);
1270                 LASSERT (!kiblnd_dma_mapping_error(conn->ibc_hdev->ibh_ibdev,
1271                                                    rx->rx_msgaddr));
1272                 KIBLND_UNMAP_ADDR_SET(rx, rx_msgunmap, rx->rx_msgaddr);
1273
1274                 CDEBUG(D_NET,"rx %d: %p "LPX64"("LPX64")\n",
1275                        i, rx->rx_msg, rx->rx_msgaddr,
1276                        lnet_page2phys(pg) + pg_off);
1277
1278                 pg_off += IBLND_MSG_SIZE;
1279                 LASSERT (pg_off <= PAGE_SIZE);
1280
1281                 if (pg_off == PAGE_SIZE) {
1282                         pg_off = 0;
1283                         ipg++;
1284                         LASSERT (ipg <= IBLND_RX_MSG_PAGES(conn->ibc_version));
1285                 }
1286         }
1287 }
1288
1289 static void
1290 kiblnd_unmap_tx_pool(kib_tx_pool_t *tpo)
1291 {
1292         kib_hca_dev_t  *hdev = tpo->tpo_hdev;
1293         kib_tx_t       *tx;
1294         int             i;
1295
1296         LASSERT (tpo->tpo_pool.po_allocated == 0);
1297
1298         if (hdev == NULL)
1299                 return;
1300
1301         for (i = 0; i < tpo->tpo_pool.po_size; i++) {
1302                 tx = &tpo->tpo_tx_descs[i];
1303                 kiblnd_dma_unmap_single(hdev->ibh_ibdev,
1304                                         KIBLND_UNMAP_ADDR(tx, tx_msgunmap,
1305                                                           tx->tx_msgaddr),
1306                                         IBLND_MSG_SIZE, DMA_TO_DEVICE);
1307         }
1308
1309         kiblnd_hdev_decref(hdev);
1310         tpo->tpo_hdev = NULL;
1311 }
1312
1313 static kib_hca_dev_t *
1314 kiblnd_current_hdev(kib_dev_t *dev)
1315 {
1316         kib_hca_dev_t *hdev;
1317         unsigned long  flags;
1318         int            i = 0;
1319
1320         cfs_read_lock_irqsave(&kiblnd_data.kib_global_lock, flags);
1321         while (dev->ibd_failover) {
1322                 cfs_read_unlock_irqrestore(&kiblnd_data.kib_global_lock, flags);
1323                 if (i++ % 50 == 0)
1324                         CDEBUG(D_NET, "Wait for dev(%s) failover\n", dev->ibd_ifname);
1325                 cfs_schedule_timeout(cfs_time_seconds(1) / 100);
1326
1327                 cfs_read_lock_irqsave(&kiblnd_data.kib_global_lock, flags);
1328         }
1329
1330         kiblnd_hdev_addref_locked(dev->ibd_hdev);
1331         hdev = dev->ibd_hdev;
1332
1333         cfs_read_unlock_irqrestore(&kiblnd_data.kib_global_lock, flags);
1334
1335         return hdev;
1336 }
1337
1338 static void
1339 kiblnd_map_tx_pool(kib_tx_pool_t *tpo)
1340 {
1341         kib_pages_t    *txpgs = tpo->tpo_tx_pages;
1342         kib_pool_t     *pool  = &tpo->tpo_pool;
1343         kib_net_t      *net   = pool->po_owner->ps_net;
1344         kib_dev_t      *dev   = net->ibn_dev;
1345         struct page    *page;
1346         kib_tx_t       *tx;
1347         int             page_offset;
1348         int             ipage;
1349         int             i;
1350
1351         LASSERT (net != NULL);
1352
1353         /* pre-mapped messages are not bigger than 1 page */
1354         CLASSERT (IBLND_MSG_SIZE <= PAGE_SIZE);
1355
1356         /* No fancy arithmetic when we do the buffer calculations */
1357         CLASSERT (PAGE_SIZE % IBLND_MSG_SIZE == 0);
1358
1359         tpo->tpo_hdev = kiblnd_current_hdev(dev);
1360
1361         for (ipage = page_offset = i = 0; i < pool->po_size; i++) {
1362                 page = txpgs->ibp_pages[ipage];
1363                 tx = &tpo->tpo_tx_descs[i];
1364
1365                 tx->tx_msg = (kib_msg_t *)(((char *)page_address(page)) +
1366                                            page_offset);
1367
1368                 tx->tx_msgaddr = kiblnd_dma_map_single(
1369                         tpo->tpo_hdev->ibh_ibdev, tx->tx_msg,
1370                         IBLND_MSG_SIZE, DMA_TO_DEVICE);
1371                 LASSERT (!kiblnd_dma_mapping_error(tpo->tpo_hdev->ibh_ibdev,
1372                                                    tx->tx_msgaddr));
1373                 KIBLND_UNMAP_ADDR_SET(tx, tx_msgunmap, tx->tx_msgaddr);
1374
1375                 cfs_list_add(&tx->tx_list, &pool->po_free_list);
1376
1377                 page_offset += IBLND_MSG_SIZE;
1378                 LASSERT (page_offset <= PAGE_SIZE);
1379
1380                 if (page_offset == PAGE_SIZE) {
1381                         page_offset = 0;
1382                         ipage++;
1383                         LASSERT (ipage <= txpgs->ibp_npages);
1384                 }
1385         }
1386 }
1387
1388 struct ib_mr *
1389 kiblnd_find_dma_mr(kib_hca_dev_t *hdev, __u64 addr, __u64 size)
1390 {
1391         __u64   index;
1392
1393         LASSERT (hdev->ibh_mrs[0] != NULL);
1394
1395         if (hdev->ibh_nmrs == 1)
1396                 return hdev->ibh_mrs[0];
1397
1398         index = addr >> hdev->ibh_mr_shift;
1399
1400         if (index <  hdev->ibh_nmrs &&
1401             index == ((addr + size - 1) >> hdev->ibh_mr_shift))
1402                 return hdev->ibh_mrs[index];
1403
1404         return NULL;
1405 }
1406
1407 struct ib_mr *
1408 kiblnd_find_rd_dma_mr(kib_hca_dev_t *hdev, kib_rdma_desc_t *rd)
1409 {
1410         struct ib_mr *prev_mr;
1411         struct ib_mr *mr;
1412         int           i;
1413
1414         LASSERT (hdev->ibh_mrs[0] != NULL);
1415
1416         if (*kiblnd_tunables.kib_map_on_demand > 0 &&
1417             *kiblnd_tunables.kib_map_on_demand <= rd->rd_nfrags)
1418                 return NULL;
1419
1420         if (hdev->ibh_nmrs == 1)
1421                 return hdev->ibh_mrs[0];
1422
1423         for (i = 0, mr = prev_mr = NULL;
1424              i < rd->rd_nfrags; i++) {
1425                 mr = kiblnd_find_dma_mr(hdev,
1426                                         rd->rd_frags[i].rf_addr,
1427                                         rd->rd_frags[i].rf_nob);
1428                 if (prev_mr == NULL)
1429                         prev_mr = mr;
1430
1431                 if (mr == NULL || prev_mr != mr) {
1432                         /* Can't covered by one single MR */
1433                         mr = NULL;
1434                         break;
1435                 }
1436         }
1437
1438         return mr;
1439 }
1440
1441 void
1442 kiblnd_destroy_fmr_pool(kib_fmr_pool_t *pool)
1443 {
1444         LASSERT (pool->fpo_map_count == 0);
1445
1446         if (pool->fpo_fmr_pool != NULL)
1447                 ib_destroy_fmr_pool(pool->fpo_fmr_pool);
1448
1449         if (pool->fpo_hdev != NULL)
1450                 kiblnd_hdev_decref(pool->fpo_hdev);
1451
1452         LIBCFS_FREE(pool, sizeof(kib_fmr_pool_t));
1453 }
1454
1455 void
1456 kiblnd_destroy_fmr_pool_list(cfs_list_t *head)
1457 {
1458         kib_fmr_pool_t *pool;
1459
1460         while (!cfs_list_empty(head)) {
1461                 pool = cfs_list_entry(head->next, kib_fmr_pool_t, fpo_list);
1462                 cfs_list_del(&pool->fpo_list);
1463                 kiblnd_destroy_fmr_pool(pool);
1464         }
1465 }
1466
1467 static int kiblnd_fmr_pool_size(int ncpts)
1468 {
1469         int size = *kiblnd_tunables.kib_fmr_pool_size / ncpts;
1470
1471         return max(IBLND_FMR_POOL, size);
1472 }
1473
1474 static int kiblnd_fmr_flush_trigger(int ncpts)
1475 {
1476         int size = *kiblnd_tunables.kib_fmr_flush_trigger / ncpts;
1477
1478         return max(IBLND_FMR_POOL_FLUSH, size);
1479 }
1480
1481 int
1482 kiblnd_create_fmr_pool(kib_fmr_poolset_t *fps, kib_fmr_pool_t **pp_fpo)
1483 {
1484         /* FMR pool for RDMA */
1485         kib_dev_t               *dev = fps->fps_net->ibn_dev;
1486         kib_fmr_pool_t          *fpo;
1487         struct ib_fmr_pool_param param = {
1488                 .max_pages_per_fmr = LNET_MAX_PAYLOAD/PAGE_SIZE,
1489                 .page_shift        = PAGE_SHIFT,
1490                 .access            = (IB_ACCESS_LOCAL_WRITE |
1491                                       IB_ACCESS_REMOTE_WRITE),
1492                 .pool_size         = fps->fps_pool_size,
1493                 .dirty_watermark   = fps->fps_flush_trigger,
1494                 .flush_function    = NULL,
1495                 .flush_arg         = NULL,
1496                 .cache             = !!*kiblnd_tunables.kib_fmr_cache};
1497         int rc;
1498
1499         LIBCFS_CPT_ALLOC(fpo, lnet_cpt_table(), fps->fps_cpt, sizeof(*fpo));
1500         if (fpo == NULL)
1501                 return -ENOMEM;
1502
1503         fpo->fpo_hdev = kiblnd_current_hdev(dev);
1504
1505         fpo->fpo_fmr_pool = ib_create_fmr_pool(fpo->fpo_hdev->ibh_pd, &param);
1506         if (IS_ERR(fpo->fpo_fmr_pool)) {
1507                 rc = PTR_ERR(fpo->fpo_fmr_pool);
1508                 CERROR("Failed to create FMR pool: %d\n", rc);
1509
1510                 kiblnd_hdev_decref(fpo->fpo_hdev);
1511                 LIBCFS_FREE(fpo, sizeof(kib_fmr_pool_t));
1512                 return rc;
1513         }
1514
1515         fpo->fpo_deadline = cfs_time_shift(IBLND_POOL_DEADLINE);
1516         fpo->fpo_owner    = fps;
1517         *pp_fpo = fpo;
1518
1519         return 0;
1520 }
1521
1522 static void
1523 kiblnd_fail_fmr_poolset(kib_fmr_poolset_t *fps, cfs_list_t *zombies)
1524 {
1525         if (fps->fps_net == NULL) /* intialized? */
1526                 return;
1527
1528         cfs_spin_lock(&fps->fps_lock);
1529
1530         while (!cfs_list_empty(&fps->fps_pool_list)) {
1531                 kib_fmr_pool_t *fpo = cfs_list_entry(fps->fps_pool_list.next,
1532                                                  kib_fmr_pool_t, fpo_list);
1533                 fpo->fpo_failed = 1;
1534                 cfs_list_del(&fpo->fpo_list);
1535                 if (fpo->fpo_map_count == 0)
1536                         cfs_list_add(&fpo->fpo_list, zombies);
1537                 else
1538                         cfs_list_add(&fpo->fpo_list, &fps->fps_failed_pool_list);
1539         }
1540
1541         cfs_spin_unlock(&fps->fps_lock);
1542 }
1543
1544 static void
1545 kiblnd_fini_fmr_poolset(kib_fmr_poolset_t *fps)
1546 {
1547         if (fps->fps_net != NULL) { /* initialized? */
1548                 kiblnd_destroy_fmr_pool_list(&fps->fps_failed_pool_list);
1549                 kiblnd_destroy_fmr_pool_list(&fps->fps_pool_list);
1550         }
1551 }
1552
1553 static int
1554 kiblnd_init_fmr_poolset(kib_fmr_poolset_t *fps, int cpt, kib_net_t *net,
1555                         int pool_size, int flush_trigger)
1556 {
1557         kib_fmr_pool_t *fpo;
1558         int             rc;
1559
1560         memset(fps, 0, sizeof(kib_fmr_poolset_t));
1561
1562         fps->fps_net = net;
1563         fps->fps_cpt = cpt;
1564         fps->fps_pool_size = pool_size;
1565         fps->fps_flush_trigger = flush_trigger;
1566         cfs_spin_lock_init(&fps->fps_lock);
1567         CFS_INIT_LIST_HEAD(&fps->fps_pool_list);
1568         CFS_INIT_LIST_HEAD(&fps->fps_failed_pool_list);
1569
1570         rc = kiblnd_create_fmr_pool(fps, &fpo);
1571         if (rc == 0)
1572                 cfs_list_add_tail(&fpo->fpo_list, &fps->fps_pool_list);
1573
1574         return rc;
1575 }
1576
1577 static int
1578 kiblnd_fmr_pool_is_idle(kib_fmr_pool_t *fpo, cfs_time_t now)
1579 {
1580         if (fpo->fpo_map_count != 0) /* still in use */
1581                 return 0;
1582         if (fpo->fpo_failed)
1583                 return 1;
1584         return cfs_time_aftereq(now, fpo->fpo_deadline);
1585 }
1586
1587 void
1588 kiblnd_fmr_pool_unmap(kib_fmr_t *fmr, int status)
1589 {
1590         CFS_LIST_HEAD     (zombies);
1591         kib_fmr_pool_t    *fpo = fmr->fmr_pool;
1592         kib_fmr_poolset_t *fps = fpo->fpo_owner;
1593         cfs_time_t         now = cfs_time_current();
1594         kib_fmr_pool_t    *tmp;
1595         int                rc;
1596
1597         rc = ib_fmr_pool_unmap(fmr->fmr_pfmr);
1598         LASSERT (rc == 0);
1599
1600         if (status != 0) {
1601                 rc = ib_flush_fmr_pool(fpo->fpo_fmr_pool);
1602                 LASSERT (rc == 0);
1603         }
1604
1605         fmr->fmr_pool = NULL;
1606         fmr->fmr_pfmr = NULL;
1607
1608         cfs_spin_lock(&fps->fps_lock);
1609         fpo->fpo_map_count --;  /* decref the pool */
1610
1611         cfs_list_for_each_entry_safe(fpo, tmp, &fps->fps_pool_list, fpo_list) {
1612                 /* the first pool is persistent */
1613                 if (fps->fps_pool_list.next == &fpo->fpo_list)
1614                         continue;
1615
1616                 if (kiblnd_fmr_pool_is_idle(fpo, now)) {
1617                         cfs_list_move(&fpo->fpo_list, &zombies);
1618                         fps->fps_version ++;
1619                 }
1620         }
1621         cfs_spin_unlock(&fps->fps_lock);
1622
1623         if (!cfs_list_empty(&zombies))
1624                 kiblnd_destroy_fmr_pool_list(&zombies);
1625 }
1626
1627 int
1628 kiblnd_fmr_pool_map(kib_fmr_poolset_t *fps, __u64 *pages, int npages,
1629                     __u64 iov, kib_fmr_t *fmr)
1630 {
1631         struct ib_pool_fmr *pfmr;
1632         kib_fmr_pool_t     *fpo;
1633         __u64               version;
1634         int                 rc;
1635
1636  again:
1637         cfs_spin_lock(&fps->fps_lock);
1638         version = fps->fps_version;
1639         cfs_list_for_each_entry(fpo, &fps->fps_pool_list, fpo_list) {
1640                 fpo->fpo_deadline = cfs_time_shift(IBLND_POOL_DEADLINE);
1641                 fpo->fpo_map_count ++;
1642                 cfs_spin_unlock(&fps->fps_lock);
1643
1644                 pfmr = ib_fmr_pool_map_phys(fpo->fpo_fmr_pool,
1645                                             pages, npages, iov);
1646                 if (likely(!IS_ERR(pfmr))) {
1647                         fmr->fmr_pool = fpo;
1648                         fmr->fmr_pfmr = pfmr;
1649                         return 0;
1650                 }
1651
1652                 cfs_spin_lock(&fps->fps_lock);
1653                 fpo->fpo_map_count --;
1654                 if (PTR_ERR(pfmr) != -EAGAIN) {
1655                         cfs_spin_unlock(&fps->fps_lock);
1656                         return PTR_ERR(pfmr);
1657                 }
1658
1659                 /* EAGAIN and ... */
1660                 if (version != fps->fps_version) {
1661                         cfs_spin_unlock(&fps->fps_lock);
1662                         goto again;
1663                 }
1664         }
1665
1666         if (fps->fps_increasing) {
1667                 cfs_spin_unlock(&fps->fps_lock);
1668                 CDEBUG(D_NET, "Another thread is allocating new "
1669                               "FMR pool, waiting for her to complete\n");
1670                 cfs_schedule();
1671                 goto again;
1672
1673         }
1674
1675         if (cfs_time_before(cfs_time_current(), fps->fps_next_retry)) {
1676                 /* someone failed recently */
1677                 cfs_spin_unlock(&fps->fps_lock);
1678                 return -EAGAIN;
1679         }
1680
1681         fps->fps_increasing = 1;
1682         cfs_spin_unlock(&fps->fps_lock);
1683
1684         CDEBUG(D_NET, "Allocate new FMR pool\n");
1685         rc = kiblnd_create_fmr_pool(fps, &fpo);
1686         cfs_spin_lock(&fps->fps_lock);
1687         fps->fps_increasing = 0;
1688         if (rc == 0) {
1689                 fps->fps_version ++;
1690                 cfs_list_add_tail(&fpo->fpo_list, &fps->fps_pool_list);
1691         } else {
1692                 fps->fps_next_retry = cfs_time_shift(IBLND_POOL_RETRY);
1693         }
1694         cfs_spin_unlock(&fps->fps_lock);
1695
1696         goto again;
1697 }
1698
1699 static void
1700 kiblnd_fini_pool(kib_pool_t *pool)
1701 {
1702         LASSERT (cfs_list_empty(&pool->po_free_list));
1703         LASSERT (pool->po_allocated == 0);
1704
1705         CDEBUG(D_NET, "Finalize %s pool\n", pool->po_owner->ps_name);
1706 }
1707
1708 static void
1709 kiblnd_init_pool(kib_poolset_t *ps, kib_pool_t *pool, int size)
1710 {
1711         CDEBUG(D_NET, "Initialize %s pool\n", ps->ps_name);
1712
1713         memset(pool, 0, sizeof(kib_pool_t));
1714         CFS_INIT_LIST_HEAD(&pool->po_free_list);
1715         pool->po_deadline = cfs_time_shift(IBLND_POOL_DEADLINE);
1716         pool->po_owner    = ps;
1717         pool->po_size     = size;
1718 }
1719
1720 void
1721 kiblnd_destroy_pool_list(cfs_list_t *head)
1722 {
1723         kib_pool_t *pool;
1724
1725         while (!cfs_list_empty(head)) {
1726                 pool = cfs_list_entry(head->next, kib_pool_t, po_list);
1727                 cfs_list_del(&pool->po_list);
1728
1729                 LASSERT (pool->po_owner != NULL);
1730                 pool->po_owner->ps_pool_destroy(pool);
1731         }
1732 }
1733
1734 static void
1735 kiblnd_fail_poolset(kib_poolset_t *ps, cfs_list_t *zombies)
1736 {
1737         if (ps->ps_net == NULL) /* intialized? */
1738                 return;
1739
1740         cfs_spin_lock(&ps->ps_lock);
1741         while (!cfs_list_empty(&ps->ps_pool_list)) {
1742                 kib_pool_t *po = cfs_list_entry(ps->ps_pool_list.next,
1743                                             kib_pool_t, po_list);
1744                 po->po_failed = 1;
1745                 cfs_list_del(&po->po_list);
1746                 if (po->po_allocated == 0)
1747                         cfs_list_add(&po->po_list, zombies);
1748                 else
1749                         cfs_list_add(&po->po_list, &ps->ps_failed_pool_list);
1750         }
1751         cfs_spin_unlock(&ps->ps_lock);
1752 }
1753
1754 static void
1755 kiblnd_fini_poolset(kib_poolset_t *ps)
1756 {
1757         if (ps->ps_net != NULL) { /* initialized? */
1758                 kiblnd_destroy_pool_list(&ps->ps_failed_pool_list);
1759                 kiblnd_destroy_pool_list(&ps->ps_pool_list);
1760         }
1761 }
1762
1763 static int
1764 kiblnd_init_poolset(kib_poolset_t *ps, int cpt,
1765                     kib_net_t *net, char *name, int size,
1766                     kib_ps_pool_create_t po_create,
1767                     kib_ps_pool_destroy_t po_destroy,
1768                     kib_ps_node_init_t nd_init,
1769                     kib_ps_node_fini_t nd_fini)
1770 {
1771         kib_pool_t      *pool;
1772         int             rc;
1773
1774         memset(ps, 0, sizeof(kib_poolset_t));
1775
1776         ps->ps_cpt          = cpt;
1777         ps->ps_net          = net;
1778         ps->ps_pool_create  = po_create;
1779         ps->ps_pool_destroy = po_destroy;
1780         ps->ps_node_init    = nd_init;
1781         ps->ps_node_fini    = nd_fini;
1782         ps->ps_pool_size    = size;
1783         strncpy(ps->ps_name, name, IBLND_POOL_NAME_LEN);
1784         cfs_spin_lock_init(&ps->ps_lock);
1785         CFS_INIT_LIST_HEAD(&ps->ps_pool_list);
1786         CFS_INIT_LIST_HEAD(&ps->ps_failed_pool_list);
1787
1788         rc = ps->ps_pool_create(ps, size, &pool);
1789         if (rc == 0)
1790                 cfs_list_add(&pool->po_list, &ps->ps_pool_list);
1791         else
1792                 CERROR("Failed to create the first pool for %s\n", ps->ps_name);
1793
1794         return rc;
1795 }
1796
1797 static int
1798 kiblnd_pool_is_idle(kib_pool_t *pool, cfs_time_t now)
1799 {
1800         if (pool->po_allocated != 0) /* still in use */
1801                 return 0;
1802         if (pool->po_failed)
1803                 return 1;
1804         return cfs_time_aftereq(now, pool->po_deadline);
1805 }
1806
1807 void
1808 kiblnd_pool_free_node(kib_pool_t *pool, cfs_list_t *node)
1809 {
1810         CFS_LIST_HEAD  (zombies);
1811         kib_poolset_t  *ps = pool->po_owner;
1812         kib_pool_t     *tmp;
1813         cfs_time_t      now = cfs_time_current();
1814
1815         cfs_spin_lock(&ps->ps_lock);
1816
1817         if (ps->ps_node_fini != NULL)
1818                 ps->ps_node_fini(pool, node);
1819
1820         LASSERT (pool->po_allocated > 0);
1821         cfs_list_add(node, &pool->po_free_list);
1822         pool->po_allocated --;
1823
1824         cfs_list_for_each_entry_safe(pool, tmp, &ps->ps_pool_list, po_list) {
1825                 /* the first pool is persistent */
1826                 if (ps->ps_pool_list.next == &pool->po_list)
1827                         continue;
1828
1829                 if (kiblnd_pool_is_idle(pool, now))
1830                         cfs_list_move(&pool->po_list, &zombies);
1831         }
1832         cfs_spin_unlock(&ps->ps_lock);
1833
1834         if (!cfs_list_empty(&zombies))
1835                 kiblnd_destroy_pool_list(&zombies);
1836 }
1837
1838 cfs_list_t *
1839 kiblnd_pool_alloc_node(kib_poolset_t *ps)
1840 {
1841         cfs_list_t            *node;
1842         kib_pool_t            *pool;
1843         int                    rc;
1844
1845  again:
1846         cfs_spin_lock(&ps->ps_lock);
1847         cfs_list_for_each_entry(pool, &ps->ps_pool_list, po_list) {
1848                 if (cfs_list_empty(&pool->po_free_list))
1849                         continue;
1850
1851                 pool->po_allocated ++;
1852                 pool->po_deadline = cfs_time_shift(IBLND_POOL_DEADLINE);
1853                 node = pool->po_free_list.next;
1854                 cfs_list_del(node);
1855
1856                 if (ps->ps_node_init != NULL) {
1857                         /* still hold the lock */
1858                         ps->ps_node_init(pool, node);
1859                 }
1860                 cfs_spin_unlock(&ps->ps_lock);
1861                 return node;
1862         }
1863
1864         /* no available tx pool and ... */
1865         if (ps->ps_increasing) {
1866                 /* another thread is allocating a new pool */
1867                 cfs_spin_unlock(&ps->ps_lock);
1868                 CDEBUG(D_NET, "Another thread is allocating new "
1869                        "%s pool, waiting for her to complete\n",
1870                        ps->ps_name);
1871                 cfs_schedule();
1872                 goto again;
1873         }
1874
1875         if (cfs_time_before(cfs_time_current(), ps->ps_next_retry)) {
1876                 /* someone failed recently */
1877                 cfs_spin_unlock(&ps->ps_lock);
1878                 return NULL;
1879         }
1880
1881         ps->ps_increasing = 1;
1882         cfs_spin_unlock(&ps->ps_lock);
1883
1884         CDEBUG(D_NET, "%s pool exhausted, allocate new pool\n", ps->ps_name);
1885
1886         rc = ps->ps_pool_create(ps, ps->ps_pool_size, &pool);
1887
1888         cfs_spin_lock(&ps->ps_lock);
1889         ps->ps_increasing = 0;
1890         if (rc == 0) {
1891                 cfs_list_add_tail(&pool->po_list, &ps->ps_pool_list);
1892         } else {
1893                 ps->ps_next_retry = cfs_time_shift(IBLND_POOL_RETRY);
1894                 CERROR("Can't allocate new %s pool because out of memory\n",
1895                        ps->ps_name);
1896         }
1897         cfs_spin_unlock(&ps->ps_lock);
1898
1899         goto again;
1900 }
1901
1902 void
1903 kiblnd_pmr_pool_unmap(kib_phys_mr_t *pmr)
1904 {
1905         kib_pmr_pool_t      *ppo = pmr->pmr_pool;
1906         struct ib_mr        *mr  = pmr->pmr_mr;
1907
1908         pmr->pmr_mr = NULL;
1909         kiblnd_pool_free_node(&ppo->ppo_pool, &pmr->pmr_list);
1910         if (mr != NULL)
1911                 ib_dereg_mr(mr);
1912 }
1913
1914 int
1915 kiblnd_pmr_pool_map(kib_pmr_poolset_t *pps, kib_hca_dev_t *hdev,
1916                     kib_rdma_desc_t *rd, __u64 *iova, kib_phys_mr_t **pp_pmr)
1917 {
1918         kib_phys_mr_t *pmr;
1919         cfs_list_t    *node;
1920         int            rc;
1921         int            i;
1922
1923         node = kiblnd_pool_alloc_node(&pps->pps_poolset);
1924         if (node == NULL) {
1925                 CERROR("Failed to allocate PMR descriptor\n");
1926                 return -ENOMEM;
1927         }
1928
1929         pmr = container_of(node, kib_phys_mr_t, pmr_list);
1930         if (pmr->pmr_pool->ppo_hdev != hdev) {
1931                 kiblnd_pool_free_node(&pmr->pmr_pool->ppo_pool, node);
1932                 return -EAGAIN;
1933         }
1934
1935         for (i = 0; i < rd->rd_nfrags; i ++) {
1936                 pmr->pmr_ipb[i].addr = rd->rd_frags[i].rf_addr;
1937                 pmr->pmr_ipb[i].size = rd->rd_frags[i].rf_nob;
1938         }
1939
1940         pmr->pmr_mr = ib_reg_phys_mr(hdev->ibh_pd,
1941                                      pmr->pmr_ipb, rd->rd_nfrags,
1942                                      IB_ACCESS_LOCAL_WRITE |
1943                                      IB_ACCESS_REMOTE_WRITE,
1944                                      iova);
1945         if (!IS_ERR(pmr->pmr_mr)) {
1946                 pmr->pmr_iova = *iova;
1947                 *pp_pmr = pmr;
1948                 return 0;
1949         }
1950
1951         rc = PTR_ERR(pmr->pmr_mr);
1952         CERROR("Failed ib_reg_phys_mr: %d\n", rc);
1953
1954         pmr->pmr_mr = NULL;
1955         kiblnd_pool_free_node(&pmr->pmr_pool->ppo_pool, node);
1956
1957         return rc;
1958 }
1959
1960 static void
1961 kiblnd_destroy_pmr_pool(kib_pool_t *pool)
1962 {
1963         kib_pmr_pool_t *ppo = container_of(pool, kib_pmr_pool_t, ppo_pool);
1964         kib_phys_mr_t  *pmr;
1965
1966         LASSERT (pool->po_allocated == 0);
1967
1968         while (!cfs_list_empty(&pool->po_free_list)) {
1969                 pmr = cfs_list_entry(pool->po_free_list.next,
1970                                      kib_phys_mr_t, pmr_list);
1971
1972                 LASSERT (pmr->pmr_mr == NULL);
1973                 cfs_list_del(&pmr->pmr_list);
1974
1975                 if (pmr->pmr_ipb != NULL) {
1976                         LIBCFS_FREE(pmr->pmr_ipb,
1977                                     IBLND_MAX_RDMA_FRAGS *
1978                                     sizeof(struct ib_phys_buf));
1979                 }
1980
1981                 LIBCFS_FREE(pmr, sizeof(kib_phys_mr_t));
1982         }
1983
1984         kiblnd_fini_pool(pool);
1985         if (ppo->ppo_hdev != NULL)
1986                 kiblnd_hdev_decref(ppo->ppo_hdev);
1987
1988         LIBCFS_FREE(ppo, sizeof(kib_pmr_pool_t));
1989 }
1990
1991 static inline int kiblnd_pmr_pool_size(int ncpts)
1992 {
1993         int size = *kiblnd_tunables.kib_pmr_pool_size / ncpts;
1994
1995         return max(IBLND_PMR_POOL, size);
1996 }
1997
1998 static int
1999 kiblnd_create_pmr_pool(kib_poolset_t *ps, int size, kib_pool_t **pp_po)
2000 {
2001         struct kib_pmr_pool     *ppo;
2002         struct kib_pool         *pool;
2003         kib_phys_mr_t           *pmr;
2004         int                     i;
2005
2006         LIBCFS_CPT_ALLOC(ppo, lnet_cpt_table(),
2007                          ps->ps_cpt, sizeof(kib_pmr_pool_t));
2008         if (ppo == NULL) {
2009                 CERROR("Failed to allocate PMR pool\n");
2010                 return -ENOMEM;
2011         }
2012
2013         pool = &ppo->ppo_pool;
2014         kiblnd_init_pool(ps, pool, size);
2015
2016         for (i = 0; i < size; i++) {
2017                 LIBCFS_CPT_ALLOC(pmr, lnet_cpt_table(),
2018                                  ps->ps_cpt, sizeof(kib_phys_mr_t));
2019                 if (pmr == NULL)
2020                         break;
2021
2022                 pmr->pmr_pool = ppo;
2023                 LIBCFS_CPT_ALLOC(pmr->pmr_ipb, lnet_cpt_table(), ps->ps_cpt,
2024                                  IBLND_MAX_RDMA_FRAGS * sizeof(*pmr->pmr_ipb));
2025                 if (pmr->pmr_ipb == NULL)
2026                         break;
2027
2028                 cfs_list_add(&pmr->pmr_list, &pool->po_free_list);
2029         }
2030
2031         if (i < size) {
2032                 ps->ps_pool_destroy(pool);
2033                 return -ENOMEM;
2034         }
2035
2036         ppo->ppo_hdev = kiblnd_current_hdev(ps->ps_net->ibn_dev);
2037         *pp_po = pool;
2038         return 0;
2039 }
2040
2041 static void
2042 kiblnd_destroy_tx_pool(kib_pool_t *pool)
2043 {
2044         kib_tx_pool_t  *tpo = container_of(pool, kib_tx_pool_t, tpo_pool);
2045         int             i;
2046
2047         LASSERT (pool->po_allocated == 0);
2048
2049         if (tpo->tpo_tx_pages != NULL) {
2050                 kiblnd_unmap_tx_pool(tpo);
2051                 kiblnd_free_pages(tpo->tpo_tx_pages);
2052         }
2053
2054         if (tpo->tpo_tx_descs == NULL)
2055                 goto out;
2056
2057         for (i = 0; i < pool->po_size; i++) {
2058                 kib_tx_t *tx = &tpo->tpo_tx_descs[i];
2059
2060                 cfs_list_del(&tx->tx_list);
2061                 if (tx->tx_pages != NULL)
2062                         LIBCFS_FREE(tx->tx_pages,
2063                                     LNET_MAX_IOV *
2064                                     sizeof(*tx->tx_pages));
2065                 if (tx->tx_frags != NULL)
2066                         LIBCFS_FREE(tx->tx_frags,
2067                                     IBLND_MAX_RDMA_FRAGS *
2068                                             sizeof(*tx->tx_frags));
2069                 if (tx->tx_wrq != NULL)
2070                         LIBCFS_FREE(tx->tx_wrq,
2071                                     (1 + IBLND_MAX_RDMA_FRAGS) *
2072                                     sizeof(*tx->tx_wrq));
2073                 if (tx->tx_sge != NULL)
2074                         LIBCFS_FREE(tx->tx_sge,
2075                                     (1 + IBLND_MAX_RDMA_FRAGS) *
2076                                     sizeof(*tx->tx_sge));
2077                 if (tx->tx_rd != NULL)
2078                         LIBCFS_FREE(tx->tx_rd,
2079                                     offsetof(kib_rdma_desc_t,
2080                                              rd_frags[IBLND_MAX_RDMA_FRAGS]));
2081         }
2082
2083         LIBCFS_FREE(tpo->tpo_tx_descs,
2084                     pool->po_size * sizeof(kib_tx_t));
2085 out:
2086         kiblnd_fini_pool(pool);
2087         LIBCFS_FREE(tpo, sizeof(kib_tx_pool_t));
2088 }
2089
2090 static int kiblnd_tx_pool_size(int ncpts)
2091 {
2092         int ntx = *kiblnd_tunables.kib_ntx / ncpts;
2093
2094         return max(IBLND_TX_POOL, ntx);
2095 }
2096
2097 static int
2098 kiblnd_create_tx_pool(kib_poolset_t *ps, int size, kib_pool_t **pp_po)
2099 {
2100         int            i;
2101         int            npg;
2102         kib_pool_t    *pool;
2103         kib_tx_pool_t *tpo;
2104
2105         LIBCFS_CPT_ALLOC(tpo, lnet_cpt_table(), ps->ps_cpt, sizeof(*tpo));
2106         if (tpo == NULL) {
2107                 CERROR("Failed to allocate TX pool\n");
2108                 return -ENOMEM;
2109         }
2110
2111         pool = &tpo->tpo_pool;
2112         kiblnd_init_pool(ps, pool, size);
2113         tpo->tpo_tx_descs = NULL;
2114         tpo->tpo_tx_pages = NULL;
2115
2116         npg = (size * IBLND_MSG_SIZE + PAGE_SIZE - 1) / PAGE_SIZE;
2117         if (kiblnd_alloc_pages(&tpo->tpo_tx_pages, ps->ps_cpt, npg) != 0) {
2118                 CERROR("Can't allocate tx pages: %d\n", npg);
2119                 LIBCFS_FREE(tpo, sizeof(kib_tx_pool_t));
2120                 return -ENOMEM;
2121         }
2122
2123         LIBCFS_CPT_ALLOC(tpo->tpo_tx_descs, lnet_cpt_table(), ps->ps_cpt,
2124                          size * sizeof(kib_tx_t));
2125         if (tpo->tpo_tx_descs == NULL) {
2126                 CERROR("Can't allocate %d tx descriptors\n", size);
2127                 ps->ps_pool_destroy(pool);
2128                 return -ENOMEM;
2129         }
2130
2131         memset(tpo->tpo_tx_descs, 0, size * sizeof(kib_tx_t));
2132
2133         for (i = 0; i < size; i++) {
2134                 kib_tx_t *tx = &tpo->tpo_tx_descs[i];
2135
2136                 tx->tx_pool = tpo;
2137                 if (ps->ps_net->ibn_fmr_ps != NULL) {
2138                         LIBCFS_CPT_ALLOC(tx->tx_pages,
2139                                          lnet_cpt_table(), ps->ps_cpt,
2140                                          LNET_MAX_IOV * sizeof(*tx->tx_pages));
2141                         if (tx->tx_pages == NULL)
2142                                 break;
2143                 }
2144
2145                 LIBCFS_CPT_ALLOC(tx->tx_frags, lnet_cpt_table(), ps->ps_cpt,
2146                                  IBLND_MAX_RDMA_FRAGS * sizeof(*tx->tx_frags));
2147                 if (tx->tx_frags == NULL)
2148                         break;
2149
2150                 sg_init_table(tx->tx_frags, IBLND_MAX_RDMA_FRAGS);
2151
2152                 LIBCFS_CPT_ALLOC(tx->tx_wrq, lnet_cpt_table(), ps->ps_cpt,
2153                                  (1 + IBLND_MAX_RDMA_FRAGS) *
2154                                  sizeof(*tx->tx_wrq));
2155                 if (tx->tx_wrq == NULL)
2156                         break;
2157
2158                 LIBCFS_CPT_ALLOC(tx->tx_sge, lnet_cpt_table(), ps->ps_cpt,
2159                                  (1 + IBLND_MAX_RDMA_FRAGS) *
2160                                  sizeof(*tx->tx_sge));
2161                 if (tx->tx_sge == NULL)
2162                         break;
2163
2164                 LIBCFS_CPT_ALLOC(tx->tx_rd, lnet_cpt_table(), ps->ps_cpt,
2165                                  offsetof(kib_rdma_desc_t,
2166                                           rd_frags[IBLND_MAX_RDMA_FRAGS]));
2167                 if (tx->tx_rd == NULL)
2168                         break;
2169         }
2170
2171         if (i == size) {
2172                 kiblnd_map_tx_pool(tpo);
2173                 *pp_po = pool;
2174                 return 0;
2175         }
2176
2177         ps->ps_pool_destroy(pool);
2178         return -ENOMEM;
2179 }
2180
2181 static void
2182 kiblnd_tx_init(kib_pool_t *pool, cfs_list_t *node)
2183 {
2184         kib_tx_poolset_t *tps = container_of(pool->po_owner, kib_tx_poolset_t,
2185                                              tps_poolset);
2186         kib_tx_t         *tx  = cfs_list_entry(node, kib_tx_t, tx_list);
2187
2188         tx->tx_cookie = tps->tps_next_tx_cookie ++;
2189 }
2190
2191 void
2192 kiblnd_net_fini_pools(kib_net_t *net)
2193 {
2194         int     i;
2195
2196         cfs_cpt_for_each(i, lnet_cpt_table()) {
2197                 kib_tx_poolset_t        *tps;
2198                 kib_fmr_poolset_t       *fps;
2199                 kib_pmr_poolset_t       *pps;
2200
2201                 if (net->ibn_tx_ps != NULL) {
2202                         tps = net->ibn_tx_ps[i];
2203                         kiblnd_fini_poolset(&tps->tps_poolset);
2204                 }
2205
2206                 if (net->ibn_fmr_ps != NULL) {
2207                         fps = net->ibn_fmr_ps[i];
2208                         kiblnd_fini_fmr_poolset(fps);
2209                 }
2210
2211                 if (net->ibn_pmr_ps != NULL) {
2212                         pps = net->ibn_pmr_ps[i];
2213                         kiblnd_fini_poolset(&pps->pps_poolset);
2214                 }
2215         }
2216
2217         if (net->ibn_tx_ps != NULL) {
2218                 cfs_percpt_free(net->ibn_tx_ps);
2219                 net->ibn_tx_ps = NULL;
2220         }
2221
2222         if (net->ibn_fmr_ps != NULL) {
2223                 cfs_percpt_free(net->ibn_fmr_ps);
2224                 net->ibn_fmr_ps = NULL;
2225         }
2226
2227         if (net->ibn_pmr_ps != NULL) {
2228                 cfs_percpt_free(net->ibn_pmr_ps);
2229                 net->ibn_pmr_ps = NULL;
2230         }
2231 }
2232
2233 int
2234 kiblnd_net_init_pools(kib_net_t *net, __u32 *cpts, int ncpts)
2235 {
2236         unsigned long   flags;
2237         int             cpt;
2238         int             rc;
2239         int             i;
2240
2241         net->ibn_tx_ps = cfs_percpt_alloc(lnet_cpt_table(),
2242                                           sizeof(kib_tx_poolset_t));
2243         if (net->ibn_tx_ps == NULL) {
2244                 CERROR("Failed to allocate tx pool array\n");
2245                 return -ENOMEM;
2246         }
2247
2248         for (i = 0; i < ncpts; i++) {
2249                 cpt = (cpts == NULL) ? i : cpts[i];
2250                 rc = kiblnd_init_poolset(&net->ibn_tx_ps[cpt]->tps_poolset,
2251                                          cpt, net, "TX",
2252                                          kiblnd_tx_pool_size(ncpts),
2253                                          kiblnd_create_tx_pool,
2254                                          kiblnd_destroy_tx_pool,
2255                                          kiblnd_tx_init, NULL);
2256                 if (rc != 0) {
2257                         CERROR("Failed to initialize TX pool\n");
2258                         goto failed;
2259                 }
2260         }
2261
2262         cfs_read_lock_irqsave(&kiblnd_data.kib_global_lock, flags);
2263         if (*kiblnd_tunables.kib_map_on_demand == 0 &&
2264             net->ibn_dev->ibd_hdev->ibh_nmrs == 1) {
2265                 cfs_read_unlock_irqrestore(&kiblnd_data.kib_global_lock,
2266                                            flags);
2267                 return 0;
2268         }
2269
2270         cfs_read_unlock_irqrestore(&kiblnd_data.kib_global_lock, flags);
2271
2272         if (*kiblnd_tunables.kib_fmr_pool_size <
2273             *kiblnd_tunables.kib_ntx / 4) {
2274                 CERROR("Can't set fmr pool size (%d) < ntx / 4(%d)\n",
2275                        *kiblnd_tunables.kib_fmr_pool_size,
2276                        *kiblnd_tunables.kib_ntx / 4);
2277                 goto failed;
2278         }
2279
2280         /* premapping can fail if ibd_nmr > 1, so we always create
2281          * FMR/PMR pool and map-on-demand if premapping failed */
2282
2283         net->ibn_fmr_ps = cfs_percpt_alloc(lnet_cpt_table(),
2284                                            sizeof(kib_fmr_poolset_t));
2285         if (net->ibn_fmr_ps == NULL) {
2286                 CERROR("Failed to allocate FMR pool array\n");
2287                 goto failed;
2288         }
2289
2290         for (i = 0; i < ncpts; i++) {
2291                 cpt = (cpts == NULL) ? i : cpts[i];
2292                 rc = kiblnd_init_fmr_poolset(net->ibn_fmr_ps[cpt], cpt, net,
2293                                              kiblnd_fmr_pool_size(ncpts),
2294                                              kiblnd_fmr_flush_trigger(ncpts));
2295                 if (rc == -ENOSYS && i == 0) /* no FMR */
2296                         break; /* create PMR pool */
2297                 if (rc != 0)
2298                         goto failed; /* a real error */
2299         }
2300
2301         cfs_percpt_free(net->ibn_fmr_ps);
2302         net->ibn_fmr_ps = NULL;
2303
2304         if (*kiblnd_tunables.kib_pmr_pool_size <
2305             *kiblnd_tunables.kib_ntx / 4) {
2306                 CERROR("Can't set pmr pool size (%d) < ntx / 4(%d)\n",
2307                        *kiblnd_tunables.kib_pmr_pool_size,
2308                        *kiblnd_tunables.kib_ntx / 4);
2309                 goto failed;
2310         }
2311
2312         net->ibn_pmr_ps = cfs_percpt_alloc(lnet_cpt_table(),
2313                                            sizeof(kib_pmr_poolset_t));
2314         if (net->ibn_pmr_ps == NULL) {
2315                 CERROR("Failed to allocate PMR pool array\n");
2316                 goto failed;
2317         }
2318
2319         for (i = 0; i < ncpts; i++) {
2320                 cpt = (cpts == NULL) ? i : cpts[i];
2321                 rc = kiblnd_init_poolset(&net->ibn_pmr_ps[cpt]->pps_poolset,
2322                                          cpt, net, "PMR",
2323                                          kiblnd_pmr_pool_size(ncpts),
2324                                          kiblnd_create_pmr_pool,
2325                                          kiblnd_destroy_pmr_pool, NULL, NULL);
2326                 if (rc != 0)
2327                         goto failed;
2328         }
2329
2330         return 0;
2331
2332  failed:
2333         kiblnd_net_fini_pools(net);
2334         return rc;
2335 }
2336
2337 static int
2338 kiblnd_hdev_get_attr(kib_hca_dev_t *hdev)
2339 {
2340         struct ib_device_attr *attr;
2341         int                    rc;
2342
2343         /* It's safe to assume a HCA can handle a page size
2344          * matching that of the native system */
2345         hdev->ibh_page_shift = PAGE_SHIFT;
2346         hdev->ibh_page_size  = 1 << PAGE_SHIFT;
2347         hdev->ibh_page_mask  = ~((__u64)hdev->ibh_page_size - 1);
2348
2349         LIBCFS_ALLOC(attr, sizeof(*attr));
2350         if (attr == NULL) {
2351                 CERROR("Out of memory\n");
2352                 return -ENOMEM;
2353         }
2354
2355         rc = ib_query_device(hdev->ibh_ibdev, attr);
2356         if (rc == 0)
2357                 hdev->ibh_mr_size = attr->max_mr_size;
2358
2359         LIBCFS_FREE(attr, sizeof(*attr));
2360
2361         if (rc != 0) {
2362                 CERROR("Failed to query IB device: %d\n", rc);
2363                 return rc;
2364         }
2365
2366         if (hdev->ibh_mr_size == ~0ULL) {
2367                 hdev->ibh_mr_shift = 64;
2368                 return 0;
2369         }
2370
2371         for (hdev->ibh_mr_shift = 0;
2372              hdev->ibh_mr_shift < 64; hdev->ibh_mr_shift ++) {
2373                 if (hdev->ibh_mr_size == (1ULL << hdev->ibh_mr_shift) ||
2374                     hdev->ibh_mr_size == (1ULL << hdev->ibh_mr_shift) - 1)
2375                         return 0;
2376         }
2377
2378         CERROR("Invalid mr size: "LPX64"\n", hdev->ibh_mr_size);
2379         return -EINVAL;
2380 }
2381
2382 void
2383 kiblnd_hdev_cleanup_mrs(kib_hca_dev_t *hdev)
2384 {
2385         int     i;
2386
2387         if (hdev->ibh_nmrs == 0 || hdev->ibh_mrs == NULL)
2388                 return;
2389
2390         for (i = 0; i < hdev->ibh_nmrs; i++) {
2391                 if (hdev->ibh_mrs[i] == NULL)
2392                         break;
2393
2394                 ib_dereg_mr(hdev->ibh_mrs[i]);
2395         }
2396
2397         LIBCFS_FREE(hdev->ibh_mrs, sizeof(*hdev->ibh_mrs) * hdev->ibh_nmrs);
2398         hdev->ibh_mrs  = NULL;
2399         hdev->ibh_nmrs = 0;
2400 }
2401
2402 void
2403 kiblnd_hdev_destroy(kib_hca_dev_t *hdev)
2404 {
2405         kiblnd_hdev_cleanup_mrs(hdev);
2406
2407         if (hdev->ibh_pd != NULL)
2408                 ib_dealloc_pd(hdev->ibh_pd);
2409
2410         if (hdev->ibh_cmid != NULL)
2411                 rdma_destroy_id(hdev->ibh_cmid);
2412
2413         LIBCFS_FREE(hdev, sizeof(*hdev));
2414 }
2415
2416 int
2417 kiblnd_hdev_setup_mrs(kib_hca_dev_t *hdev)
2418 {
2419         struct ib_mr *mr;
2420         int           i;
2421         int           rc;
2422         __u64         mm_size;
2423         __u64         mr_size;
2424         int           acflags = IB_ACCESS_LOCAL_WRITE |
2425                                 IB_ACCESS_REMOTE_WRITE;
2426
2427         rc = kiblnd_hdev_get_attr(hdev);
2428         if (rc != 0)
2429                 return rc;
2430
2431         if (hdev->ibh_mr_shift == 64) {
2432                 LIBCFS_ALLOC(hdev->ibh_mrs, 1 * sizeof(*hdev->ibh_mrs));
2433                 if (hdev->ibh_mrs == NULL) {
2434                         CERROR("Failed to allocate MRs table\n");
2435                         return -ENOMEM;
2436                 }
2437
2438                 hdev->ibh_mrs[0] = NULL;
2439                 hdev->ibh_nmrs   = 1;
2440
2441                 mr = ib_get_dma_mr(hdev->ibh_pd, acflags);
2442                 if (IS_ERR(mr)) {
2443                         CERROR("Failed ib_get_dma_mr : %ld\n", PTR_ERR(mr));
2444                         kiblnd_hdev_cleanup_mrs(hdev);
2445                         return PTR_ERR(mr);
2446                 }
2447
2448                 hdev->ibh_mrs[0] = mr;
2449
2450                 goto out;
2451         }
2452
2453         mr_size = (1ULL << hdev->ibh_mr_shift);
2454         mm_size = (unsigned long)high_memory - PAGE_OFFSET;
2455
2456         hdev->ibh_nmrs = (int)((mm_size + mr_size - 1) >> hdev->ibh_mr_shift);
2457
2458         if (hdev->ibh_mr_shift < 32 || hdev->ibh_nmrs > 1024) {
2459                 /* it's 4T..., assume we will re-code at that time */
2460                 CERROR("Can't support memory size: x"LPX64
2461                        " with MR size: x"LPX64"\n", mm_size, mr_size);
2462                 return -EINVAL;
2463         }
2464
2465         /* create an array of MRs to cover all memory */
2466         LIBCFS_ALLOC(hdev->ibh_mrs, sizeof(*hdev->ibh_mrs) * hdev->ibh_nmrs);
2467         if (hdev->ibh_mrs == NULL) {
2468                 CERROR("Failed to allocate MRs' table\n");
2469                 return -ENOMEM;
2470         }
2471
2472         memset(hdev->ibh_mrs, 0, sizeof(*hdev->ibh_mrs) * hdev->ibh_nmrs);
2473
2474         for (i = 0; i < hdev->ibh_nmrs; i++) {
2475                 struct ib_phys_buf ipb;
2476                 __u64              iova;
2477
2478                 ipb.size = hdev->ibh_mr_size;
2479                 ipb.addr = i * mr_size;
2480                 iova     = ipb.addr;
2481
2482                 mr = ib_reg_phys_mr(hdev->ibh_pd, &ipb, 1, acflags, &iova);
2483                 if (IS_ERR(mr)) {
2484                         CERROR("Failed ib_reg_phys_mr addr "LPX64
2485                                " size "LPX64" : %ld\n",
2486                                ipb.addr, ipb.size, PTR_ERR(mr));
2487                         kiblnd_hdev_cleanup_mrs(hdev);
2488                         return PTR_ERR(mr);
2489                 }
2490
2491                 LASSERT (iova == ipb.addr);
2492
2493                 hdev->ibh_mrs[i] = mr;
2494         }
2495
2496 out:
2497         if (hdev->ibh_mr_size != ~0ULL || hdev->ibh_nmrs != 1)
2498                 LCONSOLE_INFO("Register global MR array, MR size: "
2499                               LPX64", array size: %d\n",
2500                               hdev->ibh_mr_size, hdev->ibh_nmrs);
2501         return 0;
2502 }
2503
2504 static int
2505 kiblnd_dummy_callback(struct rdma_cm_id *cmid, struct rdma_cm_event *event)
2506 {       /* DUMMY */
2507         return 0;
2508 }
2509
2510 static int
2511 kiblnd_dev_need_failover(kib_dev_t *dev)
2512 {
2513         struct rdma_cm_id  *cmid;
2514         struct sockaddr_in  srcaddr;
2515         struct sockaddr_in  dstaddr;
2516         int                 rc;
2517
2518         if (dev->ibd_hdev == NULL || /* initializing */
2519             dev->ibd_hdev->ibh_cmid == NULL || /* listener is dead */
2520             *kiblnd_tunables.kib_dev_failover > 1) /* debugging */
2521                 return 1;
2522
2523         /* XXX: it's UGLY, but I don't have better way to find
2524          * ib-bonding HCA failover because:
2525          *
2526          * a. no reliable CM event for HCA failover...
2527          * b. no OFED API to get ib_device for current net_device...
2528          *
2529          * We have only two choices at this point:
2530          *
2531          * a. rdma_bind_addr(), it will conflict with listener cmid
2532          * b. rdma_resolve_addr() to zero addr */
2533         cmid = kiblnd_rdma_create_id(kiblnd_dummy_callback, dev, RDMA_PS_TCP,
2534                                      IB_QPT_RC);
2535         if (IS_ERR(cmid)) {
2536                 rc = PTR_ERR(cmid);
2537                 CERROR("Failed to create cmid for failover: %d\n", rc);
2538                 return rc;
2539         }
2540
2541         memset(&srcaddr, 0, sizeof(srcaddr));
2542         srcaddr.sin_family      = AF_INET;
2543         srcaddr.sin_addr.s_addr = (__force u32)htonl(dev->ibd_ifip);
2544
2545         memset(&dstaddr, 0, sizeof(dstaddr));
2546         dstaddr.sin_family = AF_INET;
2547         rc = rdma_resolve_addr(cmid, (struct sockaddr *)&srcaddr,
2548                                (struct sockaddr *)&dstaddr, 1);
2549         if (rc != 0 || cmid->device == NULL) {
2550                 CERROR("Failed to bind %s:%u.%u.%u.%u to device(%p): %d\n",
2551                        dev->ibd_ifname, HIPQUAD(dev->ibd_ifip),
2552                        cmid->device, rc);
2553                 rdma_destroy_id(cmid);
2554                 return rc;
2555         }
2556
2557         if (dev->ibd_hdev->ibh_ibdev == cmid->device) {
2558                 /* don't need device failover */
2559                 rdma_destroy_id(cmid);
2560                 return 0;
2561         }
2562
2563         return 1;
2564 }
2565
2566 int
2567 kiblnd_dev_failover(kib_dev_t *dev)
2568 {
2569         CFS_LIST_HEAD      (zombie_tpo);
2570         CFS_LIST_HEAD      (zombie_ppo);
2571         CFS_LIST_HEAD      (zombie_fpo);
2572         struct rdma_cm_id  *cmid  = NULL;
2573         kib_hca_dev_t      *hdev  = NULL;
2574         kib_hca_dev_t      *old;
2575         struct ib_pd       *pd;
2576         kib_net_t          *net;
2577         struct sockaddr_in  addr;
2578         unsigned long       flags;
2579         int                 rc = 0;
2580         int                 i;
2581
2582         LASSERT (*kiblnd_tunables.kib_dev_failover > 1 ||
2583                  dev->ibd_can_failover ||
2584                  dev->ibd_hdev == NULL);
2585
2586         rc = kiblnd_dev_need_failover(dev);
2587         if (rc <= 0)
2588                 goto out;
2589
2590         if (dev->ibd_hdev != NULL &&
2591             dev->ibd_hdev->ibh_cmid != NULL) {
2592                 /* XXX it's not good to close old listener at here,
2593                  * because we can fail to create new listener.
2594                  * But we have to close it now, otherwise rdma_bind_addr
2595                  * will return EADDRINUSE... How crap! */
2596                 cfs_write_lock_irqsave(&kiblnd_data.kib_global_lock, flags);
2597
2598                 cmid = dev->ibd_hdev->ibh_cmid;
2599                 /* make next schedule of kiblnd_dev_need_failover
2600                  * will return 1 for me */
2601                 dev->ibd_hdev->ibh_cmid  = NULL;
2602                 cfs_write_unlock_irqrestore(&kiblnd_data.kib_global_lock, flags);
2603
2604                 rdma_destroy_id(cmid);
2605         }
2606
2607         cmid = kiblnd_rdma_create_id(kiblnd_cm_callback, dev, RDMA_PS_TCP,
2608                                      IB_QPT_RC);
2609         if (IS_ERR(cmid)) {
2610                 rc = PTR_ERR(cmid);
2611                 CERROR("Failed to create cmid for failover: %d\n", rc);
2612                 goto out;
2613         }
2614
2615         memset(&addr, 0, sizeof(addr));
2616         addr.sin_family      = AF_INET;
2617         addr.sin_addr.s_addr = (__force u32)htonl(dev->ibd_ifip);
2618         addr.sin_port        = htons(*kiblnd_tunables.kib_service);
2619
2620         /* Bind to failover device or port */
2621         rc = rdma_bind_addr(cmid, (struct sockaddr *)&addr);
2622         if (rc != 0 || cmid->device == NULL) {
2623                 CERROR("Failed to bind %s:%u.%u.%u.%u to device(%p): %d\n",
2624                        dev->ibd_ifname, HIPQUAD(dev->ibd_ifip),
2625                        cmid->device, rc);
2626                 rdma_destroy_id(cmid);
2627                 goto out;
2628         }
2629
2630         LIBCFS_ALLOC(hdev, sizeof(*hdev));
2631         if (hdev == NULL) {
2632                 CERROR("Failed to allocate kib_hca_dev\n");
2633                 rdma_destroy_id(cmid);
2634                 rc = -ENOMEM;
2635                 goto out;
2636         }
2637
2638         atomic_set(&hdev->ibh_ref, 1);
2639         hdev->ibh_dev   = dev;
2640         hdev->ibh_cmid  = cmid;
2641         hdev->ibh_ibdev = cmid->device;
2642
2643         pd = ib_alloc_pd(cmid->device);
2644         if (IS_ERR(pd)) {
2645                 rc = PTR_ERR(pd);
2646                 CERROR("Can't allocate PD: %d\n", rc);
2647                 goto out;
2648         }
2649
2650         hdev->ibh_pd = pd;
2651
2652         rc = rdma_listen(cmid, 0);
2653         if (rc != 0) {
2654                 CERROR("Can't start new listener: %d\n", rc);
2655                 goto out;
2656         }
2657
2658         rc = kiblnd_hdev_setup_mrs(hdev);
2659         if (rc != 0) {
2660                 CERROR("Can't setup device: %d\n", rc);
2661                 goto out;
2662         }
2663
2664         cfs_write_lock_irqsave(&kiblnd_data.kib_global_lock, flags);
2665
2666         old = dev->ibd_hdev;
2667         dev->ibd_hdev = hdev; /* take over the refcount */
2668         hdev = old;
2669
2670         cfs_list_for_each_entry(net, &dev->ibd_nets, ibn_list) {
2671                 cfs_cpt_for_each(i, lnet_cpt_table()) {
2672                         kiblnd_fail_poolset(&net->ibn_tx_ps[i]->tps_poolset,
2673                                             &zombie_tpo);
2674
2675                         if (net->ibn_fmr_ps != NULL) {
2676                                 kiblnd_fail_fmr_poolset(net->ibn_fmr_ps[i],
2677                                                         &zombie_fpo);
2678
2679                         } else if (net->ibn_pmr_ps != NULL) {
2680                                 kiblnd_fail_poolset(&net->ibn_pmr_ps[i]->
2681                                                     pps_poolset, &zombie_ppo);
2682                         }
2683                 }
2684         }
2685
2686         cfs_write_unlock_irqrestore(&kiblnd_data.kib_global_lock, flags);
2687  out:
2688         if (!cfs_list_empty(&zombie_tpo))
2689                 kiblnd_destroy_pool_list(&zombie_tpo);
2690         if (!cfs_list_empty(&zombie_ppo))
2691                 kiblnd_destroy_pool_list(&zombie_ppo);
2692         if (!cfs_list_empty(&zombie_fpo))
2693                 kiblnd_destroy_fmr_pool_list(&zombie_fpo);
2694         if (hdev != NULL)
2695                 kiblnd_hdev_decref(hdev);
2696
2697         if (rc != 0)
2698                 dev->ibd_failed_failover++;
2699         else
2700                 dev->ibd_failed_failover = 0;
2701
2702         return rc;
2703 }
2704
2705 void
2706 kiblnd_destroy_dev (kib_dev_t *dev)
2707 {
2708         LASSERT (dev->ibd_nnets == 0);
2709         LASSERT (cfs_list_empty(&dev->ibd_nets));
2710
2711         cfs_list_del(&dev->ibd_fail_list);
2712         cfs_list_del(&dev->ibd_list);
2713
2714         if (dev->ibd_hdev != NULL)
2715                 kiblnd_hdev_decref(dev->ibd_hdev);
2716
2717         LIBCFS_FREE(dev, sizeof(*dev));
2718 }
2719
2720 kib_dev_t *
2721 kiblnd_create_dev(char *ifname)
2722 {
2723         struct net_device *netdev;
2724         kib_dev_t         *dev;
2725         __u32              netmask;
2726         __u32              ip;
2727         int                up;
2728         int                rc;
2729
2730         rc = libcfs_ipif_query(ifname, &up, &ip, &netmask);
2731         if (rc != 0) {
2732                 CERROR("Can't query IPoIB interface %s: %d\n",
2733                        ifname, rc);
2734                 return NULL;
2735         }
2736
2737         if (!up) {
2738                 CERROR("Can't query IPoIB interface %s: it's down\n", ifname);
2739                 return NULL;
2740         }
2741
2742         LIBCFS_ALLOC(dev, sizeof(*dev));
2743         if (dev == NULL)
2744                 return NULL;
2745
2746         memset(dev, 0, sizeof(*dev));
2747 #ifdef HAVE_DEV_GET_BY_NAME_2ARG
2748         netdev = dev_get_by_name(&init_net, ifname);
2749 #else
2750         netdev = dev_get_by_name(ifname);
2751 #endif
2752         if (netdev == NULL) {
2753                 dev->ibd_can_failover = 0;
2754         } else {
2755                 dev->ibd_can_failover = !!(netdev->flags & IFF_MASTER);
2756                 dev_put(netdev);
2757         }
2758
2759         CFS_INIT_LIST_HEAD(&dev->ibd_nets);
2760         CFS_INIT_LIST_HEAD(&dev->ibd_list); /* not yet in kib_devs */
2761         CFS_INIT_LIST_HEAD(&dev->ibd_fail_list);
2762         dev->ibd_ifip = ip;
2763         strcpy(&dev->ibd_ifname[0], ifname);
2764
2765         /* initialize the device */
2766         rc = kiblnd_dev_failover(dev);
2767         if (rc != 0) {
2768                 CERROR("Can't initialize device: %d\n", rc);
2769                 LIBCFS_FREE(dev, sizeof(*dev));
2770                 return NULL;
2771         }
2772
2773         cfs_list_add_tail(&dev->ibd_list,
2774                           &kiblnd_data.kib_devs);
2775         return dev;
2776 }
2777
2778 void
2779 kiblnd_base_shutdown(void)
2780 {
2781         struct kib_sched_info   *sched;
2782         int                     i;
2783
2784         LASSERT (cfs_list_empty(&kiblnd_data.kib_devs));
2785
2786         CDEBUG(D_MALLOC, "before LND base cleanup: kmem %d\n",
2787                cfs_atomic_read(&libcfs_kmemory));
2788
2789         switch (kiblnd_data.kib_init) {
2790         default:
2791                 LBUG();
2792
2793         case IBLND_INIT_ALL:
2794         case IBLND_INIT_DATA:
2795                 LASSERT (kiblnd_data.kib_peers != NULL);
2796                 for (i = 0; i < kiblnd_data.kib_peer_hash_size; i++) {
2797                         LASSERT (cfs_list_empty(&kiblnd_data.kib_peers[i]));
2798                 }
2799                 LASSERT (cfs_list_empty(&kiblnd_data.kib_connd_zombies));
2800                 LASSERT (cfs_list_empty(&kiblnd_data.kib_connd_conns));
2801
2802                 /* flag threads to terminate; wake and wait for them to die */
2803                 kiblnd_data.kib_shutdown = 1;
2804
2805                 /* NB: we really want to stop scheduler threads net by net
2806                  * instead of the whole module, this should be improved
2807                  * with dynamic configuration LNet */
2808                 cfs_percpt_for_each(sched, i, kiblnd_data.kib_scheds)
2809                         cfs_waitq_broadcast(&sched->ibs_waitq);
2810
2811                 cfs_waitq_broadcast(&kiblnd_data.kib_connd_waitq);
2812                 cfs_waitq_broadcast(&kiblnd_data.kib_failover_waitq);
2813
2814                 i = 2;
2815                 while (cfs_atomic_read(&kiblnd_data.kib_nthreads) != 0) {
2816                         i++;
2817                         CDEBUG(((i & (-i)) == i) ? D_WARNING : D_NET, /* power of 2? */
2818                                "Waiting for %d threads to terminate\n",
2819                                cfs_atomic_read(&kiblnd_data.kib_nthreads));
2820                         cfs_pause(cfs_time_seconds(1));
2821                 }
2822
2823                 /* fall through */
2824
2825         case IBLND_INIT_NOTHING:
2826                 break;
2827         }
2828
2829         if (kiblnd_data.kib_peers != NULL) {
2830                 LIBCFS_FREE(kiblnd_data.kib_peers,
2831                             sizeof(cfs_list_t) *
2832                             kiblnd_data.kib_peer_hash_size);
2833         }
2834
2835         if (kiblnd_data.kib_scheds != NULL)
2836                 cfs_percpt_free(kiblnd_data.kib_scheds);
2837
2838         CDEBUG(D_MALLOC, "after LND base cleanup: kmem %d\n",
2839                cfs_atomic_read(&libcfs_kmemory));
2840
2841         kiblnd_data.kib_init = IBLND_INIT_NOTHING;
2842         PORTAL_MODULE_UNUSE;
2843 }
2844
2845 void
2846 kiblnd_shutdown (lnet_ni_t *ni)
2847 {
2848         kib_net_t        *net = ni->ni_data;
2849         cfs_rwlock_t     *g_lock = &kiblnd_data.kib_global_lock;
2850         int               i;
2851         unsigned long     flags;
2852
2853         LASSERT(kiblnd_data.kib_init == IBLND_INIT_ALL);
2854
2855         if (net == NULL)
2856                 goto out;
2857
2858         CDEBUG(D_MALLOC, "before LND net cleanup: kmem %d\n",
2859                cfs_atomic_read(&libcfs_kmemory));
2860
2861         cfs_write_lock_irqsave(g_lock, flags);
2862         net->ibn_shutdown = 1;
2863         cfs_write_unlock_irqrestore(g_lock, flags);
2864
2865         switch (net->ibn_init) {
2866         default:
2867                 LBUG();
2868
2869         case IBLND_INIT_ALL:
2870                 /* nuke all existing peers within this net */
2871                 kiblnd_del_peer(ni, LNET_NID_ANY);
2872
2873                 /* Wait for all peer state to clean up */
2874                 i = 2;
2875                 while (cfs_atomic_read(&net->ibn_npeers) != 0) {
2876                         i++;
2877                         CDEBUG(((i & (-i)) == i) ? D_WARNING : D_NET, /* 2**n? */
2878                                "%s: waiting for %d peers to disconnect\n",
2879                                libcfs_nid2str(ni->ni_nid),
2880                                cfs_atomic_read(&net->ibn_npeers));
2881                         cfs_pause(cfs_time_seconds(1));
2882                 }
2883
2884                 kiblnd_net_fini_pools(net);
2885
2886                 cfs_write_lock_irqsave(g_lock, flags);
2887                 LASSERT (net->ibn_dev->ibd_nnets > 0);
2888                 net->ibn_dev->ibd_nnets--;
2889                 cfs_list_del(&net->ibn_list);
2890                 cfs_write_unlock_irqrestore(g_lock, flags);
2891
2892                 /* fall through */
2893
2894         case IBLND_INIT_NOTHING:
2895                 LASSERT (cfs_atomic_read(&net->ibn_nconns) == 0);
2896
2897                 if (net->ibn_dev != NULL &&
2898                     net->ibn_dev->ibd_nnets == 0)
2899                         kiblnd_destroy_dev(net->ibn_dev);
2900
2901                 break;
2902         }
2903
2904         CDEBUG(D_MALLOC, "after LND net cleanup: kmem %d\n",
2905                cfs_atomic_read(&libcfs_kmemory));
2906
2907         net->ibn_init = IBLND_INIT_NOTHING;
2908         ni->ni_data = NULL;
2909
2910         LIBCFS_FREE(net, sizeof(*net));
2911
2912 out:
2913         if (cfs_list_empty(&kiblnd_data.kib_devs))
2914                 kiblnd_base_shutdown();
2915         return;
2916 }
2917
2918 int
2919 kiblnd_base_startup(void)
2920 {
2921         struct kib_sched_info   *sched;
2922         int                     rc;
2923         int                     i;
2924
2925         LASSERT (kiblnd_data.kib_init == IBLND_INIT_NOTHING);
2926
2927         PORTAL_MODULE_USE;
2928         memset(&kiblnd_data, 0, sizeof(kiblnd_data)); /* zero pointers, flags etc */
2929
2930         cfs_rwlock_init(&kiblnd_data.kib_global_lock);
2931
2932         CFS_INIT_LIST_HEAD(&kiblnd_data.kib_devs);
2933         CFS_INIT_LIST_HEAD(&kiblnd_data.kib_failed_devs);
2934
2935         kiblnd_data.kib_peer_hash_size = IBLND_PEER_HASH_SIZE;
2936         LIBCFS_ALLOC(kiblnd_data.kib_peers,
2937                      sizeof(cfs_list_t) *
2938                             kiblnd_data.kib_peer_hash_size);
2939         if (kiblnd_data.kib_peers == NULL) {
2940                 goto failed;
2941         }
2942         for (i = 0; i < kiblnd_data.kib_peer_hash_size; i++)
2943                 CFS_INIT_LIST_HEAD(&kiblnd_data.kib_peers[i]);
2944
2945         cfs_spin_lock_init(&kiblnd_data.kib_connd_lock);
2946         CFS_INIT_LIST_HEAD(&kiblnd_data.kib_connd_conns);
2947         CFS_INIT_LIST_HEAD(&kiblnd_data.kib_connd_zombies);
2948         cfs_waitq_init(&kiblnd_data.kib_connd_waitq);
2949         cfs_waitq_init(&kiblnd_data.kib_failover_waitq);
2950
2951         kiblnd_data.kib_scheds = cfs_percpt_alloc(lnet_cpt_table(),
2952                                                   sizeof(*sched));
2953         if (kiblnd_data.kib_scheds == NULL)
2954                 goto failed;
2955
2956         cfs_percpt_for_each(sched, i, kiblnd_data.kib_scheds) {
2957                 int     nthrs;
2958
2959                 cfs_spin_lock_init(&sched->ibs_lock);
2960                 CFS_INIT_LIST_HEAD(&sched->ibs_conns);
2961                 cfs_waitq_init(&sched->ibs_waitq);
2962
2963                 nthrs = cfs_cpt_weight(lnet_cpt_table(), i);
2964                 if (*kiblnd_tunables.kib_nscheds > 0) {
2965                         nthrs = min(nthrs, *kiblnd_tunables.kib_nscheds);
2966                 } else {
2967                         /* max to half of CPUs, another half is reserved for
2968                          * upper layer modules */
2969                         nthrs = min(max(IBLND_N_SCHED, nthrs >> 1), nthrs);
2970                 }
2971
2972                 sched->ibs_nthreads_max = nthrs;
2973                 sched->ibs_cpt = i;
2974         }
2975
2976         kiblnd_data.kib_error_qpa.qp_state = IB_QPS_ERR;
2977
2978         /* lists/ptrs/locks initialised */
2979         kiblnd_data.kib_init = IBLND_INIT_DATA;
2980         /*****************************************************/
2981
2982         rc = kiblnd_thread_start(kiblnd_connd, NULL);
2983         if (rc != 0) {
2984                 CERROR("Can't spawn o2iblnd connd: %d\n", rc);
2985                 goto failed;
2986         }
2987
2988         if (*kiblnd_tunables.kib_dev_failover != 0)
2989                 rc = kiblnd_thread_start(kiblnd_failover_thread, NULL);
2990
2991         if (rc != 0) {
2992                 CERROR("Can't spawn o2iblnd failover thread: %d\n", rc);
2993                 goto failed;
2994         }
2995
2996         /* flag everything initialised */
2997         kiblnd_data.kib_init = IBLND_INIT_ALL;
2998         /*****************************************************/
2999
3000         return 0;
3001
3002  failed:
3003         kiblnd_base_shutdown();
3004         return -ENETDOWN;
3005 }
3006
3007 int
3008 kiblnd_start_schedulers(struct kib_sched_info *sched)
3009 {
3010         int     rc = 0;
3011         int     nthrs;
3012         int     i;
3013
3014         if (sched->ibs_nthreads == 0) {
3015                 if (*kiblnd_tunables.kib_nscheds > 0) {
3016                         nthrs = sched->ibs_nthreads_max;
3017                 } else {
3018                         nthrs = cfs_cpt_weight(lnet_cpt_table(),
3019                                                sched->ibs_cpt);
3020                         nthrs = min(max(IBLND_N_SCHED, nthrs >> 1), nthrs);
3021                         nthrs = min(IBLND_N_SCHED_HIGH, nthrs);
3022                 }
3023         } else {
3024                 LASSERT(sched->ibs_nthreads <= sched->ibs_nthreads_max);
3025                 /* increase one thread if there is new interface */
3026                 nthrs = (sched->ibs_nthreads < sched->ibs_nthreads_max);
3027         }
3028
3029         for (i = 0; i < nthrs; i++) {
3030                 long    id;
3031
3032                 id = KIB_THREAD_ID(sched->ibs_cpt, sched->ibs_nthreads + i);
3033                 rc = kiblnd_thread_start(kiblnd_scheduler, (void *)id);
3034                 if (rc == 0)
3035                         continue;
3036
3037                 CERROR("Can't spawn thread %d for scheduler[%d]: %d\n",
3038                        sched->ibs_cpt, sched->ibs_nthreads + i, rc);
3039                 break;
3040         }
3041
3042         sched->ibs_nthreads += i;
3043         return rc;
3044 }
3045
3046 int
3047 kiblnd_dev_start_threads(kib_dev_t *dev, int newdev, __u32 *cpts, int ncpts)
3048 {
3049         int     cpt;
3050         int     rc;
3051         int     i;
3052
3053         for (i = 0; i < ncpts; i++) {
3054                 struct kib_sched_info *sched;
3055
3056                 cpt = (cpts == NULL) ? i : cpts[i];
3057                 sched = kiblnd_data.kib_scheds[cpt];
3058
3059                 if (!newdev && sched->ibs_nthreads > 0)
3060                         continue;
3061
3062                 rc = kiblnd_start_schedulers(kiblnd_data.kib_scheds[cpt]);
3063                 if (rc != 0) {
3064                         CERROR("Failed to start scheduler threads for %s\n",
3065                                dev->ibd_ifname);
3066                         return rc;
3067                 }
3068         }
3069         return 0;
3070 }
3071
3072 kib_dev_t *
3073 kiblnd_dev_search(char *ifname)
3074 {
3075         kib_dev_t       *alias = NULL;
3076         kib_dev_t       *dev;
3077         char            *colon;
3078         char            *colon2;
3079
3080         colon = strchr(ifname, ':');
3081         cfs_list_for_each_entry(dev, &kiblnd_data.kib_devs, ibd_list) {
3082                 if (strcmp(&dev->ibd_ifname[0], ifname) == 0)
3083                         return dev;
3084
3085                 if (alias != NULL)
3086                         continue;
3087
3088                 colon2 = strchr(dev->ibd_ifname, ':');
3089                 if (colon != NULL)
3090                         *colon = 0;
3091                 if (colon2 != NULL)
3092                         *colon2 = 0;
3093
3094                 if (strcmp(&dev->ibd_ifname[0], ifname) == 0)
3095                         alias = dev;
3096
3097                 if (colon != NULL)
3098                         *colon = ':';
3099                 if (colon2 != NULL)
3100                         *colon2 = ':';
3101         }
3102         return alias;
3103 }
3104
3105 int
3106 kiblnd_startup (lnet_ni_t *ni)
3107 {
3108         char                     *ifname;
3109         kib_dev_t                *ibdev = NULL;
3110         kib_net_t                *net;
3111         struct timeval            tv;
3112         unsigned long             flags;
3113         int                       rc;
3114         int                       newdev;
3115
3116         LASSERT (ni->ni_lnd == &the_o2iblnd);
3117
3118         if (kiblnd_data.kib_init == IBLND_INIT_NOTHING) {
3119                 rc = kiblnd_base_startup();
3120                 if (rc != 0)
3121                         return rc;
3122         }
3123
3124         LIBCFS_ALLOC(net, sizeof(*net));
3125         ni->ni_data = net;
3126         if (net == NULL)
3127                 goto failed;
3128
3129         memset(net, 0, sizeof(*net));
3130
3131         cfs_gettimeofday(&tv);
3132         net->ibn_incarnation = (((__u64)tv.tv_sec) * 1000000) + tv.tv_usec;
3133
3134         ni->ni_peertimeout    = *kiblnd_tunables.kib_peertimeout;
3135         ni->ni_maxtxcredits   = *kiblnd_tunables.kib_credits;
3136         ni->ni_peertxcredits  = *kiblnd_tunables.kib_peertxcredits;
3137         ni->ni_peerrtrcredits = *kiblnd_tunables.kib_peerrtrcredits;
3138
3139         if (ni->ni_interfaces[0] != NULL) {
3140                 /* Use the IPoIB interface specified in 'networks=' */
3141
3142                 CLASSERT (LNET_MAX_INTERFACES > 1);
3143                 if (ni->ni_interfaces[1] != NULL) {
3144                         CERROR("Multiple interfaces not supported\n");
3145                         goto failed;
3146                 }
3147
3148                 ifname = ni->ni_interfaces[0];
3149         } else {
3150                 ifname = *kiblnd_tunables.kib_default_ipif;
3151         }
3152
3153         if (strlen(ifname) >= sizeof(ibdev->ibd_ifname)) {
3154                 CERROR("IPoIB interface name too long: %s\n", ifname);
3155                 goto failed;
3156         }
3157
3158         ibdev = kiblnd_dev_search(ifname);
3159
3160         newdev = ibdev == NULL;
3161         /* hmm...create kib_dev even for alias */
3162         if (ibdev == NULL || strcmp(&ibdev->ibd_ifname[0], ifname) != 0)
3163                 ibdev = kiblnd_create_dev(ifname);
3164
3165         if (ibdev == NULL)
3166                 goto failed;
3167
3168         net->ibn_dev = ibdev;
3169         ni->ni_nid = LNET_MKNID(LNET_NIDNET(ni->ni_nid), ibdev->ibd_ifip);
3170
3171         rc = kiblnd_dev_start_threads(ibdev, newdev,
3172                                       ni->ni_cpts, ni->ni_ncpts);
3173         if (rc != 0)
3174                 goto failed;
3175
3176         rc = kiblnd_net_init_pools(net, ni->ni_cpts, ni->ni_ncpts);
3177         if (rc != 0) {
3178                 CERROR("Failed to initialize NI pools: %d\n", rc);
3179                 goto failed;
3180         }
3181
3182         cfs_write_lock_irqsave(&kiblnd_data.kib_global_lock, flags);
3183         ibdev->ibd_nnets++;
3184         cfs_list_add_tail(&net->ibn_list, &ibdev->ibd_nets);
3185         cfs_write_unlock_irqrestore(&kiblnd_data.kib_global_lock, flags);
3186
3187         net->ibn_init = IBLND_INIT_ALL;
3188
3189         return 0;
3190
3191 failed:
3192         if (net->ibn_dev == NULL && ibdev != NULL)
3193                 kiblnd_destroy_dev(ibdev);
3194
3195         kiblnd_shutdown(ni);
3196
3197         CDEBUG(D_NET, "kiblnd_startup failed\n");
3198         return -ENETDOWN;
3199 }
3200
3201 void __exit
3202 kiblnd_module_fini (void)
3203 {
3204         lnet_unregister_lnd(&the_o2iblnd);
3205         kiblnd_tunables_fini();
3206 }
3207
3208 int __init
3209 kiblnd_module_init (void)
3210 {
3211         int    rc;
3212
3213         CLASSERT (sizeof(kib_msg_t) <= IBLND_MSG_SIZE);
3214         CLASSERT (offsetof(kib_msg_t, ibm_u.get.ibgm_rd.rd_frags[IBLND_MAX_RDMA_FRAGS])
3215                   <= IBLND_MSG_SIZE);
3216         CLASSERT (offsetof(kib_msg_t, ibm_u.putack.ibpam_rd.rd_frags[IBLND_MAX_RDMA_FRAGS])
3217                   <= IBLND_MSG_SIZE);
3218
3219         rc = kiblnd_tunables_init();
3220         if (rc != 0)
3221                 return rc;
3222
3223         lnet_register_lnd(&the_o2iblnd);
3224
3225         return 0;
3226 }
3227
3228 MODULE_AUTHOR("Sun Microsystems, Inc. <http://www.lustre.org/>");
3229 MODULE_DESCRIPTION("Kernel OpenIB gen2 LND v2.00");
3230 MODULE_LICENSE("GPL");
3231
3232 module_init(kiblnd_module_init);
3233 module_exit(kiblnd_module_fini);