1 /* -*- mode: c; c-basic-offset: 8; indent-tabs-mode: nil; -*-
2 * vim:expandtab:shiftwidth=8:tabstop=8:
4 * Copyright (C) 2004 Cluster File Systems, Inc.
5 * Author: Eric Barton <eric@bartonsoftware.com>
7 * This file is part of Lustre, http://www.lustre.org.
9 * Lustre is free software; you can redistribute it and/or
10 * modify it under the terms of version 2 of the GNU General Public
11 * License as published by the Free Software Foundation.
13 * Lustre is distributed in the hope that it will be useful,
14 * but WITHOUT ANY WARRANTY; without even the implied warranty of
15 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
16 * GNU General Public License for more details.
18 * You should have received a copy of the GNU General Public License
19 * along with Lustre; if not, write to the Free Software
20 * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
27 ptl_handle_ni_t kibnal_ni;
28 kib_tunables_t kibnal_tunables;
30 kib_data_t kibnal_data = {
31 .kib_service_id = IBNAL_SERVICE_NUMBER,
35 #define IBNAL_SYSCTL 202
37 #define IBNAL_SYSCTL_TIMEOUT 1
39 static ctl_table kibnal_ctl_table[] = {
40 {IBNAL_SYSCTL_TIMEOUT, "timeout",
41 &kibnal_tunables.kib_io_timeout, sizeof (int),
42 0644, NULL, &proc_dointvec},
46 static ctl_table kibnal_top_ctl_table[] = {
47 {IBNAL_SYSCTL, "iibnal", NULL, 0, 0555, kibnal_ctl_table},
54 print_service(IB_SERVICE_RECORD *service, char *tag, int rc)
61 "status : %d (NULL)\n", tag, rc);
64 strncpy (name, service->ServiceName, sizeof(name)-1);
65 name[sizeof(name)-1] = 0;
69 "service id: "LPX64"\n"
71 "NID : "LPX64"\n", tag, rc,
72 service->RID.ServiceID, name,
73 *kibnal_service_nid_field(service));
78 kibnal_service_setunset_done (void *arg, FABRIC_OPERATION_DATA *fod,
79 FSTATUS frc, uint32 madrc)
81 *(FSTATUS *)arg = frc;
82 up (&kibnal_data.kib_nid_signal);
85 #if IBNAL_CHECK_ADVERT
87 kibnal_service_query_done (void *arg, QUERY *qry,
88 QUERY_RESULT_VALUES *qry_result)
90 FSTATUS frc = qry_result->Status;
92 if (frc != FSUCCESS &&
93 qry_result->ResultDataSize == 0)
96 *(FSTATUS *)arg = frc;
97 up (&kibnal_data.kib_nid_signal);
101 kibnal_check_advert (void)
104 IB_SERVICE_RECORD *svc;
108 PORTAL_ALLOC(qry, sizeof(*qry));
112 memset (qry, 0, sizeof(*qry));
113 qry->InputType = InputTypeServiceRecord;
114 qry->OutputType = OutputTypeServiceRecord;
115 qry->InputValue.ServiceRecordValue.ComponentMask = KIBNAL_SERVICE_KEY_MASK;
116 svc = &qry->InputValue.ServiceRecordValue.ServiceRecord;
117 kibnal_set_service_keys(svc, kibnal_data.kib_nid);
119 frc = iibt_sd_query_port_fabric_information(kibnal_data.kib_sd,
120 kibnal_data.kib_port_guid,
122 kibnal_service_query_done,
124 if (frc != FSUCCESS && frc != FPENDING) {
125 CERROR ("Immediate error %d checking SM service\n", frc);
127 down (&kibnal_data.kib_nid_signal);
131 CERROR ("Error %d checking SM service\n", rc);
138 static void fill_fod(FABRIC_OPERATION_DATA *fod, FABRIC_OPERATION_TYPE type)
140 IB_SERVICE_RECORD *svc;
142 memset (fod, 0, sizeof(*fod));
145 svc = &fod->Value.ServiceRecordValue.ServiceRecord;
146 svc->RID.ServiceID = kibnal_data.kib_service_id;
147 svc->RID.ServiceGID.Type.Global.InterfaceID = kibnal_data.kib_port_guid;
148 svc->RID.ServiceGID.Type.Global.SubnetPrefix = DEFAULT_SUBNET_PREFIX;
149 svc->RID.ServiceP_Key = kibnal_data.kib_port_pkey;
150 svc->ServiceLease = 0xffffffff;
152 kibnal_set_service_keys(svc, kibnal_data.kib_nid);
156 kibnal_advertise (void)
158 FABRIC_OPERATION_DATA *fod;
159 IB_SERVICE_RECORD *svc;
163 LASSERT (kibnal_data.kib_nid != PTL_NID_ANY);
165 PORTAL_ALLOC(fod, sizeof(*fod));
169 fill_fod(fod, FabOpSetServiceRecord);
170 svc = &fod->Value.ServiceRecordValue.ServiceRecord;
172 CDEBUG(D_NET, "Advertising service id "LPX64" %s:"LPX64"\n",
174 svc->ServiceName, *kibnal_service_nid_field(svc));
176 frc = iibt_sd_port_fabric_operation(kibnal_data.kib_sd,
177 kibnal_data.kib_port_guid,
178 fod, kibnal_service_setunset_done,
181 if (frc != FSUCCESS && frc != FPENDING) {
182 CERROR ("Immediate error %d advertising NID "LPX64"\n",
183 frc, kibnal_data.kib_nid);
187 down (&kibnal_data.kib_nid_signal);
191 CERROR ("Error %d advertising BUD "LPX64"\n",
192 frc, kibnal_data.kib_nid);
194 PORTAL_FREE(fod, sizeof(*fod));
195 return (frc == FSUCCESS) ? 0 : -EINVAL;
199 kibnal_unadvertise (int expect_success)
201 FABRIC_OPERATION_DATA *fod;
202 IB_SERVICE_RECORD *svc;
206 LASSERT (kibnal_data.kib_nid != PTL_NID_ANY);
208 PORTAL_ALLOC(fod, sizeof(*fod));
212 fill_fod(fod, FabOpDeleteServiceRecord);
213 svc = &fod->Value.ServiceRecordValue.ServiceRecord;
215 CDEBUG(D_NET, "Unadvertising service %s:"LPX64"\n",
216 svc->ServiceName, *kibnal_service_nid_field(svc));
218 frc = iibt_sd_port_fabric_operation(kibnal_data.kib_sd,
219 kibnal_data.kib_port_guid,
220 fod, kibnal_service_setunset_done,
223 if (frc != FSUCCESS && frc != FPENDING) {
224 CERROR ("Immediate error %d unadvertising NID "LPX64"\n",
225 frc, kibnal_data.kib_nid);
229 down (&kibnal_data.kib_nid_signal);
231 if ((frc2 == FSUCCESS) == !!expect_success)
235 CERROR("Error %d unadvertising NID "LPX64"\n",
236 frc2, kibnal_data.kib_nid);
238 CWARN("Removed conflicting NID "LPX64"\n",
239 kibnal_data.kib_nid);
241 PORTAL_FREE(fod, sizeof(*fod));
245 kibnal_set_mynid(ptl_nid_t nid)
248 lib_ni_t *ni = &kibnal_lib.libnal_ni;
252 CDEBUG(D_IOCTL, "setting mynid to "LPX64" (old nid="LPX64")\n",
253 nid, ni->ni_pid.nid);
255 do_gettimeofday(&tv);
257 down (&kibnal_data.kib_nid_mutex);
259 if (nid == kibnal_data.kib_nid) {
260 /* no change of NID */
261 up (&kibnal_data.kib_nid_mutex);
265 CDEBUG(D_NET, "NID "LPX64"("LPX64")\n",
266 kibnal_data.kib_nid, nid);
268 if (kibnal_data.kib_nid != PTL_NID_ANY) {
270 kibnal_unadvertise (1);
272 frc = iibt_cm_cancel(kibnal_data.kib_cep);
273 if (frc != FSUCCESS && frc != FPENDING)
274 CERROR ("Error %d stopping listener\n", frc);
276 frc = iibt_cm_destroy_cep(kibnal_data.kib_cep);
278 CERROR ("Error %d destroying CEP\n", frc);
280 kibnal_data.kib_cep = NULL;
283 kibnal_data.kib_nid = ni->ni_pid.nid = nid;
284 kibnal_data.kib_incarnation = (((__u64)tv.tv_sec) * 1000000) + tv.tv_usec;
286 /* Delete all existing peers and their connections after new
287 * NID/incarnation set to ensure no old connections in our brave
289 kibnal_del_peer (PTL_NID_ANY, 0);
291 if (kibnal_data.kib_nid == PTL_NID_ANY) {
292 /* No new NID to install */
293 up (&kibnal_data.kib_nid_mutex);
297 /* remove any previous advert (crashed node etc) */
298 kibnal_unadvertise(0);
300 kibnal_data.kib_cep = iibt_cm_create_cep(CM_RC_TYPE);
301 if (kibnal_data.kib_cep == NULL) {
302 CERROR ("Can't create CEP\n");
306 memset (&info, 0, sizeof(info));
307 info.ListenAddr.EndPt.SID = kibnal_data.kib_service_id;
309 frc = iibt_cm_listen(kibnal_data.kib_cep, &info,
310 kibnal_listen_callback, NULL);
311 if (frc != FSUCCESS && frc != FPENDING) {
312 CERROR ("iibt_cm_listen error: %d\n", frc);
320 rc = kibnal_advertise();
322 #if IBNAL_CHECK_ADVERT
323 kibnal_check_advert();
325 up (&kibnal_data.kib_nid_mutex);
329 iibt_cm_cancel (kibnal_data.kib_cep);
330 iibt_cm_destroy_cep (kibnal_data.kib_cep);
331 /* remove any peers that sprung up while I failed to
332 * advertise myself */
333 kibnal_del_peer (PTL_NID_ANY, 0);
336 kibnal_data.kib_nid = PTL_NID_ANY;
337 up (&kibnal_data.kib_nid_mutex);
342 kibnal_create_peer (ptl_nid_t nid)
346 LASSERT (nid != PTL_NID_ANY);
348 PORTAL_ALLOC (peer, sizeof (*peer));
352 memset(peer, 0, sizeof(*peer)); /* zero flags etc */
355 atomic_set (&peer->ibp_refcount, 1); /* 1 ref for caller */
357 INIT_LIST_HEAD (&peer->ibp_list); /* not in the peer table yet */
358 INIT_LIST_HEAD (&peer->ibp_conns);
359 INIT_LIST_HEAD (&peer->ibp_tx_queue);
361 peer->ibp_reconnect_time = jiffies;
362 peer->ibp_reconnect_interval = IBNAL_MIN_RECONNECT_INTERVAL;
364 atomic_inc (&kibnal_data.kib_npeers);
369 kibnal_destroy_peer (kib_peer_t *peer)
372 LASSERT (atomic_read (&peer->ibp_refcount) == 0);
373 LASSERT (peer->ibp_persistence == 0);
374 LASSERT (!kibnal_peer_active(peer));
375 LASSERT (peer->ibp_connecting == 0);
376 LASSERT (list_empty (&peer->ibp_conns));
377 LASSERT (list_empty (&peer->ibp_tx_queue));
379 PORTAL_FREE (peer, sizeof (*peer));
381 /* NB a peer's connections keep a reference on their peer until
382 * they are destroyed, so we can be assured that _all_ state to do
383 * with this peer has been cleaned up when its refcount drops to
385 atomic_dec (&kibnal_data.kib_npeers);
388 /* the caller is responsible for accounting for the additional reference
389 * that this creates */
391 kibnal_find_peer_locked (ptl_nid_t nid)
393 struct list_head *peer_list = kibnal_nid2peerlist (nid);
394 struct list_head *tmp;
397 list_for_each (tmp, peer_list) {
399 peer = list_entry (tmp, kib_peer_t, ibp_list);
401 LASSERT (peer->ibp_persistence != 0 || /* persistent peer */
402 peer->ibp_connecting != 0 || /* creating conns */
403 !list_empty (&peer->ibp_conns)); /* active conn */
405 if (peer->ibp_nid != nid)
408 CDEBUG(D_NET, "got peer [%p] -> "LPX64" (%d)\n",
409 peer, nid, atomic_read (&peer->ibp_refcount));
416 kibnal_get_peer (ptl_nid_t nid)
421 read_lock_irqsave(&kibnal_data.kib_global_lock, flags);
422 peer = kibnal_find_peer_locked (nid);
423 if (peer != NULL) /* +1 ref for caller? */
424 kib_peer_addref(peer);
425 read_unlock_irqrestore(&kibnal_data.kib_global_lock, flags);
431 kibnal_unlink_peer_locked (kib_peer_t *peer)
433 LASSERT (peer->ibp_persistence == 0);
434 LASSERT (list_empty(&peer->ibp_conns));
436 LASSERT (kibnal_peer_active(peer));
437 list_del_init (&peer->ibp_list);
438 /* lose peerlist's ref */
439 kib_peer_decref(peer);
443 kibnal_get_peer_info (int index, ptl_nid_t *nidp, int *persistencep)
446 struct list_head *ptmp;
450 read_lock_irqsave(&kibnal_data.kib_global_lock, flags);
452 for (i = 0; i < kibnal_data.kib_peer_hash_size; i++) {
454 list_for_each (ptmp, &kibnal_data.kib_peers[i]) {
456 peer = list_entry (ptmp, kib_peer_t, ibp_list);
457 LASSERT (peer->ibp_persistence != 0 ||
458 peer->ibp_connecting != 0 ||
459 !list_empty (&peer->ibp_conns));
464 *nidp = peer->ibp_nid;
465 *persistencep = peer->ibp_persistence;
467 read_unlock_irqrestore(&kibnal_data.kib_global_lock,
473 read_unlock_irqrestore(&kibnal_data.kib_global_lock, flags);
478 kibnal_add_persistent_peer (ptl_nid_t nid)
484 if (nid == PTL_NID_ANY)
487 peer = kibnal_create_peer (nid);
491 write_lock_irqsave (&kibnal_data.kib_global_lock, flags);
493 peer2 = kibnal_find_peer_locked (nid);
495 kib_peer_decref (peer);
498 /* peer table takes existing ref on peer */
499 list_add_tail (&peer->ibp_list,
500 kibnal_nid2peerlist (nid));
503 peer->ibp_persistence++;
505 write_unlock_irqrestore (&kibnal_data.kib_global_lock, flags);
510 kibnal_del_peer_locked (kib_peer_t *peer, int single_share)
512 struct list_head *ctmp;
513 struct list_head *cnxt;
517 peer->ibp_persistence = 0;
518 else if (peer->ibp_persistence > 0)
519 peer->ibp_persistence--;
521 if (peer->ibp_persistence != 0)
524 if (list_empty(&peer->ibp_conns)) {
525 kibnal_unlink_peer_locked(peer);
527 list_for_each_safe (ctmp, cnxt, &peer->ibp_conns) {
528 conn = list_entry(ctmp, kib_conn_t, ibc_list);
530 kibnal_close_conn_locked (conn, 0);
532 /* NB peer is no longer persistent; closing its last conn
535 /* NB peer now unlinked; might even be freed if the peer table had the
540 kibnal_del_peer (ptl_nid_t nid, int single_share)
543 struct list_head *ptmp;
544 struct list_head *pnxt;
551 write_lock_irqsave (&kibnal_data.kib_global_lock, flags);
553 if (nid != PTL_NID_ANY)
554 lo = hi = kibnal_nid2peerlist(nid) - kibnal_data.kib_peers;
557 hi = kibnal_data.kib_peer_hash_size - 1;
560 for (i = lo; i <= hi; i++) {
561 list_for_each_safe (ptmp, pnxt, &kibnal_data.kib_peers[i]) {
562 peer = list_entry (ptmp, kib_peer_t, ibp_list);
563 LASSERT (peer->ibp_persistence != 0 ||
564 peer->ibp_connecting != 0 ||
565 !list_empty (&peer->ibp_conns));
567 if (!(nid == PTL_NID_ANY || peer->ibp_nid == nid))
570 kibnal_del_peer_locked (peer, single_share);
571 rc = 0; /* matched something */
578 write_unlock_irqrestore (&kibnal_data.kib_global_lock, flags);
584 kibnal_get_conn_by_idx (int index)
587 struct list_head *ptmp;
589 struct list_head *ctmp;
593 read_lock_irqsave(&kibnal_data.kib_global_lock, flags);
595 for (i = 0; i < kibnal_data.kib_peer_hash_size; i++) {
596 list_for_each (ptmp, &kibnal_data.kib_peers[i]) {
598 peer = list_entry (ptmp, kib_peer_t, ibp_list);
599 LASSERT (peer->ibp_persistence > 0 ||
600 peer->ibp_connecting != 0 ||
601 !list_empty (&peer->ibp_conns));
603 list_for_each (ctmp, &peer->ibp_conns) {
607 conn = list_entry (ctmp, kib_conn_t, ibc_list);
608 CDEBUG(D_NET, "++conn[%p] state %d -> "LPX64" (%d)\n",
609 conn, conn->ibc_state, conn->ibc_peer->ibp_nid,
610 atomic_read (&conn->ibc_refcount));
611 atomic_inc (&conn->ibc_refcount);
612 read_unlock_irqrestore(&kibnal_data.kib_global_lock,
619 read_unlock_irqrestore(&kibnal_data.kib_global_lock, flags);
624 kibnal_create_conn (void)
635 IB_QP_ATTRIBUTES_CREATE qp_create;
636 IB_QP_ATTRIBUTES_MODIFY qp_attr;
639 PORTAL_ALLOC (conn, sizeof (*conn));
641 CERROR ("Can't allocate connection\n");
645 /* zero flags, NULL pointers etc... */
646 memset (conn, 0, sizeof (*conn));
648 INIT_LIST_HEAD (&conn->ibc_tx_queue);
649 INIT_LIST_HEAD (&conn->ibc_active_txs);
650 spin_lock_init (&conn->ibc_lock);
652 atomic_inc (&kibnal_data.kib_nconns);
653 /* well not really, but I call destroy() on failure, which decrements */
655 PORTAL_ALLOC (conn->ibc_rxs, IBNAL_RX_MSGS * sizeof (kib_rx_t));
656 if (conn->ibc_rxs == NULL)
658 memset (conn->ibc_rxs, 0, IBNAL_RX_MSGS * sizeof(kib_rx_t));
660 rc = kibnal_alloc_pages(&conn->ibc_rx_pages, IBNAL_RX_MSG_PAGES, 1);
664 vaddr_base = vaddr = conn->ibc_rx_pages->ibp_vaddr;
666 for (i = ipage = page_offset = 0; i < IBNAL_RX_MSGS; i++) {
667 struct page *page = conn->ibc_rx_pages->ibp_pages[ipage];
668 kib_rx_t *rx = &conn->ibc_rxs[i];
671 rx->rx_msg = (kib_msg_t *)(((char *)page_address(page)) +
674 if (kibnal_whole_mem())
675 rx->rx_vaddr = kibnal_page2phys(page) +
677 kibnal_data.kib_md.md_addr;
679 rx->rx_vaddr = vaddr;
681 vaddr += IBNAL_MSG_SIZE;
682 LASSERT (vaddr <= vaddr_base + IBNAL_RX_MSG_BYTES);
684 page_offset += IBNAL_MSG_SIZE;
685 LASSERT (page_offset <= PAGE_SIZE);
687 if (page_offset == PAGE_SIZE) {
690 LASSERT (ipage <= IBNAL_RX_MSG_PAGES);
694 params.qp_create = (IB_QP_ATTRIBUTES_CREATE) {
695 .Type = QPTypeReliableConnected,
696 .SendQDepth = IBNAL_TX_MAX_SG *
697 IBNAL_MSG_QUEUE_SIZE,
698 .RecvQDepth = IBNAL_MSG_QUEUE_SIZE,
699 .SendDSListDepth = 1,
700 .RecvDSListDepth = 1,
701 .SendCQHandle = kibnal_data.kib_cq,
702 .RecvCQHandle = kibnal_data.kib_cq,
703 .PDHandle = kibnal_data.kib_pd,
704 .SendSignaledCompletions = TRUE,
706 frc = iibt_qp_create(kibnal_data.kib_hca, ¶ms.qp_create, NULL,
707 &conn->ibc_qp, &conn->ibc_qp_attrs);
709 CERROR ("Failed to create queue pair: %d\n", rc);
713 /* Mark QP created */
714 conn->ibc_state = IBNAL_CONN_INIT_QP;
716 params.qp_attr = (IB_QP_ATTRIBUTES_MODIFY) {
717 .RequestState = QPStateInit,
718 .Attrs = (IB_QP_ATTR_PORTGUID |
719 IB_QP_ATTR_PKEYINDEX |
720 IB_QP_ATTR_ACCESSCONTROL),
721 .PortGUID = kibnal_data.kib_port_guid,
730 rc = iibt_qp_modify(conn->ibc_qp, ¶ms.qp_attr, NULL);
732 CERROR ("Failed to modify queue pair: %d\n", rc);
736 /* 1 ref for caller */
737 atomic_set (&conn->ibc_refcount, 1);
741 kibnal_destroy_conn (conn);
746 kibnal_destroy_conn (kib_conn_t *conn)
751 CDEBUG (D_NET, "connection %p\n", conn);
753 LASSERT (atomic_read (&conn->ibc_refcount) == 0);
754 LASSERT (list_empty(&conn->ibc_tx_queue));
755 LASSERT (list_empty(&conn->ibc_active_txs));
756 LASSERT (conn->ibc_nsends_posted == 0);
757 LASSERT (conn->ibc_connreq == NULL);
759 switch (conn->ibc_state) {
760 case IBNAL_CONN_DISCONNECTED:
761 /* called after connection sequence initiated */
764 case IBNAL_CONN_INIT_QP:
765 /* _destroy includes an implicit Reset of the QP which
766 * discards posted work */
767 rc = iibt_qp_destroy(conn->ibc_qp);
769 CERROR("Can't destroy QP: %d\n", rc);
772 case IBNAL_CONN_INIT_NOTHING:
779 if (conn->ibc_cep != NULL) {
780 frc = iibt_cm_destroy_cep(conn->ibc_cep);
782 CERROR("Can't destroy CEP %p: %d\n", conn->ibc_cep,
786 if (conn->ibc_rx_pages != NULL)
787 kibnal_free_pages(conn->ibc_rx_pages);
789 if (conn->ibc_rxs != NULL)
790 PORTAL_FREE(conn->ibc_rxs,
791 IBNAL_RX_MSGS * sizeof(kib_rx_t));
793 if (conn->ibc_peer != NULL)
794 kib_peer_decref(conn->ibc_peer);
796 PORTAL_FREE(conn, sizeof (*conn));
798 atomic_dec(&kibnal_data.kib_nconns);
800 if (atomic_read (&kibnal_data.kib_nconns) == 0 &&
801 kibnal_data.kib_shutdown) {
802 /* I just nuked the last connection on shutdown; wake up
803 * everyone so they can exit. */
804 wake_up_all(&kibnal_data.kib_sched_waitq);
805 wake_up_all(&kibnal_data.kib_connd_waitq);
810 kibnal_put_conn (kib_conn_t *conn)
814 CDEBUG (D_NET, "putting conn[%p] state %d -> "LPX64" (%d)\n",
815 conn, conn->ibc_state, conn->ibc_peer->ibp_nid,
816 atomic_read (&conn->ibc_refcount));
818 LASSERT (atomic_read (&conn->ibc_refcount) > 0);
819 if (!atomic_dec_and_test (&conn->ibc_refcount))
822 /* must disconnect before dropping the final ref */
823 LASSERT (conn->ibc_state == IBNAL_CONN_DISCONNECTED);
825 spin_lock_irqsave (&kibnal_data.kib_connd_lock, flags);
827 list_add (&conn->ibc_list, &kibnal_data.kib_connd_conns);
828 wake_up (&kibnal_data.kib_connd_waitq);
830 spin_unlock_irqrestore (&kibnal_data.kib_connd_lock, flags);
834 kibnal_close_peer_conns_locked (kib_peer_t *peer, int why)
837 struct list_head *ctmp;
838 struct list_head *cnxt;
841 list_for_each_safe (ctmp, cnxt, &peer->ibp_conns) {
842 conn = list_entry (ctmp, kib_conn_t, ibc_list);
845 kibnal_close_conn_locked (conn, why);
852 kibnal_close_stale_conns_locked (kib_peer_t *peer, __u64 incarnation)
855 struct list_head *ctmp;
856 struct list_head *cnxt;
859 list_for_each_safe (ctmp, cnxt, &peer->ibp_conns) {
860 conn = list_entry (ctmp, kib_conn_t, ibc_list);
862 if (conn->ibc_incarnation == incarnation)
865 CDEBUG(D_NET, "Closing stale conn nid:"LPX64" incarnation:"LPX64"("LPX64")\n",
866 peer->ibp_nid, conn->ibc_incarnation, incarnation);
869 kibnal_close_conn_locked (conn, -ESTALE);
876 kibnal_close_matching_conns (ptl_nid_t nid)
880 struct list_head *ptmp;
881 struct list_head *pnxt;
887 write_lock_irqsave (&kibnal_data.kib_global_lock, flags);
889 if (nid != PTL_NID_ANY)
890 lo = hi = kibnal_nid2peerlist(nid) - kibnal_data.kib_peers;
893 hi = kibnal_data.kib_peer_hash_size - 1;
896 for (i = lo; i <= hi; i++) {
897 list_for_each_safe (ptmp, pnxt, &kibnal_data.kib_peers[i]) {
899 peer = list_entry (ptmp, kib_peer_t, ibp_list);
900 LASSERT (peer->ibp_persistence != 0 ||
901 peer->ibp_connecting != 0 ||
902 !list_empty (&peer->ibp_conns));
904 if (!(nid == PTL_NID_ANY || nid == peer->ibp_nid))
907 count += kibnal_close_peer_conns_locked (peer, 0);
911 write_unlock_irqrestore (&kibnal_data.kib_global_lock, flags);
913 /* wildcards always succeed */
914 if (nid == PTL_NID_ANY)
917 return (count == 0 ? -ENOENT : 0);
921 kibnal_cmd(struct portals_cfg *pcfg, void * private)
926 LASSERT (pcfg != NULL);
928 switch(pcfg->pcfg_command) {
929 case NAL_CMD_GET_PEER: {
933 rc = kibnal_get_peer_info(pcfg->pcfg_count,
935 pcfg->pcfg_nid = nid;
939 pcfg->pcfg_count = 0;
940 pcfg->pcfg_wait = share_count;
943 case NAL_CMD_ADD_PEER: {
944 rc = kibnal_add_persistent_peer (pcfg->pcfg_nid);
947 case NAL_CMD_DEL_PEER: {
948 rc = kibnal_del_peer (pcfg->pcfg_nid,
949 /* flags == single_share */
950 pcfg->pcfg_flags != 0);
953 case NAL_CMD_GET_CONN: {
954 kib_conn_t *conn = kibnal_get_conn_by_idx (pcfg->pcfg_count);
960 pcfg->pcfg_nid = conn->ibc_peer->ibp_nid;
963 pcfg->pcfg_flags = 0;
964 kibnal_put_conn (conn);
968 case NAL_CMD_CLOSE_CONNECTION: {
969 rc = kibnal_close_matching_conns (pcfg->pcfg_nid);
972 case NAL_CMD_REGISTER_MYNID: {
973 if (pcfg->pcfg_nid == PTL_NID_ANY)
976 rc = kibnal_set_mynid (pcfg->pcfg_nid);
985 kibnal_free_pages (kib_pages_t *p)
987 int npages = p->ibp_npages;
992 rc = iibt_deregister_memory(p->ibp_handle);
994 CERROR ("Deregister error: %d\n", rc);
997 for (i = 0; i < npages; i++)
998 if (p->ibp_pages[i] != NULL)
999 __free_page(p->ibp_pages[i]);
1001 PORTAL_FREE (p, offsetof(kib_pages_t, ibp_pages[npages]));
1005 kibnal_alloc_pages (kib_pages_t **pp, int npages, int allow_write)
1011 IB_ACCESS_CONTROL access;
1013 memset(&access, 0, sizeof(access));
1014 access.s.MWBindable = 1;
1015 access.s.LocalWrite = 1;
1016 access.s.RdmaRead = 1;
1017 access.s.RdmaWrite = 1;
1019 PORTAL_ALLOC(p, offsetof(kib_pages_t, ibp_pages[npages]));
1021 CERROR ("Can't allocate buffer %d\n", npages);
1025 memset (p, 0, offsetof(kib_pages_t, ibp_pages[npages]));
1026 p->ibp_npages = npages;
1028 for (i = 0; i < npages; i++) {
1029 p->ibp_pages[i] = alloc_page (GFP_KERNEL);
1030 if (p->ibp_pages[i] == NULL) {
1031 CERROR ("Can't allocate page %d of %d\n", i, npages);
1032 kibnal_free_pages(p);
1037 if (kibnal_whole_mem())
1040 PORTAL_ALLOC(phys_pages, npages * sizeof(*phys_pages));
1041 if (phys_pages == NULL) {
1042 CERROR ("Can't allocate physarray for %d pages\n", npages);
1043 /* XXX free ibp_pages? */
1044 kibnal_free_pages(p);
1048 /* if we were using the _contig_ registration variant we would have
1049 * an array of PhysAddr/Length pairs, but the discontiguous variant
1050 * just takes the PhysAddr */
1051 for (i = 0; i < npages; i++)
1052 phys_pages[i] = kibnal_page2phys(p->ibp_pages[i]);
1054 frc = iibt_register_physical_memory(kibnal_data.kib_hca,
1055 0, /* requested vaddr */
1060 &p->ibp_handle, &p->ibp_vaddr,
1061 &p->ibp_lkey, &p->ibp_rkey);
1063 PORTAL_FREE(phys_pages, npages * sizeof(*phys_pages));
1065 if (frc != FSUCCESS) {
1066 CERROR ("Error %d mapping %d pages\n", frc, npages);
1067 kibnal_free_pages(p);
1071 CDEBUG(D_NET, "registered %d pages; handle: %p vaddr "LPX64" "
1072 "lkey %x rkey %x\n", npages, p->ibp_handle,
1073 p->ibp_vaddr, p->ibp_lkey, p->ibp_rkey);
1082 kibnal_setup_tx_descs (void)
1085 int page_offset = 0;
1093 /* pre-mapped messages are not bigger than 1 page */
1094 LASSERT (IBNAL_MSG_SIZE <= PAGE_SIZE);
1096 /* No fancy arithmetic when we do the buffer calculations */
1097 LASSERT (PAGE_SIZE % IBNAL_MSG_SIZE == 0);
1099 rc = kibnal_alloc_pages(&kibnal_data.kib_tx_pages, IBNAL_TX_MSG_PAGES,
1104 /* ignored for the whole_mem case */
1105 vaddr = vaddr_base = kibnal_data.kib_tx_pages->ibp_vaddr;
1107 for (i = 0; i < IBNAL_TX_MSGS; i++) {
1108 page = kibnal_data.kib_tx_pages->ibp_pages[ipage];
1109 tx = &kibnal_data.kib_tx_descs[i];
1111 memset (tx, 0, sizeof(*tx)); /* zero flags etc */
1113 tx->tx_msg = (kib_msg_t *)(((char *)page_address(page)) +
1116 if (kibnal_whole_mem())
1117 tx->tx_vaddr = kibnal_page2phys(page) +
1119 kibnal_data.kib_md.md_addr;
1121 tx->tx_vaddr = vaddr;
1123 tx->tx_isnblk = (i >= IBNAL_NTX);
1124 tx->tx_mapped = KIB_TX_UNMAPPED;
1126 CDEBUG(D_NET, "Tx[%d] %p->%p - "LPX64"\n",
1127 i, tx, tx->tx_msg, tx->tx_vaddr);
1130 list_add (&tx->tx_list,
1131 &kibnal_data.kib_idle_nblk_txs);
1133 list_add (&tx->tx_list,
1134 &kibnal_data.kib_idle_txs);
1136 vaddr += IBNAL_MSG_SIZE;
1137 LASSERT (vaddr <= vaddr_base + IBNAL_TX_MSG_BYTES);
1139 page_offset += IBNAL_MSG_SIZE;
1140 LASSERT (page_offset <= PAGE_SIZE);
1142 if (page_offset == PAGE_SIZE) {
1145 LASSERT (ipage <= IBNAL_TX_MSG_PAGES);
1153 kibnal_api_shutdown (nal_t *nal)
1158 if (nal->nal_refct != 0) {
1159 /* This module got the first ref */
1160 PORTAL_MODULE_UNUSE;
1164 CDEBUG(D_MALLOC, "before NAL cleanup: kmem %d\n",
1165 atomic_read (&portal_kmemory));
1167 LASSERT(nal == &kibnal_api);
1169 switch (kibnal_data.kib_init) {
1171 CERROR ("Unexpected state %d\n", kibnal_data.kib_init);
1174 case IBNAL_INIT_ALL:
1175 /* stop calls to nal_cmd */
1176 libcfs_nal_cmd_unregister(IIBNAL);
1179 /* resetting my NID to unadvertises me, removes my
1180 * listener and nukes all current peers */
1181 kibnal_set_mynid (PTL_NID_ANY);
1183 /* Wait for all peer state to clean up (crazy) */
1185 while (atomic_read (&kibnal_data.kib_npeers) != 0) {
1187 CDEBUG(((i & (-i)) == i) ? D_WARNING : D_NET, /* power of 2? */
1188 "waiting for %d peers to disconnect (can take a few seconds)\n",
1189 atomic_read (&kibnal_data.kib_npeers));
1190 set_current_state (TASK_UNINTERRUPTIBLE);
1191 schedule_timeout (HZ);
1196 rc = iibt_cq_destroy(kibnal_data.kib_cq);
1198 CERROR ("Destroy CQ error: %d\n", rc);
1201 case IBNAL_INIT_TXD:
1202 kibnal_free_pages (kibnal_data.kib_tx_pages);
1206 if (kibnal_data.kib_md.md_handle != NULL) {
1207 rc = iibt_deregister_memory(kibnal_data.kib_md.md_handle);
1209 CERROR ("Deregister memory: %d\n", rc);
1214 case IBNAL_INIT_FMR:
1215 rc = ib_fmr_pool_destroy (kibnal_data.kib_fmr_pool);
1217 CERROR ("Destroy FMR pool error: %d\n", rc);
1221 rc = iibt_pd_free(kibnal_data.kib_pd);
1223 CERROR ("Destroy PD error: %d\n", rc);
1227 rc = iibt_sd_deregister(kibnal_data.kib_sd);
1229 CERROR ("Deregister SD error: %d\n", rc);
1232 case IBNAL_INIT_PORT:
1236 case IBNAL_INIT_PORTATTRS:
1237 PORTAL_FREE(kibnal_data.kib_hca_attrs.PortAttributesList,
1238 kibnal_data.kib_hca_attrs.PortAttributesListSize);
1241 case IBNAL_INIT_HCA:
1242 rc = iibt_close_hca(kibnal_data.kib_hca);
1244 CERROR ("Close HCA error: %d\n", rc);
1247 case IBNAL_INIT_LIB:
1248 lib_fini(&kibnal_lib);
1251 case IBNAL_INIT_DATA:
1252 /* Module refcount only gets to zero when all peers
1253 * have been closed so all lists must be empty */
1254 LASSERT (atomic_read (&kibnal_data.kib_npeers) == 0);
1255 LASSERT (kibnal_data.kib_peers != NULL);
1256 for (i = 0; i < kibnal_data.kib_peer_hash_size; i++) {
1257 LASSERT (list_empty (&kibnal_data.kib_peers[i]));
1259 LASSERT (atomic_read (&kibnal_data.kib_nconns) == 0);
1260 LASSERT (list_empty (&kibnal_data.kib_sched_rxq));
1261 LASSERT (list_empty (&kibnal_data.kib_sched_txq));
1262 LASSERT (list_empty (&kibnal_data.kib_connd_conns));
1263 LASSERT (list_empty (&kibnal_data.kib_connd_peers));
1265 /* flag threads to terminate; wake and wait for them to die */
1266 kibnal_data.kib_shutdown = 1;
1267 wake_up_all (&kibnal_data.kib_sched_waitq);
1268 wake_up_all (&kibnal_data.kib_connd_waitq);
1271 while (atomic_read (&kibnal_data.kib_nthreads) != 0) {
1273 CDEBUG(((i & (-i)) == i) ? D_WARNING : D_NET, /* power of 2? */
1274 "Waiting for %d threads to terminate\n",
1275 atomic_read (&kibnal_data.kib_nthreads));
1276 set_current_state (TASK_INTERRUPTIBLE);
1277 schedule_timeout (HZ);
1281 case IBNAL_INIT_NOTHING:
1285 if (kibnal_data.kib_tx_descs != NULL)
1286 PORTAL_FREE (kibnal_data.kib_tx_descs,
1287 IBNAL_TX_MSGS * sizeof(kib_tx_t));
1289 if (kibnal_data.kib_peers != NULL)
1290 PORTAL_FREE (kibnal_data.kib_peers,
1291 sizeof (struct list_head) *
1292 kibnal_data.kib_peer_hash_size);
1294 CDEBUG(D_MALLOC, "after NAL cleanup: kmem %d\n",
1295 atomic_read (&portal_kmemory));
1296 printk(KERN_INFO "Lustre: Infinicon IB NAL unloaded (final mem %d)\n",
1297 atomic_read(&portal_kmemory));
1299 kibnal_data.kib_init = IBNAL_INIT_NOTHING;
1302 #define roundup_power(val, power) \
1303 ( (val + (__u64)(power - 1)) & ~((__u64)(power - 1)) )
1305 /* this isn't very portable or sturdy in the face of funny mem/bus configs */
1306 static __u64 max_phys_mem(IB_CA_ATTRIBUTES *ca_attr)
1311 /* XXX we don't bother with first-gen cards */
1312 if (ca_attr->VendorId == 0xd0b7 && ca_attr->DeviceId == 0x3101)
1316 ret = (__u64)max(si.totalram, max_mapnr) * si.mem_unit;
1317 return roundup_power(ret, 128 * 1024 * 1024);
1319 #undef roundup_power
1322 kibnal_api_startup (nal_t *nal, ptl_pid_t requested_pid,
1323 ptl_ni_limits_t *requested_limits,
1324 ptl_ni_limits_t *actual_limits)
1326 ptl_process_id_t process_id;
1327 int pkmem = atomic_read(&portal_kmemory);
1328 IB_PORT_ATTRIBUTES *pattr;
1334 LASSERT (nal == &kibnal_api);
1336 if (nal->nal_refct != 0) {
1337 if (actual_limits != NULL)
1338 *actual_limits = kibnal_lib.libnal_ni.ni_actual_limits;
1339 /* This module got the first ref */
1344 LASSERT (kibnal_data.kib_init == IBNAL_INIT_NOTHING);
1346 frc = IbtGetInterfaceByVersion(IBT_INTERFACE_VERSION_2,
1347 &kibnal_data.kib_interfaces);
1348 if (frc != FSUCCESS) {
1349 CERROR("IbtGetInterfaceByVersion(IBT_INTERFACE_VERSION_2) = %d\n",
1354 init_MUTEX (&kibnal_data.kib_nid_mutex);
1355 init_MUTEX_LOCKED (&kibnal_data.kib_nid_signal);
1356 kibnal_data.kib_nid = PTL_NID_ANY;
1358 rwlock_init(&kibnal_data.kib_global_lock);
1360 kibnal_data.kib_peer_hash_size = IBNAL_PEER_HASH_SIZE;
1361 PORTAL_ALLOC (kibnal_data.kib_peers,
1362 sizeof (struct list_head) * kibnal_data.kib_peer_hash_size);
1363 if (kibnal_data.kib_peers == NULL) {
1366 for (i = 0; i < kibnal_data.kib_peer_hash_size; i++)
1367 INIT_LIST_HEAD(&kibnal_data.kib_peers[i]);
1369 spin_lock_init (&kibnal_data.kib_connd_lock);
1370 INIT_LIST_HEAD (&kibnal_data.kib_connd_peers);
1371 INIT_LIST_HEAD (&kibnal_data.kib_connd_conns);
1372 init_waitqueue_head (&kibnal_data.kib_connd_waitq);
1374 spin_lock_init (&kibnal_data.kib_sched_lock);
1375 INIT_LIST_HEAD (&kibnal_data.kib_sched_txq);
1376 INIT_LIST_HEAD (&kibnal_data.kib_sched_rxq);
1377 init_waitqueue_head (&kibnal_data.kib_sched_waitq);
1379 spin_lock_init (&kibnal_data.kib_tx_lock);
1380 INIT_LIST_HEAD (&kibnal_data.kib_idle_txs);
1381 INIT_LIST_HEAD (&kibnal_data.kib_idle_nblk_txs);
1382 init_waitqueue_head(&kibnal_data.kib_idle_tx_waitq);
1384 PORTAL_ALLOC (kibnal_data.kib_tx_descs,
1385 IBNAL_TX_MSGS * sizeof(kib_tx_t));
1386 if (kibnal_data.kib_tx_descs == NULL) {
1387 CERROR ("Can't allocate tx descs\n");
1391 /* lists/ptrs/locks initialised */
1392 kibnal_data.kib_init = IBNAL_INIT_DATA;
1393 /*****************************************************/
1395 process_id.pid = requested_pid;
1396 process_id.nid = kibnal_data.kib_nid;
1398 rc = lib_init(&kibnal_lib, nal, process_id,
1399 requested_limits, actual_limits);
1401 CERROR("lib_init failed: error %d\n", rc);
1405 /* lib interface initialised */
1406 kibnal_data.kib_init = IBNAL_INIT_LIB;
1407 /*****************************************************/
1409 for (i = 0; i < IBNAL_N_SCHED; i++) {
1410 rc = kibnal_thread_start (kibnal_scheduler, (void *)i);
1412 CERROR("Can't spawn iibnal scheduler[%d]: %d\n",
1418 rc = kibnal_thread_start (kibnal_connd, NULL);
1420 CERROR ("Can't spawn iibnal connd: %d\n", rc);
1424 n = sizeof(kibnal_data.kib_hca_guids) /
1425 sizeof(kibnal_data.kib_hca_guids[0]);
1426 frc = iibt_get_hca_guids(&n, kibnal_data.kib_hca_guids);
1427 if (frc != FSUCCESS) {
1428 CERROR ("Can't get channel adapter guids: %d\n", frc);
1432 CERROR ("No channel adapters found\n");
1436 /* Infinicon has per-HCA rather than per CQ completion handlers */
1437 frc = iibt_open_hca(kibnal_data.kib_hca_guids[0],
1439 kibnal_ca_async_callback,
1440 &kibnal_data.kib_hca,
1441 &kibnal_data.kib_hca);
1442 if (frc != FSUCCESS) {
1443 CERROR ("Can't open CA[0]: %d\n", frc);
1447 /* Channel Adapter opened */
1448 kibnal_data.kib_init = IBNAL_INIT_HCA;
1449 /*****************************************************/
1451 kibnal_data.kib_hca_attrs.PortAttributesList = NULL;
1452 kibnal_data.kib_hca_attrs.PortAttributesListSize = 0;
1453 frc = iibt_query_hca(kibnal_data.kib_hca,
1454 &kibnal_data.kib_hca_attrs, NULL);
1455 if (frc != FSUCCESS) {
1456 CERROR ("Can't size port attrs: %d\n", frc);
1460 PORTAL_ALLOC(kibnal_data.kib_hca_attrs.PortAttributesList,
1461 kibnal_data.kib_hca_attrs.PortAttributesListSize);
1462 if (kibnal_data.kib_hca_attrs.PortAttributesList == NULL)
1465 /* Port attrs allocated */
1466 kibnal_data.kib_init = IBNAL_INIT_PORTATTRS;
1467 /*****************************************************/
1469 frc = iibt_query_hca(kibnal_data.kib_hca, &kibnal_data.kib_hca_attrs,
1471 if (frc != FSUCCESS) {
1472 CERROR ("Can't get port attrs for CA 0: %d\n", frc);
1476 for (i = 0, pattr = kibnal_data.kib_hca_attrs.PortAttributesList;
1478 i++, pattr = pattr->Next) {
1479 switch (pattr->PortState) {
1481 CERROR("Unexpected port[%d] state %d\n",
1482 i, pattr->PortState);
1485 CDEBUG(D_NET, "port[%d] Down\n", i);
1488 CDEBUG(D_NET, "port[%d] Init\n", i);
1490 case PortStateArmed:
1491 CDEBUG(D_NET, "port[%d] Armed\n", i);
1494 case PortStateActive:
1495 CDEBUG(D_NET, "port[%d] Active\n", i);
1496 kibnal_data.kib_port = i;
1497 kibnal_data.kib_port_guid = pattr->GUID;
1498 kibnal_data.kib_port_pkey = pattr->PkeyTable[0];
1504 if (pattr == NULL) {
1505 CERROR ("Can't find an active port\n");
1509 CDEBUG(D_NET, "got guid "LPX64"\n", kibnal_data.kib_port_guid);
1511 /* Active port found */
1512 kibnal_data.kib_init = IBNAL_INIT_PORT;
1513 /*****************************************************/
1515 frc = iibt_sd_register(&kibnal_data.kib_sd, NULL);
1516 if (frc != FSUCCESS) {
1517 CERROR ("Can't register with SD: %d\n", frc);
1521 /* Registered with SD OK */
1522 kibnal_data.kib_init = IBNAL_INIT_SD;
1523 /*****************************************************/
1525 frc = iibt_pd_allocate(kibnal_data.kib_hca, 0, &kibnal_data.kib_pd);
1526 if (frc != FSUCCESS) {
1527 CERROR ("Can't create PD: %d\n", rc);
1531 /* flag PD initialised */
1532 kibnal_data.kib_init = IBNAL_INIT_PD;
1533 /*****************************************************/
1537 const int pool_size = IBNAL_NTX + IBNAL_NTX_NBLK;
1538 struct ib_fmr_pool_param params = {
1539 .max_pages_per_fmr = PTL_MTU/PAGE_SIZE,
1540 .access = (IB_ACCESS_LOCAL_WRITE |
1541 IB_ACCESS_REMOTE_WRITE |
1542 IB_ACCESS_REMOTE_READ),
1543 .pool_size = pool_size,
1544 .dirty_watermark = (pool_size * 3)/4,
1545 .flush_function = NULL,
1549 rc = ib_fmr_pool_create(kibnal_data.kib_pd, ¶ms,
1550 &kibnal_data.kib_fmr_pool);
1552 CERROR ("Can't create FMR pool size %d: %d\n",
1558 /* flag FMR pool initialised */
1559 kibnal_data.kib_init = IBNAL_INIT_FMR;
1561 /*****************************************************/
1562 if (IBNAL_WHOLE_MEM) {
1563 IB_MR_PHYS_BUFFER phys;
1564 IB_ACCESS_CONTROL access;
1565 kib_md_t *md = &kibnal_data.kib_md;
1567 memset(&access, 0, sizeof(access));
1568 access.s.MWBindable = 1;
1569 access.s.LocalWrite = 1;
1570 access.s.RdmaRead = 1;
1571 access.s.RdmaWrite = 1;
1574 phys.Length = max_phys_mem(&kibnal_data.kib_hca_attrs);
1575 if (phys.Length == 0) {
1576 CERROR ("couldn't determine the end of phys mem\n");
1580 rc = iibt_register_contig_physical_memory(kibnal_data.kib_hca,
1590 if (rc != FSUCCESS) {
1591 CERROR("registering physical memory failed: %d\n",
1593 CERROR("falling back to registration per-rdma\n");
1594 md->md_handle = NULL;
1596 CDEBUG(D_NET, "registered "LPU64" bytes of mem\n",
1598 kibnal_data.kib_init = IBNAL_INIT_MR;
1602 /*****************************************************/
1604 rc = kibnal_setup_tx_descs();
1606 CERROR ("Can't register tx descs: %d\n", rc);
1610 /* flag TX descs initialised */
1611 kibnal_data.kib_init = IBNAL_INIT_TXD;
1612 /*****************************************************/
1617 frc = iibt_cq_create(kibnal_data.kib_hca, IBNAL_CQ_ENTRIES,
1618 &kibnal_data.kib_cq, &kibnal_data.kib_cq,
1620 if (frc != FSUCCESS) {
1621 CERROR ("Can't create RX CQ: %d\n", frc);
1625 /* flag CQ initialised */
1626 kibnal_data.kib_init = IBNAL_INIT_CQ;
1628 if (nentries < IBNAL_CQ_ENTRIES) {
1629 CERROR ("CQ only has %d entries, need %d\n",
1630 nentries, IBNAL_CQ_ENTRIES);
1634 rc = iibt_cq_rearm(kibnal_data.kib_cq, CQEventSelNextWC);
1636 CERROR ("Failed to re-arm completion queue: %d\n", rc);
1641 /*****************************************************/
1643 rc = libcfs_nal_cmd_register(IIBNAL, &kibnal_cmd, NULL);
1645 CERROR ("Can't initialise command interface (rc = %d)\n", rc);
1649 /* flag everything initialised */
1650 kibnal_data.kib_init = IBNAL_INIT_ALL;
1651 /*****************************************************/
1653 printk(KERN_INFO "Lustre: Infinicon IB NAL loaded "
1654 "(initial mem %d)\n", pkmem);
1659 kibnal_api_shutdown (&kibnal_api);
1664 kibnal_module_fini (void)
1666 #ifdef CONFIG_SYSCTL
1667 if (kibnal_tunables.kib_sysctl != NULL)
1668 unregister_sysctl_table (kibnal_tunables.kib_sysctl);
1670 PtlNIFini(kibnal_ni);
1672 ptl_unregister_nal(IIBNAL);
1676 kibnal_module_init (void)
1680 if (sizeof(kib_wire_connreq_t) > CM_REQUEST_INFO_USER_LEN) {
1681 CERROR("sizeof(kib_wire_connreq_t) > CM_REQUEST_INFO_USER_LEN\n");
1685 /* the following must be sizeof(int) for proc_dointvec() */
1686 if (sizeof (kibnal_tunables.kib_io_timeout) != sizeof (int)) {
1687 CERROR("sizeof (kibnal_tunables.kib_io_timeout) != sizeof (int)\n");
1691 kibnal_api.nal_ni_init = kibnal_api_startup;
1692 kibnal_api.nal_ni_fini = kibnal_api_shutdown;
1694 /* Initialise dynamic tunables to defaults once only */
1695 kibnal_tunables.kib_io_timeout = IBNAL_IO_TIMEOUT;
1697 rc = ptl_register_nal(IIBNAL, &kibnal_api);
1699 CERROR("Can't register IBNAL: %d\n", rc);
1700 return (-ENOMEM); /* or something... */
1703 /* Pure gateways want the NAL started up at module load time... */
1704 rc = PtlNIInit(IIBNAL, LUSTRE_SRV_PTL_PID, NULL, NULL, &kibnal_ni);
1705 if (rc != PTL_OK && rc != PTL_IFACE_DUP) {
1706 ptl_unregister_nal(IIBNAL);
1710 #ifdef CONFIG_SYSCTL
1711 /* Press on regardless even if registering sysctl doesn't work */
1712 kibnal_tunables.kib_sysctl =
1713 register_sysctl_table (kibnal_top_ctl_table, 0);
1718 MODULE_AUTHOR("Cluster File Systems, Inc. <info@clusterfs.com>");
1719 MODULE_DESCRIPTION("Kernel Infinicon IB NAL v0.01");
1720 MODULE_LICENSE("GPL");
1722 module_init(kibnal_module_init);
1723 module_exit(kibnal_module_fini);