1 /* -*- mode: c; c-basic-offset: 8; indent-tabs-mode: nil; -*-
2 * vim:expandtab:shiftwidth=8:tabstop=8:
4 * Copyright (C) 2004 Cluster File Systems, Inc.
5 * Author: Eric Barton <eric@bartonsoftware.com>
7 * This file is part of Lustre, http://www.lustre.org.
9 * Lustre is free software; you can redistribute it and/or
10 * modify it under the terms of version 2 of the GNU General Public
11 * License as published by the Free Software Foundation.
13 * Lustre is distributed in the hope that it will be useful,
14 * but WITHOUT ANY WARRANTY; without even the implied warranty of
15 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
16 * GNU General Public License for more details.
18 * You should have received a copy of the GNU General Public License
19 * along with Lustre; if not, write to the Free Software
20 * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
24 #include "openibnal.h"
27 ptl_handle_ni_t kibnal_ni;
28 kib_data_t kibnal_data;
29 kib_tunables_t kibnal_tunables;
31 #define IBNAL_SYSCTL 202
34 IBNAL_SYSCTL_TIMEOUT=1,
35 IBNAL_SYSCTL_LISTENER_TIMEOUT,
40 static ctl_table kibnal_ctl_table[] = {
41 {IBNAL_SYSCTL_TIMEOUT, "timeout",
42 &kibnal_tunables.kib_io_timeout, sizeof (int),
43 0644, NULL, &proc_dointvec},
44 {IBNAL_SYSCTL_LISTENER_TIMEOUT, "listener_timeout",
45 &kibnal_tunables.kib_listener_timeout, sizeof(int),
46 0644, NULL, &proc_dointvec},
47 {IBNAL_SYSCTL_BACKLOG, "backlog",
48 &kibnal_tunables.kib_backlog, sizeof(int),
49 0644, NULL, kibnal_listener_procint},
50 {IBNAL_SYSCTL_PORT, "port",
51 &kibnal_tunables.kib_port, sizeof(int),
52 0644, NULL, kibnal_listener_procint},
56 static ctl_table kibnal_top_ctl_table[] = {
57 {IBNAL_SYSCTL, "openibnal", NULL, 0, 0555, kibnal_ctl_table},
62 kibnal_cksum (void *ptr, int nob)
68 sum = ((sum << 1) | (sum >> 31)) + *c++;
70 /* ensure I don't return 0 (== no checksum) */
71 return (sum == 0) ? 1 : sum;
75 kibnal_init_msg(kib_msg_t *msg, int type, int body_nob)
78 msg->ibm_nob = offsetof(kib_msg_t, ibm_u) + body_nob;
82 kibnal_pack_msg(kib_msg_t *msg, int credits, ptl_nid_t dstnid, __u64 dststamp)
84 /* CAVEAT EMPTOR! all message fields not set here should have been
85 * initialised previously. */
86 msg->ibm_magic = IBNAL_MSG_MAGIC;
87 msg->ibm_version = IBNAL_MSG_VERSION;
89 msg->ibm_credits = credits;
92 msg->ibm_srcnid = kibnal_lib.libnal_ni.ni_pid.nid;
93 msg->ibm_srcstamp = kibnal_data.kib_incarnation;
94 msg->ibm_dstnid = dstnid;
95 msg->ibm_dststamp = dststamp;
97 /* NB ibm_cksum zero while computing cksum */
98 msg->ibm_cksum = kibnal_cksum(msg, msg->ibm_nob);
103 kibnal_unpack_msg(kib_msg_t *msg, int nob)
105 const int hdr_size = offsetof(kib_msg_t, ibm_u);
111 CERROR("Short message: %d\n", nob);
115 if (msg->ibm_magic == IBNAL_MSG_MAGIC) {
117 } else if (msg->ibm_magic == __swab32(IBNAL_MSG_MAGIC)) {
120 CERROR("Bad magic: %08x\n", msg->ibm_magic);
124 if (msg->ibm_version !=
125 (flip ? __swab16(IBNAL_MSG_VERSION) : IBNAL_MSG_VERSION)) {
126 CERROR("Bad version: %d\n", msg->ibm_version);
130 if (nob < hdr_size) {
131 CERROR("Short message: %d\n", nob);
135 msg_nob = flip ? __swab32(msg->ibm_nob) : msg->ibm_nob;
137 CERROR("Short message: got %d, wanted %d\n", nob, msg_nob);
141 /* checksum must be computed with ibm_cksum zero and BEFORE anything
143 msg_cksum = flip ? __swab32(msg->ibm_cksum) : msg->ibm_cksum;
145 if (msg_cksum != 0 &&
146 msg_cksum != kibnal_cksum(msg, msg_nob)) {
147 CERROR("Bad checksum\n");
150 msg->ibm_cksum = msg_cksum;
153 /* leave magic unflipped as a clue to peer endianness */
154 __swab16s(&msg->ibm_version);
155 LASSERT (sizeof(msg->ibm_type) == 1);
156 LASSERT (sizeof(msg->ibm_credits) == 1);
157 msg->ibm_nob = msg_nob;
158 __swab64s(&msg->ibm_srcnid);
159 __swab64s(&msg->ibm_srcstamp);
160 __swab64s(&msg->ibm_dstnid);
161 __swab64s(&msg->ibm_dststamp);
164 if (msg->ibm_srcnid == PTL_NID_ANY) {
165 CERROR("Bad src nid: "LPX64"\n", msg->ibm_srcnid);
169 switch (msg->ibm_type) {
171 CERROR("Unknown message type %x\n", msg->ibm_type);
174 case IBNAL_MSG_SVCQRY:
178 case IBNAL_MSG_SVCRSP:
179 if (msg_nob < hdr_size + sizeof(msg->ibm_u.svcrsp)) {
180 CERROR("Short SVCRSP: %d(%d)\n", msg_nob,
181 (int)(hdr_size + sizeof(msg->ibm_u.svcrsp)));
185 __swab64s(&msg->ibm_u.svcrsp.ibsr_svc_id);
186 __swab16s(&msg->ibm_u.svcrsp.ibsr_svc_pkey);
190 case IBNAL_MSG_CONNREQ:
191 case IBNAL_MSG_CONNACK:
192 if (msg_nob < hdr_size + sizeof(msg->ibm_u.connparams)) {
193 CERROR("Short CONNREQ: %d(%d)\n", msg_nob,
194 (int)(hdr_size + sizeof(msg->ibm_u.connparams)));
198 __swab32s(&msg->ibm_u.connparams.ibcp_queue_depth);
201 case IBNAL_MSG_IMMEDIATE:
202 if (msg_nob < offsetof(kib_msg_t, ibm_u.immediate.ibim_payload[0])) {
203 CERROR("Short IMMEDIATE: %d(%d)\n", msg_nob,
204 (int)offsetof(kib_msg_t, ibm_u.immediate.ibim_payload[0]));
209 case IBNAL_MSG_PUT_RDMA:
210 case IBNAL_MSG_GET_RDMA:
211 if (msg_nob < hdr_size + sizeof(msg->ibm_u.rdma)) {
212 CERROR("Short RDMA req: %d(%d)\n", msg_nob,
213 (int)(hdr_size + sizeof(msg->ibm_u.rdma)));
217 __swab32s(&msg->ibm_u.rdma.ibrm_desc.rd_key);
218 __swab32s(&msg->ibm_u.rdma.ibrm_desc.rd_nob);
219 __swab64s(&msg->ibm_u.rdma.ibrm_desc.rd_addr);
223 case IBNAL_MSG_PUT_DONE:
224 case IBNAL_MSG_GET_DONE:
225 if (msg_nob < hdr_size + sizeof(msg->ibm_u.completion)) {
226 CERROR("Short RDMA completion: %d(%d)\n", msg_nob,
227 (int)(hdr_size + sizeof(msg->ibm_u.completion)));
231 __swab32s(&msg->ibm_u.completion.ibcm_status);
238 kibnal_sock_write (struct socket *sock, void *buffer, int nob)
241 mm_segment_t oldmm = get_fs();
246 struct msghdr msg = {
253 .msg_flags = MSG_DONTWAIT
256 /* We've set up the socket's send buffer to be large enough for
257 * everything we send, so a single non-blocking send should
258 * complete without error. */
261 rc = sock_sendmsg(sock, &msg, iov.iov_len);
274 kibnal_sock_read (struct socket *sock, void *buffer, int nob, int timeout)
277 mm_segment_t oldmm = get_fs();
278 long ticks = timeout * HZ;
290 struct msghdr msg = {
300 /* Set receive timeout to remaining time */
301 tv = (struct timeval) {
302 .tv_sec = ticks / HZ,
303 .tv_usec = ((ticks % HZ) * 1000000) / HZ
306 rc = sock_setsockopt(sock, SOL_SOCKET, SO_RCVTIMEO,
307 (char *)&tv, sizeof(tv));
310 CERROR("Can't set socket recv timeout %d: %d\n",
317 rc = sock_recvmsg(sock, &msg, iov.iov_len, 0);
318 ticks -= jiffies - then;
325 return -ECONNABORTED;
327 buffer = ((char *)buffer) + rc;
339 kibnal_create_sock(struct socket **sockp)
344 mm_segment_t oldmm = get_fs();
346 rc = sock_create(PF_INET, SOCK_STREAM, 0, &sock);
348 CERROR("Can't create socket: %d\n", rc);
352 /* Ensure sends will not block */
353 option = 2 * sizeof(kib_msg_t);
355 rc = sock_setsockopt(sock, SOL_SOCKET, SO_SNDBUF,
356 (char *)&option, sizeof(option));
359 CERROR("Can't set send buffer %d: %d\n", option, rc);
365 rc = sock_setsockopt(sock, SOL_SOCKET, SO_REUSEADDR,
366 (char *)&option, sizeof(option));
369 CERROR("Can't set SO_REUSEADDR: %d\n", rc);
382 kibnal_pause(int ticks)
384 set_current_state(TASK_UNINTERRUPTIBLE);
385 schedule_timeout(ticks);
389 kibnal_connect_sock(kib_peer_t *peer, struct socket **sockp)
391 struct sockaddr_in locaddr;
392 struct sockaddr_in srvaddr;
397 for (port = 1023; port >= 512; port--) {
399 memset(&locaddr, 0, sizeof(locaddr));
400 locaddr.sin_family = AF_INET;
401 locaddr.sin_port = htons(port);
402 locaddr.sin_addr.s_addr = htonl(INADDR_ANY);
404 memset (&srvaddr, 0, sizeof (srvaddr));
405 srvaddr.sin_family = AF_INET;
406 srvaddr.sin_port = htons (peer->ibp_port);
407 srvaddr.sin_addr.s_addr = htonl (peer->ibp_ip);
409 rc = kibnal_create_sock(&sock);
413 rc = sock->ops->bind(sock,
414 (struct sockaddr *)&locaddr, sizeof(locaddr));
418 if (rc == -EADDRINUSE) {
419 CDEBUG(D_NET, "Port %d already in use\n", port);
423 CERROR("Can't bind to reserved port %d: %d\n", port, rc);
427 rc = sock->ops->connect(sock,
428 (struct sockaddr *)&srvaddr, sizeof(srvaddr),
437 if (rc != -EADDRNOTAVAIL) {
438 CERROR("Can't connect port %d to %u.%u.%u.%u/%d: %d\n",
439 port, HIPQUAD(peer->ibp_ip), peer->ibp_port, rc);
443 CDEBUG(D_NET, "Port %d not available for %u.%u.%u.%u/%d\n",
444 port, HIPQUAD(peer->ibp_ip), peer->ibp_port);
448 return -EHOSTUNREACH;
452 kibnal_make_svcqry (kib_conn_t *conn)
454 kib_peer_t *peer = conn->ibc_peer;
460 LASSERT (conn->ibc_connreq != NULL);
461 msg = &conn->ibc_connreq->cr_msg;
463 kibnal_init_msg(msg, IBNAL_MSG_SVCQRY, 0);
464 kibnal_pack_msg(msg, 0, peer->ibp_nid, 0);
466 rc = kibnal_connect_sock(peer, &sock);
470 rc = kibnal_sock_write(sock, msg, msg->ibm_nob);
472 CERROR("Error %d sending svcqry to "
473 LPX64"@%u.%u.%u.%u/%d\n", rc,
474 peer->ibp_nid, HIPQUAD(peer->ibp_ip), peer->ibp_port);
478 nob = offsetof(kib_msg_t, ibm_u) + sizeof(msg->ibm_u.svcrsp);
479 rc = kibnal_sock_read(sock, msg, nob, kibnal_tunables.kib_io_timeout);
481 CERROR("Error %d receiving svcrsp from "
482 LPX64"@%u.%u.%u.%u/%d\n", rc,
483 peer->ibp_nid, HIPQUAD(peer->ibp_ip), peer->ibp_port);
487 rc = kibnal_unpack_msg(msg, nob);
489 CERROR("Error %d unpacking svcrsp from "
490 LPX64"@%u.%u.%u.%u/%d\n", rc,
491 peer->ibp_nid, HIPQUAD(peer->ibp_ip), peer->ibp_port);
495 if (msg->ibm_type != IBNAL_MSG_SVCRSP) {
496 CERROR("Unexpected response type %d from "
497 LPX64"@%u.%u.%u.%u/%d\n", msg->ibm_type,
498 peer->ibp_nid, HIPQUAD(peer->ibp_ip), peer->ibp_port);
503 if (msg->ibm_dstnid != kibnal_lib.libnal_ni.ni_pid.nid ||
504 msg->ibm_dststamp != kibnal_data.kib_incarnation) {
505 CERROR("Unexpected dst NID/stamp "LPX64"/"LPX64" from "
506 LPX64"@%u.%u.%u.%u/%d\n",
507 msg->ibm_dstnid, msg->ibm_dststamp,
508 peer->ibp_nid, HIPQUAD(peer->ibp_ip), peer->ibp_port);
513 if (msg->ibm_srcnid != peer->ibp_nid) {
514 CERROR("Unexpected src NID "LPX64" from "
515 LPX64"@%u.%u.%u.%u/%d\n", msg->ibm_srcnid,
516 peer->ibp_nid, HIPQUAD(peer->ibp_ip), peer->ibp_port);
521 conn->ibc_incarnation = msg->ibm_srcstamp;
522 conn->ibc_connreq->cr_svcrsp = msg->ibm_u.svcrsp;
529 kibnal_handle_svcqry (struct socket *sock)
531 struct sockaddr_in addr;
533 unsigned int peer_port;
541 rc = sock->ops->getname(sock, (struct sockaddr *)&addr, &len, 2);
543 CERROR("Can't get peer's IP: %d\n", rc);
547 peer_ip = ntohl(addr.sin_addr.s_addr);
548 peer_port = ntohs(addr.sin_port);
550 if (peer_port >= 1024) {
551 CERROR("Refusing unprivileged connection from %u.%u.%u.%u/%d\n",
552 HIPQUAD(peer_ip), peer_port);
556 PORTAL_ALLOC(msg, sizeof(*msg));
558 CERROR("Can't allocate msgs for %u.%u.%u.%u/%d\n",
559 HIPQUAD(peer_ip), peer_port);
563 rc = kibnal_sock_read(sock, msg, offsetof(kib_msg_t, ibm_u),
564 kibnal_tunables.kib_listener_timeout);
566 CERROR("Error %d receiving svcqry from %u.%u.%u.%u/%d\n",
567 rc, HIPQUAD(peer_ip), peer_port);
571 rc = kibnal_unpack_msg(msg, offsetof(kib_msg_t, ibm_u));
573 CERROR("Error %d unpacking svcqry from %u.%u.%u.%u/%d\n",
574 rc, HIPQUAD(peer_ip), peer_port);
578 if (msg->ibm_type != IBNAL_MSG_SVCQRY) {
579 CERROR("Unexpected message %d from %u.%u.%u.%u/%d\n",
580 msg->ibm_type, HIPQUAD(peer_ip), peer_port);
584 if (msg->ibm_dstnid != kibnal_lib.libnal_ni.ni_pid.nid) {
585 CERROR("Unexpected dstnid "LPX64"(expected "LPX64" "
586 "from %u.%u.%u.%u/%d\n", msg->ibm_dstnid,
587 kibnal_lib.libnal_ni.ni_pid.nid,
588 HIPQUAD(peer_ip), peer_port);
592 srcnid = msg->ibm_srcnid;
593 srcstamp = msg->ibm_srcstamp;
595 kibnal_init_msg(msg, IBNAL_MSG_SVCRSP, sizeof(msg->ibm_u.svcrsp));
597 msg->ibm_u.svcrsp.ibsr_svc_id = kibnal_data.kib_svc_id;
598 memcpy(msg->ibm_u.svcrsp.ibsr_svc_gid, kibnal_data.kib_svc_gid,
599 sizeof(kibnal_data.kib_svc_gid));
600 msg->ibm_u.svcrsp.ibsr_svc_pkey = kibnal_data.kib_svc_pkey;
602 kibnal_pack_msg(msg, 0, srcnid, srcstamp);
604 rc = kibnal_sock_write (sock, msg, msg->ibm_nob);
606 CERROR("Error %d replying to svcqry from %u.%u.%u.%u/%d\n",
607 rc, HIPQUAD(peer_ip), peer_port);
612 PORTAL_FREE(msg, sizeof(*msg));
616 kibnal_free_acceptsock (kib_acceptsock_t *as)
618 sock_release(as->ibas_sock);
619 PORTAL_FREE(as, sizeof(*as));
623 kibnal_ip_listener(void *arg)
625 struct sockaddr_in addr;
628 kib_acceptsock_t *as;
634 /* Parent thread holds kib_nid_mutex, and is, or is about to
635 * block on kib_listener_signal */
637 port = kibnal_tunables.kib_port;
638 snprintf(name, sizeof(name), "kibnal_lstn%03d", port);
639 kportal_daemonize(name);
640 kportal_blockallsigs();
642 init_waitqueue_entry(&wait, current);
644 rc = kibnal_create_sock(&sock);
648 memset(&addr, 0, sizeof(addr));
649 addr.sin_family = AF_INET;
650 addr.sin_port = htons(port);
651 addr.sin_addr.s_addr = INADDR_ANY;
653 rc = sock->ops->bind(sock, (struct sockaddr *)&addr, sizeof(addr));
655 CERROR("Can't bind to port %d\n", port);
659 rc = sock->ops->listen(sock, kibnal_tunables.kib_backlog);
661 CERROR("Can't set listen backlog %d: %d\n",
662 kibnal_tunables.kib_backlog, rc);
666 LASSERT (kibnal_data.kib_listener_sock == NULL);
667 kibnal_data.kib_listener_sock = sock;
669 /* unblock waiting parent */
670 LASSERT (kibnal_data.kib_listener_shutdown == 0);
671 up(&kibnal_data.kib_listener_signal);
673 /* Wake me any time something happens on my socket */
674 add_wait_queue(sock->sk->sk_sleep, &wait);
677 while (kibnal_data.kib_listener_shutdown == 0) {
680 PORTAL_ALLOC(as, sizeof(*as));
682 CERROR("Out of Memory: pausing...\n");
686 as->ibas_sock = NULL;
689 if (as->ibas_sock == NULL) {
690 as->ibas_sock = sock_alloc();
691 if (as->ibas_sock == NULL) {
692 CERROR("Can't allocate socket: pausing...\n");
696 /* XXX this should add a ref to sock->ops->owner, if
697 * TCP could be a module */
698 as->ibas_sock->type = sock->type;
699 as->ibas_sock->ops = sock->ops;
702 set_current_state(TASK_INTERRUPTIBLE);
704 rc = sock->ops->accept(sock, as->ibas_sock, O_NONBLOCK);
706 /* Sleep for socket activity? */
708 kibnal_data.kib_listener_shutdown == 0)
711 set_current_state(TASK_RUNNING);
714 spin_lock_irqsave(&kibnal_data.kib_connd_lock, flags);
716 list_add_tail(&as->ibas_list,
717 &kibnal_data.kib_connd_acceptq);
719 spin_unlock_irqrestore(&kibnal_data.kib_connd_lock, flags);
720 wake_up(&kibnal_data.kib_connd_waitq);
727 CERROR("Accept failed: %d, pausing...\n", rc);
733 if (as->ibas_sock != NULL)
734 sock_release(as->ibas_sock);
735 PORTAL_FREE(as, sizeof(*as));
739 remove_wait_queue(sock->sk->sk_sleep, &wait);
742 kibnal_data.kib_listener_sock = NULL;
744 /* set completion status and unblock thread waiting for me
745 * (parent on startup failure, executioner on normal shutdown) */
746 kibnal_data.kib_listener_shutdown = rc;
747 up(&kibnal_data.kib_listener_signal);
753 kibnal_start_ip_listener (void)
758 CDEBUG(D_NET, "Starting listener\n");
760 /* Called holding kib_nid_mutex: listener stopped */
761 LASSERT (kibnal_data.kib_listener_sock == NULL);
763 kibnal_data.kib_listener_shutdown = 0;
764 pid = kernel_thread(kibnal_ip_listener, NULL, 0);
766 CERROR("Can't spawn listener: %ld\n", pid);
770 /* Block until listener has started up. */
771 down(&kibnal_data.kib_listener_signal);
773 rc = kibnal_data.kib_listener_shutdown;
774 LASSERT ((rc != 0) == (kibnal_data.kib_listener_sock == NULL));
776 CDEBUG((rc == 0) ? D_WARNING : D_ERROR,
777 "Listener %s: pid:%ld port:%d backlog:%d\n",
778 (rc == 0) ? "started OK" : "startup failed",
779 pid, kibnal_tunables.kib_port, kibnal_tunables.kib_backlog);
785 kibnal_stop_ip_listener(int clear_acceptq)
787 struct list_head zombie_accepts;
788 kib_acceptsock_t *as;
791 CDEBUG(D_NET, "Stopping listener\n");
793 /* Called holding kib_nid_mutex: listener running */
794 LASSERT (kibnal_data.kib_listener_sock != NULL);
796 kibnal_data.kib_listener_shutdown = 1;
797 wake_up_all(kibnal_data.kib_listener_sock->sk->sk_sleep);
799 /* Block until listener has torn down. */
800 down(&kibnal_data.kib_listener_signal);
802 LASSERT (kibnal_data.kib_listener_sock == NULL);
803 CDEBUG(D_WARNING, "Listener stopped\n");
808 /* Close any unhandled accepts */
809 spin_lock_irqsave(&kibnal_data.kib_connd_lock, flags);
811 list_add(&zombie_accepts, &kibnal_data.kib_connd_acceptq);
812 list_del_init(&kibnal_data.kib_connd_acceptq);
814 spin_unlock_irqrestore(&kibnal_data.kib_connd_lock, flags);
816 while (!list_empty(&zombie_accepts)) {
817 as = list_entry(zombie_accepts.next,
818 kib_acceptsock_t, ibas_list);
819 list_del(&as->ibas_list);
820 kibnal_free_acceptsock(as);
825 kibnal_listener_procint(ctl_table *table, int write, struct file *filp,
826 void *buffer, size_t *lenp)
828 int *tunable = (int *)table->data;
832 /* No race with nal initialisation since the nal is setup all the time
833 * it's loaded. When that changes, change this! */
834 LASSERT (kibnal_data.kib_init == IBNAL_INIT_ALL);
836 down(&kibnal_data.kib_nid_mutex);
838 LASSERT (tunable == &kibnal_tunables.kib_port ||
839 tunable == &kibnal_tunables.kib_backlog);
842 rc = proc_dointvec(table, write, filp, buffer, lenp);
845 (*tunable != old_val ||
846 kibnal_data.kib_listener_sock == NULL)) {
848 if (kibnal_data.kib_listener_sock != NULL)
849 kibnal_stop_ip_listener(0);
851 rc = kibnal_start_ip_listener();
853 CERROR("Unable to restart listener with new tunable:"
854 " reverting to old value\n");
856 kibnal_start_ip_listener();
860 up(&kibnal_data.kib_nid_mutex);
862 LASSERT (kibnal_data.kib_init == IBNAL_INIT_ALL);
867 kibnal_start_ib_listener (void)
871 LASSERT (kibnal_data.kib_listen_handle == NULL);
873 kibnal_data.kib_svc_id = ib_cm_service_assign();
874 CDEBUG(D_NET, "svc id "LPX64"\n", kibnal_data.kib_svc_id);
876 rc = ib_cached_gid_get(kibnal_data.kib_device,
877 kibnal_data.kib_port, 0,
878 kibnal_data.kib_svc_gid);
880 CERROR("Can't get port %d GID: %d\n",
881 kibnal_data.kib_port, rc);
885 rc = ib_cached_pkey_get(kibnal_data.kib_device,
886 kibnal_data.kib_port, 0,
887 &kibnal_data.kib_svc_pkey);
889 CERROR ("Can't get port %d PKEY: %d\n",
890 kibnal_data.kib_port, rc);
894 rc = ib_cm_listen(kibnal_data.kib_svc_id,
895 TS_IB_CM_SERVICE_EXACT_MASK,
896 kibnal_passive_conn_callback, NULL,
897 &kibnal_data.kib_listen_handle);
899 kibnal_data.kib_listen_handle = NULL;
900 CERROR ("Can't create IB listener: %d\n", rc);
904 LASSERT (kibnal_data.kib_listen_handle != NULL);
909 kibnal_stop_ib_listener (void)
913 LASSERT (kibnal_data.kib_listen_handle != NULL);
915 rc = ib_cm_listen_stop (kibnal_data.kib_listen_handle);
917 CERROR("Error stopping IB listener: %d\n", rc);
919 kibnal_data.kib_listen_handle = NULL;
923 kibnal_set_mynid (ptl_nid_t nid)
925 lib_ni_t *ni = &kibnal_lib.libnal_ni;
928 CDEBUG(D_IOCTL, "setting mynid to "LPX64" (old nid="LPX64")\n",
929 nid, ni->ni_pid.nid);
931 down (&kibnal_data.kib_nid_mutex);
933 if (nid == kibnal_data.kib_nid) {
934 /* no change of NID */
935 up (&kibnal_data.kib_nid_mutex);
939 CDEBUG(D_NET, "NID "LPX64"("LPX64")\n",
940 kibnal_data.kib_nid, nid);
942 if (kibnal_data.kib_listener_sock != NULL)
943 kibnal_stop_ip_listener(1);
945 if (kibnal_data.kib_listen_handle != NULL)
946 kibnal_stop_ib_listener();
948 ni->ni_pid.nid = nid;
949 kibnal_data.kib_incarnation++;
951 /* Delete all existing peers and their connections after new
952 * NID/incarnation set to ensure no old connections in our brave new
954 kibnal_del_peer (PTL_NID_ANY, 0);
956 if (ni->ni_pid.nid != PTL_NID_ANY) {
957 /* got a new NID to install */
958 rc = kibnal_start_ib_listener();
960 CERROR("Can't start IB listener: %d\n", rc);
964 rc = kibnal_start_ip_listener();
966 CERROR("Can't start IP listener: %d\n", rc);
971 up(&kibnal_data.kib_nid_mutex);
975 kibnal_stop_ib_listener();
977 ni->ni_pid.nid = PTL_NID_ANY;
978 kibnal_data.kib_incarnation++;
980 kibnal_del_peer (PTL_NID_ANY, 0);
981 up(&kibnal_data.kib_nid_mutex);
986 kibnal_create_peer (ptl_nid_t nid)
990 LASSERT (nid != PTL_NID_ANY);
992 PORTAL_ALLOC (peer, sizeof (*peer));
996 memset(peer, 0, sizeof(*peer)); /* zero flags etc */
999 atomic_set (&peer->ibp_refcount, 1); /* 1 ref for caller */
1001 INIT_LIST_HEAD (&peer->ibp_list); /* not in the peer table yet */
1002 INIT_LIST_HEAD (&peer->ibp_conns);
1003 INIT_LIST_HEAD (&peer->ibp_tx_queue);
1004 INIT_LIST_HEAD (&peer->ibp_connd_list); /* not queued for connecting */
1006 peer->ibp_reconnect_time = jiffies;
1007 peer->ibp_reconnect_interval = IBNAL_MIN_RECONNECT_INTERVAL;
1009 atomic_inc (&kibnal_data.kib_npeers);
1010 CDEBUG(D_NET, "peer %p "LPX64"\n", peer, nid);
1016 kibnal_destroy_peer (kib_peer_t *peer)
1018 CDEBUG (D_NET, "peer "LPX64" %p deleted\n", peer->ibp_nid, peer);
1020 LASSERT (atomic_read (&peer->ibp_refcount) == 0);
1021 LASSERT (peer->ibp_persistence == 0);
1022 LASSERT (!kibnal_peer_active(peer));
1023 LASSERT (peer->ibp_connecting == 0);
1024 LASSERT (list_empty (&peer->ibp_connd_list));
1025 LASSERT (list_empty (&peer->ibp_conns));
1026 LASSERT (list_empty (&peer->ibp_tx_queue));
1028 PORTAL_FREE (peer, sizeof (*peer));
1030 /* NB a peer's connections keep a reference on their peer until
1031 * they are destroyed, so we can be assured that _all_ state to do
1032 * with this peer has been cleaned up when its refcount drops to
1034 atomic_dec (&kibnal_data.kib_npeers);
1038 kibnal_put_peer (kib_peer_t *peer)
1040 CDEBUG (D_OTHER, "putting peer[%p] -> "LPX64" (%d)\n",
1041 peer, peer->ibp_nid,
1042 atomic_read (&peer->ibp_refcount));
1044 LASSERT (atomic_read (&peer->ibp_refcount) > 0);
1045 if (!atomic_dec_and_test (&peer->ibp_refcount))
1048 kibnal_destroy_peer (peer);
1052 kibnal_find_peer_locked (ptl_nid_t nid)
1054 struct list_head *peer_list = kibnal_nid2peerlist (nid);
1055 struct list_head *tmp;
1058 list_for_each (tmp, peer_list) {
1060 peer = list_entry (tmp, kib_peer_t, ibp_list);
1062 LASSERT (peer->ibp_persistence != 0 || /* persistent peer */
1063 peer->ibp_connecting != 0 || /* creating conns */
1064 !list_empty (&peer->ibp_conns)); /* active conn */
1066 if (peer->ibp_nid != nid)
1069 CDEBUG(D_NET, "got peer [%p] -> "LPX64" (%d)\n",
1070 peer, nid, atomic_read (&peer->ibp_refcount));
1077 kibnal_get_peer (ptl_nid_t nid)
1080 unsigned long flags;
1082 read_lock_irqsave(&kibnal_data.kib_global_lock, flags);
1083 peer = kibnal_find_peer_locked (nid);
1084 if (peer != NULL) /* +1 ref for caller? */
1085 atomic_inc (&peer->ibp_refcount);
1086 read_unlock_irqrestore(&kibnal_data.kib_global_lock, flags);
1092 kibnal_unlink_peer_locked (kib_peer_t *peer)
1094 LASSERT (peer->ibp_persistence == 0);
1095 LASSERT (list_empty(&peer->ibp_conns));
1097 LASSERT (kibnal_peer_active(peer));
1098 list_del_init (&peer->ibp_list);
1099 /* lose peerlist's ref */
1100 kibnal_put_peer (peer);
1104 kibnal_get_peer_info (int index, ptl_nid_t *nidp, __u32 *ipp, int *portp,
1108 struct list_head *ptmp;
1109 unsigned long flags;
1112 read_lock_irqsave(&kibnal_data.kib_global_lock, flags);
1114 for (i = 0; i < kibnal_data.kib_peer_hash_size; i++) {
1116 list_for_each (ptmp, &kibnal_data.kib_peers[i]) {
1118 peer = list_entry (ptmp, kib_peer_t, ibp_list);
1119 LASSERT (peer->ibp_persistence != 0 ||
1120 peer->ibp_connecting != 0 ||
1121 !list_empty (&peer->ibp_conns));
1126 *nidp = peer->ibp_nid;
1127 *ipp = peer->ibp_ip;
1128 *portp = peer->ibp_port;
1129 *persistencep = peer->ibp_persistence;
1131 read_unlock_irqrestore(&kibnal_data.kib_global_lock,
1137 read_unlock_irqrestore(&kibnal_data.kib_global_lock, flags);
1142 kibnal_add_persistent_peer (ptl_nid_t nid, __u32 ip, int port)
1144 unsigned long flags;
1148 if (nid == PTL_NID_ANY)
1151 peer = kibnal_create_peer (nid);
1155 write_lock_irqsave (&kibnal_data.kib_global_lock, flags);
1157 peer2 = kibnal_find_peer_locked (nid);
1158 if (peer2 != NULL) {
1159 kibnal_put_peer (peer);
1162 /* peer table takes existing ref on peer */
1163 list_add_tail (&peer->ibp_list,
1164 kibnal_nid2peerlist (nid));
1168 peer->ibp_port = port;
1169 peer->ibp_persistence++;
1171 write_unlock_irqrestore (&kibnal_data.kib_global_lock, flags);
1176 kibnal_del_peer_locked (kib_peer_t *peer, int single_share)
1178 struct list_head *ctmp;
1179 struct list_head *cnxt;
1183 peer->ibp_persistence = 0;
1184 else if (peer->ibp_persistence > 0)
1185 peer->ibp_persistence--;
1187 if (peer->ibp_persistence != 0)
1190 if (list_empty(&peer->ibp_conns)) {
1191 kibnal_unlink_peer_locked(peer);
1193 list_for_each_safe (ctmp, cnxt, &peer->ibp_conns) {
1194 conn = list_entry(ctmp, kib_conn_t, ibc_list);
1196 kibnal_close_conn_locked (conn, 0);
1198 /* NB peer is no longer persistent; closing its last conn
1201 /* NB peer now unlinked; might even be freed if the peer table had the
1202 * last ref on it. */
1206 kibnal_del_peer (ptl_nid_t nid, int single_share)
1208 unsigned long flags;
1209 struct list_head *ptmp;
1210 struct list_head *pnxt;
1217 write_lock_irqsave (&kibnal_data.kib_global_lock, flags);
1219 if (nid != PTL_NID_ANY)
1220 lo = hi = kibnal_nid2peerlist(nid) - kibnal_data.kib_peers;
1223 hi = kibnal_data.kib_peer_hash_size - 1;
1226 for (i = lo; i <= hi; i++) {
1227 list_for_each_safe (ptmp, pnxt, &kibnal_data.kib_peers[i]) {
1228 peer = list_entry (ptmp, kib_peer_t, ibp_list);
1229 LASSERT (peer->ibp_persistence != 0 ||
1230 peer->ibp_connecting != 0 ||
1231 !list_empty (&peer->ibp_conns));
1233 if (!(nid == PTL_NID_ANY || peer->ibp_nid == nid))
1236 kibnal_del_peer_locked (peer, single_share);
1237 rc = 0; /* matched something */
1244 write_unlock_irqrestore (&kibnal_data.kib_global_lock, flags);
1250 kibnal_get_conn_by_idx (int index)
1253 struct list_head *ptmp;
1255 struct list_head *ctmp;
1256 unsigned long flags;
1259 read_lock_irqsave(&kibnal_data.kib_global_lock, flags);
1261 for (i = 0; i < kibnal_data.kib_peer_hash_size; i++) {
1262 list_for_each (ptmp, &kibnal_data.kib_peers[i]) {
1264 peer = list_entry (ptmp, kib_peer_t, ibp_list);
1265 LASSERT (peer->ibp_persistence > 0 ||
1266 peer->ibp_connecting != 0 ||
1267 !list_empty (&peer->ibp_conns));
1269 list_for_each (ctmp, &peer->ibp_conns) {
1273 conn = list_entry (ctmp, kib_conn_t, ibc_list);
1274 CDEBUG(D_NET, "++conn[%p] state %d -> "LPX64" (%d)\n",
1275 conn, conn->ibc_state, conn->ibc_peer->ibp_nid,
1276 atomic_read (&conn->ibc_refcount));
1277 atomic_inc (&conn->ibc_refcount);
1278 read_unlock_irqrestore(&kibnal_data.kib_global_lock,
1285 read_unlock_irqrestore(&kibnal_data.kib_global_lock, flags);
1290 kibnal_create_conn (void)
1300 struct ib_qp_create_param qp_create;
1301 struct ib_qp_attribute qp_attr;
1304 PORTAL_ALLOC (conn, sizeof (*conn));
1306 CERROR ("Can't allocate connection\n");
1310 /* zero flags, NULL pointers etc... */
1311 memset (conn, 0, sizeof (*conn));
1313 INIT_LIST_HEAD (&conn->ibc_tx_queue);
1314 INIT_LIST_HEAD (&conn->ibc_active_txs);
1315 spin_lock_init (&conn->ibc_lock);
1317 atomic_inc (&kibnal_data.kib_nconns);
1318 /* well not really, but I call destroy() on failure, which decrements */
1320 PORTAL_ALLOC (conn->ibc_rxs, IBNAL_RX_MSGS * sizeof (kib_rx_t));
1321 if (conn->ibc_rxs == NULL)
1323 memset (conn->ibc_rxs, 0, IBNAL_RX_MSGS * sizeof(kib_rx_t));
1325 rc = kibnal_alloc_pages(&conn->ibc_rx_pages,
1327 IB_ACCESS_LOCAL_WRITE);
1331 vaddr_base = vaddr = conn->ibc_rx_pages->ibp_vaddr;
1333 for (i = ipage = page_offset = 0; i < IBNAL_RX_MSGS; i++) {
1334 struct page *page = conn->ibc_rx_pages->ibp_pages[ipage];
1335 kib_rx_t *rx = &conn->ibc_rxs[i];
1338 rx->rx_vaddr = vaddr;
1339 rx->rx_msg = (kib_msg_t *)(((char *)page_address(page)) + page_offset);
1341 vaddr += IBNAL_MSG_SIZE;
1342 LASSERT (vaddr <= vaddr_base + IBNAL_RX_MSG_BYTES);
1344 page_offset += IBNAL_MSG_SIZE;
1345 LASSERT (page_offset <= PAGE_SIZE);
1347 if (page_offset == PAGE_SIZE) {
1350 LASSERT (ipage <= IBNAL_RX_MSG_PAGES);
1354 params.qp_create = (struct ib_qp_create_param) {
1356 /* Sends have an optional RDMA */
1357 .max_outstanding_send_request = 2 * IBNAL_MSG_QUEUE_SIZE,
1358 .max_outstanding_receive_request = IBNAL_MSG_QUEUE_SIZE,
1359 .max_send_gather_element = 1,
1360 .max_receive_scatter_element = 1,
1362 .pd = kibnal_data.kib_pd,
1363 .send_queue = kibnal_data.kib_cq,
1364 .receive_queue = kibnal_data.kib_cq,
1365 .send_policy = IB_WQ_SIGNAL_SELECTABLE,
1366 .receive_policy = IB_WQ_SIGNAL_SELECTABLE,
1368 .transport = IB_TRANSPORT_RC,
1369 .device_specific = NULL,
1372 rc = ib_qp_create (¶ms.qp_create, &conn->ibc_qp, &conn->ibc_qpn);
1374 CERROR ("Failed to create queue pair: %d\n", rc);
1378 /* Mark QP created */
1379 conn->ibc_state = IBNAL_CONN_INIT_QP;
1381 params.qp_attr = (struct ib_qp_attribute) {
1382 .state = IB_QP_STATE_INIT,
1383 .port = kibnal_data.kib_port,
1384 .enable_rdma_read = 1,
1385 .enable_rdma_write = 1,
1386 .valid_fields = (IB_QP_ATTRIBUTE_STATE |
1387 IB_QP_ATTRIBUTE_PORT |
1388 IB_QP_ATTRIBUTE_PKEY_INDEX |
1389 IB_QP_ATTRIBUTE_RDMA_ATOMIC_ENABLE),
1391 rc = ib_qp_modify(conn->ibc_qp, ¶ms.qp_attr);
1393 CERROR ("Failed to modify queue pair: %d\n", rc);
1397 /* 1 ref for caller */
1398 atomic_set (&conn->ibc_refcount, 1);
1402 kibnal_destroy_conn (conn);
1407 kibnal_destroy_conn (kib_conn_t *conn)
1411 CDEBUG (D_NET, "connection %p\n", conn);
1413 LASSERT (atomic_read (&conn->ibc_refcount) == 0);
1414 LASSERT (list_empty(&conn->ibc_tx_queue));
1415 LASSERT (list_empty(&conn->ibc_active_txs));
1416 LASSERT (conn->ibc_nsends_posted == 0);
1417 LASSERT (conn->ibc_connreq == NULL);
1419 switch (conn->ibc_state) {
1420 case IBNAL_CONN_ZOMBIE:
1421 /* called after connection sequence initiated */
1423 case IBNAL_CONN_INIT_QP:
1424 rc = ib_qp_destroy(conn->ibc_qp);
1426 CERROR("Can't destroy QP: %d\n", rc);
1429 case IBNAL_CONN_INIT_NOTHING:
1436 if (conn->ibc_rx_pages != NULL)
1437 kibnal_free_pages(conn->ibc_rx_pages);
1439 if (conn->ibc_rxs != NULL)
1440 PORTAL_FREE(conn->ibc_rxs,
1441 IBNAL_RX_MSGS * sizeof(kib_rx_t));
1443 if (conn->ibc_peer != NULL)
1444 kibnal_put_peer(conn->ibc_peer);
1446 PORTAL_FREE(conn, sizeof (*conn));
1448 atomic_dec(&kibnal_data.kib_nconns);
1450 if (atomic_read (&kibnal_data.kib_nconns) == 0 &&
1451 kibnal_data.kib_shutdown) {
1452 /* I just nuked the last connection on shutdown; wake up
1453 * everyone so they can exit. */
1454 wake_up_all(&kibnal_data.kib_sched_waitq);
1455 wake_up_all(&kibnal_data.kib_reaper_waitq);
1460 kibnal_put_conn (kib_conn_t *conn)
1462 unsigned long flags;
1464 CDEBUG (D_NET, "putting conn[%p] state %d -> "LPX64" (%d)\n",
1465 conn, conn->ibc_state, conn->ibc_peer->ibp_nid,
1466 atomic_read (&conn->ibc_refcount));
1468 LASSERT (atomic_read (&conn->ibc_refcount) > 0);
1469 if (!atomic_dec_and_test (&conn->ibc_refcount))
1472 /* last ref only goes on zombies */
1473 LASSERT (conn->ibc_state == IBNAL_CONN_ZOMBIE);
1475 spin_lock_irqsave (&kibnal_data.kib_reaper_lock, flags);
1477 list_add (&conn->ibc_list, &kibnal_data.kib_reaper_conns);
1478 wake_up (&kibnal_data.kib_reaper_waitq);
1480 spin_unlock_irqrestore (&kibnal_data.kib_reaper_lock, flags);
1484 kibnal_close_peer_conns_locked (kib_peer_t *peer, int why)
1487 struct list_head *ctmp;
1488 struct list_head *cnxt;
1491 list_for_each_safe (ctmp, cnxt, &peer->ibp_conns) {
1492 conn = list_entry (ctmp, kib_conn_t, ibc_list);
1495 kibnal_close_conn_locked (conn, why);
1502 kibnal_close_stale_conns_locked (kib_peer_t *peer, __u64 incarnation)
1505 struct list_head *ctmp;
1506 struct list_head *cnxt;
1509 list_for_each_safe (ctmp, cnxt, &peer->ibp_conns) {
1510 conn = list_entry (ctmp, kib_conn_t, ibc_list);
1512 if (conn->ibc_incarnation == incarnation)
1515 CDEBUG(D_NET, "Closing stale conn %p nid:"LPX64
1516 " incarnation:"LPX64"("LPX64")\n", conn,
1517 peer->ibp_nid, conn->ibc_incarnation, incarnation);
1520 kibnal_close_conn_locked (conn, -ESTALE);
1527 kibnal_close_matching_conns (ptl_nid_t nid)
1529 unsigned long flags;
1531 struct list_head *ptmp;
1532 struct list_head *pnxt;
1538 write_lock_irqsave (&kibnal_data.kib_global_lock, flags);
1540 if (nid != PTL_NID_ANY)
1541 lo = hi = kibnal_nid2peerlist(nid) - kibnal_data.kib_peers;
1544 hi = kibnal_data.kib_peer_hash_size - 1;
1547 for (i = lo; i <= hi; i++) {
1548 list_for_each_safe (ptmp, pnxt, &kibnal_data.kib_peers[i]) {
1550 peer = list_entry (ptmp, kib_peer_t, ibp_list);
1551 LASSERT (peer->ibp_persistence != 0 ||
1552 peer->ibp_connecting != 0 ||
1553 !list_empty (&peer->ibp_conns));
1555 if (!(nid == PTL_NID_ANY || nid == peer->ibp_nid))
1558 count += kibnal_close_peer_conns_locked (peer, 0);
1562 write_unlock_irqrestore (&kibnal_data.kib_global_lock, flags);
1564 /* wildcards always succeed */
1565 if (nid == PTL_NID_ANY)
1568 return (count == 0 ? -ENOENT : 0);
1572 kibnal_cmd(struct portals_cfg *pcfg, void * private)
1576 LASSERT (pcfg != NULL);
1578 switch(pcfg->pcfg_command) {
1579 case NAL_CMD_GET_PEER: {
1583 int share_count = 0;
1585 rc = kibnal_get_peer_info(pcfg->pcfg_count,
1586 &nid, &ip, &port, &share_count);
1587 pcfg->pcfg_nid = nid;
1588 pcfg->pcfg_size = 0;
1590 pcfg->pcfg_misc = port;
1591 pcfg->pcfg_count = 0;
1592 pcfg->pcfg_wait = share_count;
1595 case NAL_CMD_ADD_PEER: {
1596 rc = kibnal_add_persistent_peer (pcfg->pcfg_nid,
1597 pcfg->pcfg_id, /* IP */
1598 pcfg->pcfg_misc); /* port */
1601 case NAL_CMD_DEL_PEER: {
1602 rc = kibnal_del_peer (pcfg->pcfg_nid,
1603 /* flags == single_share */
1604 pcfg->pcfg_flags != 0);
1607 case NAL_CMD_GET_CONN: {
1608 kib_conn_t *conn = kibnal_get_conn_by_idx (pcfg->pcfg_count);
1614 pcfg->pcfg_nid = conn->ibc_peer->ibp_nid;
1616 pcfg->pcfg_misc = 0;
1617 pcfg->pcfg_flags = 0;
1618 kibnal_put_conn (conn);
1622 case NAL_CMD_CLOSE_CONNECTION: {
1623 rc = kibnal_close_matching_conns (pcfg->pcfg_nid);
1626 case NAL_CMD_REGISTER_MYNID: {
1627 if (pcfg->pcfg_nid == PTL_NID_ANY)
1630 rc = kibnal_set_mynid (pcfg->pcfg_nid);
1639 kibnal_free_pages (kib_pages_t *p)
1641 int npages = p->ibp_npages;
1645 if (p->ibp_mapped) {
1646 rc = ib_memory_deregister(p->ibp_handle);
1648 CERROR ("Deregister error: %d\n", rc);
1651 for (i = 0; i < npages; i++)
1652 if (p->ibp_pages[i] != NULL)
1653 __free_page(p->ibp_pages[i]);
1655 PORTAL_FREE (p, offsetof(kib_pages_t, ibp_pages[npages]));
1659 kibnal_alloc_pages (kib_pages_t **pp, int npages, int access)
1662 struct ib_physical_buffer *phys_pages;
1666 PORTAL_ALLOC(p, offsetof(kib_pages_t, ibp_pages[npages]));
1668 CERROR ("Can't allocate buffer %d\n", npages);
1672 memset (p, 0, offsetof(kib_pages_t, ibp_pages[npages]));
1673 p->ibp_npages = npages;
1675 for (i = 0; i < npages; i++) {
1676 p->ibp_pages[i] = alloc_page (GFP_KERNEL);
1677 if (p->ibp_pages[i] == NULL) {
1678 CERROR ("Can't allocate page %d of %d\n", i, npages);
1679 kibnal_free_pages(p);
1684 PORTAL_ALLOC(phys_pages, npages * sizeof(*phys_pages));
1685 if (phys_pages == NULL) {
1686 CERROR ("Can't allocate physarray for %d pages\n", npages);
1687 kibnal_free_pages(p);
1691 for (i = 0; i < npages; i++) {
1692 phys_pages[i].size = PAGE_SIZE;
1693 phys_pages[i].address =
1694 kibnal_page2phys(p->ibp_pages[i]);
1698 rc = ib_memory_register_physical(kibnal_data.kib_pd,
1701 npages * PAGE_SIZE, 0,
1707 PORTAL_FREE(phys_pages, npages * sizeof(*phys_pages));
1710 CERROR ("Error %d mapping %d pages\n", rc, npages);
1711 kibnal_free_pages(p);
1721 kibnal_setup_tx_descs (void)
1724 int page_offset = 0;
1732 /* pre-mapped messages are not bigger than 1 page */
1733 LASSERT (IBNAL_MSG_SIZE <= PAGE_SIZE);
1735 /* No fancy arithmetic when we do the buffer calculations */
1736 LASSERT (PAGE_SIZE % IBNAL_MSG_SIZE == 0);
1738 rc = kibnal_alloc_pages(&kibnal_data.kib_tx_pages,
1740 0); /* local read access only */
1744 vaddr = vaddr_base = kibnal_data.kib_tx_pages->ibp_vaddr;
1746 for (i = 0; i < IBNAL_TX_MSGS; i++) {
1747 page = kibnal_data.kib_tx_pages->ibp_pages[ipage];
1748 tx = &kibnal_data.kib_tx_descs[i];
1750 memset (tx, 0, sizeof(*tx)); /* zero flags etc */
1752 tx->tx_msg = (kib_msg_t *)(((char *)page_address(page)) + page_offset);
1753 tx->tx_vaddr = vaddr;
1754 tx->tx_isnblk = (i >= IBNAL_NTX);
1755 tx->tx_mapped = KIB_TX_UNMAPPED;
1757 CDEBUG(D_NET, "Tx[%d] %p->%p - "LPX64"\n",
1758 i, tx, tx->tx_msg, tx->tx_vaddr);
1761 list_add (&tx->tx_list,
1762 &kibnal_data.kib_idle_nblk_txs);
1764 list_add (&tx->tx_list,
1765 &kibnal_data.kib_idle_txs);
1767 vaddr += IBNAL_MSG_SIZE;
1768 LASSERT (vaddr <= vaddr_base + IBNAL_TX_MSG_BYTES);
1770 page_offset += IBNAL_MSG_SIZE;
1771 LASSERT (page_offset <= PAGE_SIZE);
1773 if (page_offset == PAGE_SIZE) {
1776 LASSERT (ipage <= IBNAL_TX_MSG_PAGES);
1784 kibnal_api_shutdown (nal_t *nal)
1789 if (nal->nal_refct != 0) {
1790 /* This module got the first ref */
1791 PORTAL_MODULE_UNUSE;
1795 CDEBUG(D_MALLOC, "before NAL cleanup: kmem %d\n",
1796 atomic_read (&portal_kmemory));
1798 LASSERT(nal == &kibnal_api);
1800 switch (kibnal_data.kib_init) {
1802 CERROR ("Unexpected state %d\n", kibnal_data.kib_init);
1805 case IBNAL_INIT_ALL:
1806 /* stop calls to nal_cmd */
1807 libcfs_nal_cmd_unregister(OPENIBNAL);
1810 /* resetting my NID unadvertises me, removes my
1811 * listener and nukes all current peers */
1812 kibnal_set_mynid (PTL_NID_ANY);
1814 /* Wait for all peer state to clean up */
1816 while (atomic_read (&kibnal_data.kib_npeers) != 0) {
1818 CDEBUG(((i & (-i)) == i) ? D_WARNING : D_NET, /* power of 2? */
1819 "waiting for %d peers to close down\n",
1820 atomic_read (&kibnal_data.kib_npeers));
1821 set_current_state (TASK_INTERRUPTIBLE);
1822 schedule_timeout (HZ);
1827 rc = ib_cq_destroy (kibnal_data.kib_cq);
1829 CERROR ("Destroy CQ error: %d\n", rc);
1832 case IBNAL_INIT_TXD:
1833 kibnal_free_pages (kibnal_data.kib_tx_pages);
1836 case IBNAL_INIT_FMR:
1837 rc = ib_fmr_pool_destroy (kibnal_data.kib_fmr_pool);
1839 CERROR ("Destroy FMR pool error: %d\n", rc);
1843 rc = ib_pd_destroy(kibnal_data.kib_pd);
1845 CERROR ("Destroy PD error: %d\n", rc);
1848 case IBNAL_INIT_LIB:
1849 lib_fini(&kibnal_lib);
1852 case IBNAL_INIT_DATA:
1853 /* Module refcount only gets to zero when all peers
1854 * have been closed so all lists must be empty */
1855 LASSERT (atomic_read (&kibnal_data.kib_npeers) == 0);
1856 LASSERT (kibnal_data.kib_peers != NULL);
1857 for (i = 0; i < kibnal_data.kib_peer_hash_size; i++) {
1858 LASSERT (list_empty (&kibnal_data.kib_peers[i]));
1860 LASSERT (atomic_read (&kibnal_data.kib_nconns) == 0);
1861 LASSERT (list_empty (&kibnal_data.kib_sched_rxq));
1862 LASSERT (list_empty (&kibnal_data.kib_sched_txq));
1863 LASSERT (list_empty (&kibnal_data.kib_reaper_conns));
1864 LASSERT (list_empty (&kibnal_data.kib_connd_peers));
1865 LASSERT (list_empty (&kibnal_data.kib_connd_acceptq));
1867 /* flag threads to terminate; wake and wait for them to die */
1868 kibnal_data.kib_shutdown = 1;
1869 wake_up_all (&kibnal_data.kib_sched_waitq);
1870 wake_up_all (&kibnal_data.kib_reaper_waitq);
1871 wake_up_all (&kibnal_data.kib_connd_waitq);
1874 while (atomic_read (&kibnal_data.kib_nthreads) != 0) {
1876 CDEBUG(((i & (-i)) == i) ? D_WARNING : D_NET, /* power of 2? */
1877 "Waiting for %d threads to terminate\n",
1878 atomic_read (&kibnal_data.kib_nthreads));
1879 set_current_state (TASK_INTERRUPTIBLE);
1880 schedule_timeout (HZ);
1884 case IBNAL_INIT_NOTHING:
1888 if (kibnal_data.kib_tx_descs != NULL)
1889 PORTAL_FREE (kibnal_data.kib_tx_descs,
1890 IBNAL_TX_MSGS * sizeof(kib_tx_t));
1892 if (kibnal_data.kib_peers != NULL)
1893 PORTAL_FREE (kibnal_data.kib_peers,
1894 sizeof (struct list_head) *
1895 kibnal_data.kib_peer_hash_size);
1897 CDEBUG(D_MALLOC, "after NAL cleanup: kmem %d\n",
1898 atomic_read (&portal_kmemory));
1899 printk(KERN_INFO "Lustre: OpenIB NAL unloaded (final mem %d)\n",
1900 atomic_read(&portal_kmemory));
1902 kibnal_data.kib_init = IBNAL_INIT_NOTHING;
1906 kibnal_api_startup (nal_t *nal, ptl_pid_t requested_pid,
1907 ptl_ni_limits_t *requested_limits,
1908 ptl_ni_limits_t *actual_limits)
1911 ptl_process_id_t process_id;
1912 int pkmem = atomic_read(&portal_kmemory);
1916 LASSERT (nal == &kibnal_api);
1918 if (nal->nal_refct != 0) {
1919 if (actual_limits != NULL)
1920 *actual_limits = kibnal_lib.libnal_ni.ni_actual_limits;
1921 /* This module got the first ref */
1926 LASSERT (kibnal_data.kib_init == IBNAL_INIT_NOTHING);
1928 memset (&kibnal_data, 0, sizeof (kibnal_data)); /* zero pointers, flags etc */
1930 do_gettimeofday(&tv);
1931 kibnal_data.kib_incarnation = (((__u64)tv.tv_sec) * 1000000) + tv.tv_usec;
1933 init_MUTEX (&kibnal_data.kib_nid_mutex);
1934 init_MUTEX_LOCKED (&kibnal_data.kib_listener_signal);
1936 rwlock_init(&kibnal_data.kib_global_lock);
1938 kibnal_data.kib_peer_hash_size = IBNAL_PEER_HASH_SIZE;
1939 PORTAL_ALLOC (kibnal_data.kib_peers,
1940 sizeof (struct list_head) * kibnal_data.kib_peer_hash_size);
1941 if (kibnal_data.kib_peers == NULL) {
1944 for (i = 0; i < kibnal_data.kib_peer_hash_size; i++)
1945 INIT_LIST_HEAD(&kibnal_data.kib_peers[i]);
1947 spin_lock_init (&kibnal_data.kib_reaper_lock);
1948 INIT_LIST_HEAD (&kibnal_data.kib_reaper_conns);
1949 init_waitqueue_head (&kibnal_data.kib_reaper_waitq);
1951 spin_lock_init (&kibnal_data.kib_connd_lock);
1952 INIT_LIST_HEAD (&kibnal_data.kib_connd_acceptq);
1953 INIT_LIST_HEAD (&kibnal_data.kib_connd_peers);
1954 init_waitqueue_head (&kibnal_data.kib_connd_waitq);
1956 spin_lock_init (&kibnal_data.kib_sched_lock);
1957 INIT_LIST_HEAD (&kibnal_data.kib_sched_txq);
1958 INIT_LIST_HEAD (&kibnal_data.kib_sched_rxq);
1959 init_waitqueue_head (&kibnal_data.kib_sched_waitq);
1961 spin_lock_init (&kibnal_data.kib_tx_lock);
1962 INIT_LIST_HEAD (&kibnal_data.kib_idle_txs);
1963 INIT_LIST_HEAD (&kibnal_data.kib_idle_nblk_txs);
1964 init_waitqueue_head(&kibnal_data.kib_idle_tx_waitq);
1966 PORTAL_ALLOC (kibnal_data.kib_tx_descs,
1967 IBNAL_TX_MSGS * sizeof(kib_tx_t));
1968 if (kibnal_data.kib_tx_descs == NULL) {
1969 CERROR ("Can't allocate tx descs\n");
1973 /* lists/ptrs/locks initialised */
1974 kibnal_data.kib_init = IBNAL_INIT_DATA;
1975 /*****************************************************/
1978 process_id.pid = requested_pid;
1979 process_id.nid = PTL_NID_ANY; /* don't know my NID yet */
1981 rc = lib_init(&kibnal_lib, nal, process_id,
1982 requested_limits, actual_limits);
1984 CERROR("lib_init failed: error %d\n", rc);
1988 /* lib interface initialised */
1989 kibnal_data.kib_init = IBNAL_INIT_LIB;
1990 /*****************************************************/
1992 for (i = 0; i < IBNAL_N_SCHED; i++) {
1993 rc = kibnal_thread_start (kibnal_scheduler,
1994 (void *)((unsigned long)i));
1996 CERROR("Can't spawn openibnal scheduler[%d]: %d\n",
2002 for (i = 0; i < IBNAL_N_CONND; i++) {
2003 rc = kibnal_thread_start (kibnal_connd,
2004 (void *)((unsigned long)i));
2006 CERROR("Can't spawn openibnal connd[%d]: %d\n",
2012 rc = kibnal_thread_start (kibnal_reaper, NULL);
2014 CERROR ("Can't spawn openibnal reaper: %d\n", rc);
2018 kibnal_data.kib_device = ib_device_get_by_index(0);
2019 if (kibnal_data.kib_device == NULL) {
2020 CERROR ("Can't open ib device 0\n");
2024 rc = ib_device_properties_get(kibnal_data.kib_device,
2025 &kibnal_data.kib_device_props);
2027 CERROR ("Can't get device props: %d\n", rc);
2031 CDEBUG(D_NET, "Max Initiator: %d Max Responder %d\n",
2032 kibnal_data.kib_device_props.max_initiator_per_qp,
2033 kibnal_data.kib_device_props.max_responder_per_qp);
2035 kibnal_data.kib_port = 0;
2036 for (i = 1; i <= 2; i++) {
2037 rc = ib_port_properties_get(kibnal_data.kib_device, i,
2038 &kibnal_data.kib_port_props);
2040 kibnal_data.kib_port = i;
2044 if (kibnal_data.kib_port == 0) {
2045 CERROR ("Can't find a port\n");
2049 rc = ib_pd_create(kibnal_data.kib_device,
2050 NULL, &kibnal_data.kib_pd);
2052 CERROR ("Can't create PD: %d\n", rc);
2056 /* flag PD initialised */
2057 kibnal_data.kib_init = IBNAL_INIT_PD;
2058 /*****************************************************/
2061 const int pool_size = IBNAL_NTX + IBNAL_NTX_NBLK;
2062 struct ib_fmr_pool_param params = {
2063 .max_pages_per_fmr = PTL_MTU/PAGE_SIZE,
2064 .access = (IB_ACCESS_LOCAL_WRITE |
2065 IB_ACCESS_REMOTE_WRITE |
2066 IB_ACCESS_REMOTE_READ),
2067 .pool_size = pool_size,
2068 .dirty_watermark = (pool_size * 3)/4,
2069 .flush_function = NULL,
2073 rc = ib_fmr_pool_create(kibnal_data.kib_pd, ¶ms,
2074 &kibnal_data.kib_fmr_pool);
2076 CERROR ("Can't create FMR pool size %d: %d\n",
2082 /* flag FMR pool initialised */
2083 kibnal_data.kib_init = IBNAL_INIT_FMR;
2085 /*****************************************************/
2087 rc = kibnal_setup_tx_descs();
2089 CERROR ("Can't register tx descs: %d\n", rc);
2093 /* flag TX descs initialised */
2094 kibnal_data.kib_init = IBNAL_INIT_TXD;
2095 /*****************************************************/
2098 struct ib_cq_callback callback = {
2099 .context = IBNAL_CALLBACK_CTXT,
2100 .policy = IB_CQ_PROVIDER_REARM,
2102 .entry = kibnal_callback,
2106 int nentries = IBNAL_CQ_ENTRIES;
2108 rc = ib_cq_create (kibnal_data.kib_device,
2109 &nentries, &callback, NULL,
2110 &kibnal_data.kib_cq);
2112 CERROR ("Can't create CQ: %d\n", rc);
2116 /* I only want solicited events */
2117 rc = ib_cq_request_notification(kibnal_data.kib_cq, 1);
2121 /* flag CQ initialised */
2122 kibnal_data.kib_init = IBNAL_INIT_CQ;
2123 /*****************************************************/
2125 rc = libcfs_nal_cmd_register(OPENIBNAL, &kibnal_cmd, NULL);
2127 CERROR ("Can't initialise command interface (rc = %d)\n", rc);
2131 /* flag everything initialised */
2132 kibnal_data.kib_init = IBNAL_INIT_ALL;
2133 /*****************************************************/
2135 printk(KERN_INFO "Lustre: OpenIB NAL loaded "
2136 "(initial mem %d)\n", pkmem);
2141 kibnal_api_shutdown (&kibnal_api);
2146 kibnal_module_fini (void)
2148 if (kibnal_tunables.kib_sysctl != NULL)
2149 unregister_sysctl_table (kibnal_tunables.kib_sysctl);
2150 PtlNIFini(kibnal_ni);
2152 ptl_unregister_nal(OPENIBNAL);
2156 kibnal_module_init (void)
2160 /* the following must be sizeof(int) for proc_dointvec() */
2161 LASSERT (sizeof(kibnal_tunables.kib_io_timeout) == sizeof(int));
2162 LASSERT (sizeof(kibnal_tunables.kib_listener_timeout) == sizeof(int));
2163 LASSERT (sizeof(kibnal_tunables.kib_backlog) == sizeof(int));
2164 LASSERT (sizeof(kibnal_tunables.kib_port) == sizeof(int));
2166 kibnal_api.nal_ni_init = kibnal_api_startup;
2167 kibnal_api.nal_ni_fini = kibnal_api_shutdown;
2169 /* Initialise dynamic tunables to defaults once only */
2170 kibnal_tunables.kib_io_timeout = IBNAL_IO_TIMEOUT;
2171 kibnal_tunables.kib_listener_timeout = IBNAL_LISTENER_TIMEOUT;
2172 kibnal_tunables.kib_backlog = IBNAL_BACKLOG;
2173 kibnal_tunables.kib_port = IBNAL_PORT;
2175 rc = ptl_register_nal(OPENIBNAL, &kibnal_api);
2177 CERROR("Can't register IBNAL: %d\n", rc);
2178 return (-ENOMEM); /* or something... */
2181 /* Pure gateways want the NAL started up at module load time... */
2182 rc = PtlNIInit(OPENIBNAL, LUSTRE_SRV_PTL_PID, NULL, NULL, &kibnal_ni);
2183 if (rc != PTL_OK && rc != PTL_IFACE_DUP) {
2184 ptl_unregister_nal(OPENIBNAL);
2188 kibnal_tunables.kib_sysctl =
2189 register_sysctl_table (kibnal_top_ctl_table, 0);
2190 if (kibnal_tunables.kib_sysctl == NULL) {
2191 CERROR("Can't register sysctl table\n");
2192 PtlNIFini(kibnal_ni);
2193 ptl_unregister_nal(OPENIBNAL);
2200 MODULE_AUTHOR("Cluster File Systems, Inc. <info@clusterfs.com>");
2201 MODULE_DESCRIPTION("Kernel OpenIB NAL v0.01");
2202 MODULE_LICENSE("GPL");
2204 module_init(kibnal_module_init);
2205 module_exit(kibnal_module_fini);