From 39b5de7aa9ca287556964843650767ff93fc56f5 Mon Sep 17 00:00:00 2001 From: eeb Date: Tue, 7 Jun 2005 23:18:56 +0000 Subject: [PATCH] * Got rid of portals/router; getting it to build in portals/portals using symlinks didn't work on 2.6 and router.c has changed enough for the diffs not to make sense any more (and life is too short). * reworked how reconnect timeouts work in all connection based NALs, to close out a bug where reconnection could be delayed until jiffies wraps. * tested vibnal with a single echo client + server @ LLNL on 2.6, x86_64 * make configure-on-demand rather than configure-on-load the default; modprobe in 2.6 does locking and deadlocks if the module it is loading tries to load more modules in its init sequence. * removed some unused vars etc caught by better warnings when building for 2.6/x86_64 --- lnet/Makefile.in | 1 - lnet/autoMakefile.am | 2 +- lnet/autoconf/lustre-lnet.m4 | 2 -- lnet/include/libcfs/libcfs.h | 1 + lnet/klnds/iiblnd/iiblnd.c | 3 +- lnet/klnds/iiblnd/iiblnd_cb.c | 20 +++++++------ lnet/klnds/openiblnd/openiblnd.c | 46 ++++++++++++++++++------------ lnet/klnds/openiblnd/openiblnd_cb.c | 22 +++++++------- lnet/klnds/ralnd/ralnd.c | 42 ++++++++++----------------- lnet/klnds/ralnd/ralnd_cb.c | 4 +-- lnet/klnds/socklnd/socklnd.c | 8 ++---- lnet/klnds/socklnd/socklnd_cb.c | 15 +++++++--- lnet/klnds/socklnd/socklnd_lib-linux.c | 21 ++++++-------- lnet/klnds/viblnd/viblnd.c | 16 +++++------ lnet/klnds/viblnd/viblnd_cb.c | 23 ++++++++------- lnet/libcfs/linux/linux-tcpip.c | 34 ++++++++++++++++++++-- lnet/libcfs/nidstrings.c | 7 ++--- lnet/lnet/Makefile.in | 10 +------ lnet/lnet/api-ni.c | 15 +++++----- lnet/lnet/autoMakefile.am | 3 +- lnet/lnet/config.c | 5 ---- lnet/lnet/lib-move.c | 2 +- lnet/lnet/module.c | 4 ++- lnet/{router => lnet}/router.c | 0 lnet/{router => lnet}/router.h | 0 lnet/{router/proc.c => lnet/router_proc.c} | 3 -- lnet/router/Makefile.in | 2 -- lnet/router/autoMakefile.am | 11 ------- 28 files changed, 161 insertions(+), 161 deletions(-) rename lnet/{router => lnet}/router.c (100%) rename lnet/{router => lnet}/router.h (100%) rename lnet/{router/proc.c => lnet/router_proc.c} (98%) delete mode 100644 lnet/router/Makefile.in delete mode 100644 lnet/router/autoMakefile.am diff --git a/lnet/Makefile.in b/lnet/Makefile.in index 71d0dc8..7a33c70 100644 --- a/lnet/Makefile.in +++ b/lnet/Makefile.in @@ -2,7 +2,6 @@ subdir-m += libcfs cray-subdirs += portals cray-subdirs += knals -cray-subdirs += router cray-subdirs += tests @CRAY_PORTALS_FALSE@subdir-m += $(cray-subdirs) diff --git a/lnet/autoMakefile.am b/lnet/autoMakefile.am index b49b8d4..36da28b 100644 --- a/lnet/autoMakefile.am +++ b/lnet/autoMakefile.am @@ -3,7 +3,7 @@ # This code is issued under the GNU General Public License. # See the file COPYING in this distribution -SUBDIRS = libcfs portals knals unals router tests doc utils include \ +SUBDIRS = libcfs portals knals unals tests doc utils include \ autoconf sources: diff --git a/lnet/autoconf/lustre-lnet.m4 b/lnet/autoconf/lustre-lnet.m4 index 87d08db..6f7d3e8 100644 --- a/lnet/autoconf/lustre-lnet.m4 +++ b/lnet/autoconf/lustre-lnet.m4 @@ -758,8 +758,6 @@ portals/libcfs/autoMakefile portals/libcfs/linux/Makefile portals/portals/Makefile portals/portals/autoMakefile -portals/router/Makefile -portals/router/autoMakefile portals/tests/Makefile portals/tests/autoMakefile portals/unals/Makefile diff --git a/lnet/include/libcfs/libcfs.h b/lnet/include/libcfs/libcfs.h index c77a495..88c9aca 100644 --- a/lnet/include/libcfs/libcfs.h +++ b/lnet/include/libcfs/libcfs.h @@ -286,6 +286,7 @@ int libcfs_sock_connect(struct socket **sockp, int *fatal, int bufsize, __u32 peer_ip, int peer_port); int libcfs_sock_setbuf(struct socket *socket, int txbufsize, int rxbufsize); int libcfs_sock_getbuf(struct socket *socket, int *txbufsize, int *rxbufsize); +int libcfs_sock_getaddr(struct socket *socket, int remote, __u32 *ip, int *port); int libcfs_sock_write(struct socket *sock, void *buffer, int nob, int timeout); int libcfs_sock_read(struct socket *sock, void *buffer, int nob, int timeout); void libcfs_sock_release(struct socket *sock); diff --git a/lnet/klnds/iiblnd/iiblnd.c b/lnet/klnds/iiblnd/iiblnd.c index 37a8051..221cb08 100644 --- a/lnet/klnds/iiblnd/iiblnd.c +++ b/lnet/klnds/iiblnd/iiblnd.c @@ -367,8 +367,7 @@ kibnal_create_peer (ptl_nid_t nid) INIT_LIST_HEAD (&peer->ibp_conns); INIT_LIST_HEAD (&peer->ibp_tx_queue); - peer->ibp_reconnect_time = jiffies; - peer->ibp_reconnect_interval = IBNAL_MIN_RECONNECT_INTERVAL; + peer->ibp_reconnect_interval = 0; /* OK to connect at any time */ atomic_inc (&kibnal_data.kib_npeers); return (peer); diff --git a/lnet/klnds/iiblnd/iiblnd_cb.c b/lnet/klnds/iiblnd/iiblnd_cb.c index bb1bb07..4c4d395 100644 --- a/lnet/klnds/iiblnd/iiblnd_cb.c +++ b/lnet/klnds/iiblnd/iiblnd_cb.c @@ -1216,7 +1216,8 @@ kibnal_launch_tx (kib_tx_t *tx, ptl_nid_t nid) } if (peer->ibp_connecting == 0) { - if (!time_after_eq(jiffies, peer->ibp_reconnect_time)) { + if (!(peer->ibp_reconnect_interval == 0 || /* first attempt */ + time_after_eq(jiffies, peer->ibp_reconnect_time))) { write_unlock_irqrestore (g_lock, flags); tx->tx_status = -EHOSTUNREACH; kibnal_tx_done (tx); @@ -1804,7 +1805,6 @@ kibnal_peer_connect_failed (kib_peer_t *peer, int active, int rc) unsigned long flags; LASSERT (rc != 0); - LASSERT (peer->ibp_reconnect_interval >= IBNAL_MIN_RECONNECT_INTERVAL); write_lock_irqsave (&kibnal_data.kib_global_lock, flags); @@ -1819,10 +1819,15 @@ kibnal_peer_connect_failed (kib_peer_t *peer, int active, int rc) if (list_empty(&peer->ibp_conns)) { /* Say when active connection can be re-attempted */ - peer->ibp_reconnect_time = jiffies + peer->ibp_reconnect_interval; - /* Increase reconnection interval */ - peer->ibp_reconnect_interval = MIN (peer->ibp_reconnect_interval * 2, - IBNAL_MAX_RECONNECT_INTERVAL); + peer->ibp_reconnect_interval *= 2; + peer->ibp_reconnect_interval = + MAX(peer->ibp_reconnect_interval, + *kibnal_tunables.kib_min_reconnect_interval); + peer->ibp_reconnect_interval = + MIN(peer->ibp_reconnect_interval, + *kibnal_tunables.kib_max_reconnect_interval); + + peer->ibp_reconnect_time += peer->ibp_reconnect_interval; /* Take peer's blocked blocked transmits; I'll complete * them with error */ @@ -1908,8 +1913,7 @@ kibnal_connreq_done (kib_conn_t *conn, int active, int status) atomic_inc (&conn->ibc_refcount); list_add (&conn->ibc_list, &peer->ibp_conns); - /* reset reconnect interval for next attempt */ - peer->ibp_reconnect_interval = IBNAL_MIN_RECONNECT_INTERVAL; + peer->ibp_reconnect_interval = 0; /* OK to reconnect at any time */ /* post blocked sends to the new connection */ spin_lock (&conn->ibc_lock); diff --git a/lnet/klnds/openiblnd/openiblnd.c b/lnet/klnds/openiblnd/openiblnd.c index f618339..de5aa0c 100644 --- a/lnet/klnds/openiblnd/openiblnd.c +++ b/lnet/klnds/openiblnd/openiblnd.c @@ -595,14 +595,17 @@ kibnal_stop_ib_listener (void) int kibnal_create_peer (kib_peer_t **peerp, ptl_nid_t nid) { - kib_peer_t *peer; - unsigned long flags; + kib_peer_t *peer; + unsigned long flags; + int rc; LASSERT (nid != PTL_NID_ANY); - PORTAL_ALLOC (peer, sizeof (*peer)); - if (peer == NULL) + PORTAL_ALLOC(peer, sizeof (*peer)); + if (peer == NULL) { + CERROR("Cannot allocate peer\n"); return -ENOMEM; + } memset(peer, 0, sizeof(*peer)); /* zero flags etc */ @@ -614,27 +617,32 @@ kibnal_create_peer (kib_peer_t **peerp, ptl_nid_t nid) INIT_LIST_HEAD (&peer->ibp_tx_queue); INIT_LIST_HEAD (&peer->ibp_connd_list); /* not queued for connecting */ - peer->ibp_reconnect_time = jiffies; - peer->ibp_reconnect_interval = *kibnal_tunables.kib_min_reconnect_interval; + peer->ibp_reconnect_interval = 0; /* OK to connect at any time */ write_lock_irqsave(&kibnal_data.kib_global_lock, flags); - - if (kibnal_data.kib_listener_shutdown) { - /* shutdown has started already */ - write_unlock_irqrestore(&kibnal_data.kib_global_lock, flags); - - PORTAL_FREE(peer, sizeof(*peer)); - CERROR("Can't create peer: network shutdown\n"); - return -ESHUTDOWN; + + if (kibnal_data.kib_npeers >= + *kibnal_tunables.kib_concurrent_peers) { + rc = -EOVERFLOW; /* !! but at least it distinguishes */ + } else if (kibnal_data.kib_listener_shutdown) { + rc = -ESHUTDOWN; /* shutdown has started */ + } else { + rc = 0; + kibnal_data.kib_npeers++; } - kibnal_data.kib_npeers++; - write_unlock_irqrestore(&kibnal_data.kib_global_lock, flags); - CDEBUG(D_NET, "peer %p "LPX64"\n", peer, nid); - *peerp = peer; - return 0; + if (rc != 0) { + CERROR("Can't create peer: %s\n", + (rc == -ESHUTDOWN) ? "shutting down" : + "too many peers"); + PORTAL_FREE(peer, sizeof(*peer)); + } else { + *peerp = peer; + } + + return rc; } void diff --git a/lnet/klnds/openiblnd/openiblnd_cb.c b/lnet/klnds/openiblnd/openiblnd_cb.c index 7812a03..c514599 100644 --- a/lnet/klnds/openiblnd/openiblnd_cb.c +++ b/lnet/klnds/openiblnd/openiblnd_cb.c @@ -995,7 +995,8 @@ kibnal_launch_tx (kib_tx_t *tx, ptl_nid_t nid) } if (peer->ibp_connecting == 0) { - if (!time_after_eq(jiffies, peer->ibp_reconnect_time)) { + if (!(peer->ibp_reconnect_interval == 0 || /* first attempt */ + time_after_eq(jiffies, peer->ibp_reconnect_time))) { write_unlock_irqrestore (g_lock, flags); tx->tx_status = -EHOSTUNREACH; kibnal_tx_done (tx); @@ -1533,8 +1534,6 @@ kibnal_peer_connect_failed (kib_peer_t *peer, int rc) unsigned long flags; LASSERT (rc != 0); - LASSERT (peer->ibp_reconnect_interval >= - *kibnal_tunables.kib_min_reconnect_interval); write_lock_irqsave (&kibnal_data.kib_global_lock, flags); @@ -1549,10 +1548,15 @@ kibnal_peer_connect_failed (kib_peer_t *peer, int rc) if (list_empty(&peer->ibp_conns)) { /* Say when active connection can be re-attempted */ + peer->ibp_reconnect_interval *= 2; + peer->ibp_reconnect_interval = + MAX(peer->ibp_reconnect_interval, + *kibnal_tunables.kib_min_reconnect_interval); + peer->ibp_reconnect_interval = + MIN(peer->ibp_reconnect_interval, + *kibnal_tunables.kib_max_reconnect_interval); + peer->ibp_reconnect_time = jiffies + peer->ibp_reconnect_interval; - /* Increase reconnection interval */ - peer->ibp_reconnect_interval = MIN (peer->ibp_reconnect_interval * 2, - IBNAL_MAX_RECONNECT_INTERVAL); /* Take peer's blocked blocked transmits; I'll complete * them with error */ @@ -1649,10 +1653,8 @@ kibnal_connreq_done (kib_conn_t *conn, int status) atomic_read (&conn->ibc_refcount)); atomic_inc (&conn->ibc_refcount); list_add (&conn->ibc_list, &peer->ibp_conns); - - /* reset reconnect interval for next attempt */ - peer->ibp_reconnect_interval = - *kibnal_tunables.kib_min_reconnect_interval; + + peer->ibp_reconnect_interval = 0; /* OK to reconnect at any time */ /* post blocked sends to the new connection */ spin_lock (&conn->ibc_lock); diff --git a/lnet/klnds/ralnd/ralnd.c b/lnet/klnds/ralnd/ralnd.c index a091b21..fc642ba 100644 --- a/lnet/klnds/ralnd/ralnd.c +++ b/lnet/klnds/ralnd/ralnd.c @@ -404,7 +404,6 @@ int kranal_passive_conn_handshake (struct socket *sock, ptl_nid_t *src_nidp, ptl_nid_t *dst_nidp, kra_conn_t **connp) { - struct sockaddr_in addr; __u32 peer_ip; unsigned int peer_port; kra_connreq_t rx_connreq; @@ -412,19 +411,14 @@ kranal_passive_conn_handshake (struct socket *sock, ptl_nid_t *src_nidp, kra_conn_t *conn; kra_device_t *dev; int rc; - int len; int i; - len = sizeof(addr); - rc = sock->ops->getname(sock, (struct sockaddr *)&addr, &len, 2); + rc = libcfs_sock_getaddr(sock, 1, &peer_ip, &peer_port); if (rc != 0) { CERROR("Can't get peer's IP: %d\n", rc); return rc; } - peer_ip = ntohl(addr.sin_addr.s_addr); - peer_port = ntohs(addr.sin_port); - if (peer_port >= 1024) { CERROR("Refusing unprivileged connection from %u.%u.%u.%u/%d\n", HIPQUAD(peer_ip), peer_port); @@ -486,7 +480,7 @@ ranal_connect_sock(kra_peer_t *peer, struct socket **sockp) for (port = 1023; port >= 512; port--) { rc = libcfs_sock_connect(sockp, &fatal, - 2 * sizeof(kra_conn_t), + 2 * sizeof(kra_msg_t), 0, port, peer->rap_ip, peer->rap_port); if (rc == 0) @@ -756,20 +750,21 @@ kranal_connect (kra_peer_t *peer) * success to avoid messages jumping the queue */ LASSERT (list_empty(&peer->rap_tx_queue)); - /* reset reconnection timeouts */ - peer->rap_reconnect_interval = - *kranal_tunables.kra_min_reconnect_interval; - peer->rap_reconnect_time = CURRENT_SECONDS; + peer->rap_reconnect_interval = 0; /* OK to reconnect at any time */ write_unlock_irqrestore(&kranal_data.kra_global_lock, flags); return; } - LASSERT (peer->rap_reconnect_interval != 0); - peer->rap_reconnect_time = CURRENT_SECONDS + peer->rap_reconnect_interval; - peer->rap_reconnect_interval = - MAX(*kranal_tunables.kra_max_reconnect_interval, - 2 * peer->rap_reconnect_interval); + peer->rap_reconnect_interval *= 2; + peer->rap_reconnect_interval = + MAX(peer->rap_reconnect_interval, + *kranal_tunables.kra_min_reconnect_interval); + peer->rap_reconnect_interval = + MIN(peer->rap_reconnect_interval, + *kranal_tunables.kra_max_reconnect_interval); + + peer->rap_reconnect_time = jiffies + peer->rap_reconnect_interval * HZ; /* Grab all blocked packets while we have the global lock */ list_add(&zombies, &peer->rap_tx_queue); @@ -802,7 +797,6 @@ kranal_free_acceptsock (kra_acceptsock_t *ras) int kranal_listener (void *arg) { - wait_queue_t wait; struct socket *sock; kra_acceptsock_t *ras; int port; @@ -817,8 +811,6 @@ kranal_listener (void *arg) kportal_daemonize(name); kportal_blockallsigs(); - init_waitqueue_entry(&wait, current); - rc = libcfs_sock_listen(&sock, 0, port, *kranal_tunables.kra_backlog); if (rc != 0) @@ -831,8 +823,6 @@ kranal_listener (void *arg) LASSERT (kranal_data.kra_listener_shutdown == 0); up(&kranal_data.kra_listener_signal); - /* Wake me any time something happens on my socket */ - add_wait_queue(sock->sk->sk_sleep, &wait); ras = NULL; while (kranal_data.kra_listener_shutdown == 0) { @@ -847,7 +837,7 @@ kranal_listener (void *arg) } rc = libcfs_sock_accept(&ras->ras_sock, sock, - 2 * sizeof(kra_conn_t)); + 2 * sizeof(kra_msg_t)); if (rc != 0) { if (rc != -EAGAIN) { CWARN("Accept error: %d, listener pausing\n", rc); @@ -904,7 +894,7 @@ kranal_start_listener (void) rc = kranal_data.kra_listener_shutdown; LASSERT ((rc != 0) == (kranal_data.kra_listener_sock == NULL)); - CDEBUG(D_NET, "Listener %ld started OK\n", pid); + CDEBUG((rc == 0) ? D_NET : D_ERROR, "Listener startup rc: %d\n", rc); return rc; } @@ -947,9 +937,7 @@ kranal_create_peer (kra_peer_t **peerp, ptl_nid_t nid) INIT_LIST_HEAD(&peer->rap_conns); INIT_LIST_HEAD(&peer->rap_tx_queue); - peer->rap_reconnect_time = CURRENT_SECONDS; - peer->rap_reconnect_interval = - *kranal_tunables.kra_min_reconnect_interval; + peer->rap_reconnect_interval = 0; /* OK to connect at any time */ write_lock_irqsave(&kranal_data.kra_global_lock, flags); diff --git a/lnet/klnds/ralnd/ralnd_cb.c b/lnet/klnds/ralnd/ralnd_cb.c index 78993f7..3a3df48 100644 --- a/lnet/klnds/ralnd/ralnd_cb.c +++ b/lnet/klnds/ralnd/ralnd_cb.c @@ -520,8 +520,8 @@ kranal_launch_tx (kra_tx_t *tx, ptl_nid_t nid) if (!peer->rap_connecting) { LASSERT (list_empty(&peer->rap_tx_queue)); - now = CURRENT_SECONDS; - if (now < peer->rap_reconnect_time) { + if (!(peer->rap_reconnect_interval == 0 || /* first attempt */ + time_after_eq(jiffies, peer->rap_reconnect_time))) { write_unlock_irqrestore(g_lock, flags); kranal_tx_done(tx, -EHOSTUNREACH); return; diff --git a/lnet/klnds/socklnd/socklnd.c b/lnet/klnds/socklnd/socklnd.c index 9655ad9..f0cfc8c 100644 --- a/lnet/klnds/socklnd/socklnd.c +++ b/lnet/klnds/socklnd/socklnd.c @@ -69,9 +69,7 @@ ksocknal_create_route (__u32 ipaddr, int port) atomic_set (&route->ksnr_refcount, 1); route->ksnr_peer = NULL; - route->ksnr_timeout = cfs_time_current(); - route->ksnr_retry_interval = - cfs_time_seconds(*ksocknal_tunables.ksnd_min_reconnectms)/1000; + route->ksnr_retry_interval = 0; /* OK to connect at any time */ route->ksnr_ipaddr = ipaddr; route->ksnr_port = port; route->ksnr_connecting = 0; @@ -341,9 +339,7 @@ ksocknal_associate_route_conn_locked(ksock_route_t *route, ksock_conn_t *conn) /* Successful connection => further attempts can * proceed immediately */ - route->ksnr_timeout = cfs_time_current(); - route->ksnr_retry_interval = - cfs_time_seconds(*ksocknal_tunables.ksnd_min_reconnectms)/1000; + route->ksnr_retry_interval = 0; } void diff --git a/lnet/klnds/socklnd/socklnd_cb.c b/lnet/klnds/socklnd/socklnd_cb.c index f6bcac0..f0f1dc7 100644 --- a/lnet/klnds/socklnd/socklnd_cb.c +++ b/lnet/klnds/socklnd/socklnd_cb.c @@ -656,7 +656,9 @@ ksocknal_find_connectable_route_locked (ksock_peer_t *peer) continue; /* too soon to retry this guy? */ - if (!cfs_time_aftereq (cfs_time_current(), route->ksnr_timeout)) + if (!(route->ksnr_retry_interval == 0 || /* first attempt */ + cfs_time_aftereq (cfs_time_current(), + route->ksnr_timeout))) continue; return (route); @@ -2005,12 +2007,17 @@ ksocknal_connect (ksock_route_t *route) route->ksnr_connecting = 0; /* This is a retry rather than a new connection */ + route->ksnr_retry_interval *= 2; + route->ksnr_retry_interval = + MAX(route->ksnr_retry_interval, + cfs_time_seconds(*ksocknal_tunables.ksnd_min_reconnectms)/1000); + route->ksnr_retry_interval = + MIN(route->ksnr_retry_interval, + cfs_time_seconds(*ksocknal_tunables.ksnd_max_reconnectms)/1000); + LASSERT (route->ksnr_retry_interval != 0); route->ksnr_timeout = cfs_time_add(cfs_time_current(), route->ksnr_retry_interval); - route->ksnr_retry_interval = - MIN (route->ksnr_retry_interval * 2, - cfs_time_seconds(*ksocknal_tunables.ksnd_max_reconnectms)/1000); if (!list_empty (&peer->ksnp_tx_queue) && ksocknal_find_connecting_route_locked (peer) == NULL) { diff --git a/lnet/klnds/socklnd/socklnd_lib-linux.c b/lnet/klnds/socklnd/socklnd_lib-linux.c index d82e4b8..515cd9ed 100644 --- a/lnet/klnds/socklnd/socklnd_lib-linux.c +++ b/lnet/klnds/socklnd/socklnd_lib-linux.c @@ -152,12 +152,10 @@ ksocknal_lib_bind_irq (unsigned int irq) int ksocknal_lib_get_conn_addrs (ksock_conn_t *conn) { - struct sockaddr_in sin; - int len = sizeof (sin); - int rc; + int rc = libcfs_sock_getaddr(conn->ksnc_sock, 1, + &conn->ksnc_ipaddr, + &conn->ksnc_port); - rc = conn->ksnc_sock->ops->getname (conn->ksnc_sock, - (struct sockaddr *)&sin, &len, 2); /* Didn't need the {get,put}connsock dance to deref ksnc_sock... */ LASSERT (!conn->ksnc_closing); @@ -166,18 +164,13 @@ ksocknal_lib_get_conn_addrs (ksock_conn_t *conn) return rc; } - conn->ksnc_ipaddr = ntohl (sin.sin_addr.s_addr); - conn->ksnc_port = ntohs (sin.sin_port); - - rc = conn->ksnc_sock->ops->getname (conn->ksnc_sock, - (struct sockaddr *)&sin, &len, 0); + rc = libcfs_sock_getaddr(conn->ksnc_sock, 0, + &conn->ksnc_myipaddr, NULL); if (rc != 0) { CERROR ("Error %d getting sock local IP\n", rc); return rc; } - conn->ksnc_myipaddr = ntohl (sin.sin_addr.s_addr); - return 0; } @@ -185,6 +178,7 @@ unsigned int ksocknal_lib_sock_irq (struct socket *sock) { int irq = 0; +#if CPU_AFFINITY struct dst_entry *dst; if (!ksocknal_tunables.ksnd_irq_affinity) @@ -202,7 +196,8 @@ ksocknal_lib_sock_irq (struct socket *sock) dst_release (dst); } - return (irq); +#endif + return irq; } #if (SOCKNAL_ZC && SOCKNAL_VADDR_ZC) diff --git a/lnet/klnds/viblnd/viblnd.c b/lnet/klnds/viblnd/viblnd.c index 4df2dee..04eb208 100644 --- a/lnet/klnds/viblnd/viblnd.c +++ b/lnet/klnds/viblnd/viblnd.c @@ -463,7 +463,7 @@ kibnal_create_peer (kib_peer_t **peerp, ptl_nid_t nid) PORTAL_ALLOC(peer, sizeof (*peer)); if (peer == NULL) { - CERROR("Canot allocate perr\n"); + CERROR("Cannot allocate peer\n"); return -ENOMEM; } @@ -476,13 +476,11 @@ kibnal_create_peer (kib_peer_t **peerp, ptl_nid_t nid) INIT_LIST_HEAD (&peer->ibp_conns); INIT_LIST_HEAD (&peer->ibp_tx_queue); - peer->ibp_reconnect_time = jiffies; - peer->ibp_reconnect_interval = - *kibnal_tunables.kib_min_reconnect_interval * HZ; + peer->ibp_reconnect_interval = 0; /* OK to connect at any time */ write_lock_irqsave(&kibnal_data.kib_global_lock, flags); - - if (kibnal_data.kib_npeers < + + if (kibnal_data.kib_npeers >= *kibnal_tunables.kib_concurrent_peers) { rc = -EOVERFLOW; /* !! but at least it distinguishes */ } else if (kibnal_data.kib_listen_handle == NULL) { @@ -493,7 +491,7 @@ kibnal_create_peer (kib_peer_t **peerp, ptl_nid_t nid) } write_unlock_irqrestore(&kibnal_data.kib_global_lock, flags); - + if (rc != 0) { CERROR("Can't create peer: %s\n", (rc == -ESHUTDOWN) ? "shutting down" : @@ -702,7 +700,7 @@ kibnal_del_peer (ptl_nid_t nid) rc = 0; /* matched something */ } } - out: + write_unlock_irqrestore(&kibnal_data.kib_global_lock, flags); return (rc); } @@ -1930,7 +1928,7 @@ kibnal_module_init (void) } MODULE_AUTHOR("Cluster File Systems, Inc. "); -MODULE_DESCRIPTION("Kernel Voltaire IB NAL v0.01"); +MODULE_DESCRIPTION("Kernel Voltaire IB NAL v1.00"); MODULE_LICENSE("GPL"); module_init(kibnal_module_init); diff --git a/lnet/klnds/viblnd/viblnd_cb.c b/lnet/klnds/viblnd/viblnd_cb.c index 6de75ad..7138d04 100644 --- a/lnet/klnds/viblnd/viblnd_cb.c +++ b/lnet/klnds/viblnd/viblnd_cb.c @@ -1315,7 +1315,8 @@ kibnal_launch_tx (kib_tx_t *tx, ptl_nid_t nid) } if (peer->ibp_connecting == 0) { - if (!time_after_eq(jiffies, peer->ibp_reconnect_time)) { + if (!(peer->ibp_reconnect_interval == 0 || /* first attempt */ + time_after_eq(jiffies, peer->ibp_reconnect_time))) { write_unlock_irqrestore(g_lock, flags); tx->tx_status = -EHOSTUNREACH; tx->tx_waiting = 0; @@ -1752,7 +1753,6 @@ kibnal_close_conn_locked (kib_conn_t *conn, int error) * already dealing with it (either to set it up or tear it down). * Caller holds kib_global_lock exclusively in irq context */ kib_peer_t *peer = conn->ibc_peer; - struct list_head *tmp; LASSERT (error != 0 || conn->ibc_state >= IBNAL_CONN_ESTABLISHED); @@ -1948,11 +1948,15 @@ kibnal_peer_connect_failed (kib_peer_t *peer, int active) if (list_empty(&peer->ibp_conns)) { /* Say when active connection can be re-attempted */ + peer->ibp_reconnect_interval *= 2; + peer->ibp_reconnect_interval = + MAX(peer->ibp_reconnect_interval, + *kibnal_tunables.kib_min_reconnect_interval); + peer->ibp_reconnect_interval = + MIN(peer->ibp_reconnect_interval, + *kibnal_tunables.kib_max_reconnect_interval); + peer->ibp_reconnect_time = jiffies + peer->ibp_reconnect_interval; - /* Increase reconnection interval */ - peer->ibp_reconnect_interval = - MIN (peer->ibp_reconnect_interval * 2, - *kibnal_tunables.kib_max_reconnect_interval * HZ); /* Take peer's blocked transmits to complete with error */ list_add(&zombies, &peer->ibp_tx_queue); @@ -2104,9 +2108,8 @@ kibnal_connreq_done(kib_conn_t *conn, int active, int status) list_add(&txs, &peer->ibp_tx_queue); list_del_init(&peer->ibp_tx_queue); - /* reset reconnect interval for next attempt */ - peer->ibp_reconnect_interval = - *kibnal_tunables.kib_min_reconnect_interval * HZ; + peer->ibp_reconnect_interval = 0; /* OK to reconnect at any time */ + write_unlock_irqrestore(&kibnal_data.kib_global_lock, flags); /* Schedule blocked txs */ @@ -2447,7 +2450,6 @@ kibnal_active_connect_callback (cm_cep_handle_t cep, cm_conn_data_t *cd, /* CAVEAT EMPTOR: tasklet context */ kib_conn_t *conn = (kib_conn_t *)arg; kib_connvars_t *cv = conn->ibc_connvars; - unsigned long flags; LASSERT (conn->ibc_state == IBNAL_CONN_ACTIVE_CONNECT); cv->cv_conndata = *cd; @@ -2794,7 +2796,6 @@ kibnal_arp_callback (ibat_stat_t arprc, ibat_arp_data_t *arp_data, void *arg) /* CAVEAT EMPTOR: tasklet context */ kib_conn_t *conn = (kib_conn_t *)arg; kib_peer_t *peer = conn->ibc_peer; - unsigned long flags; if (arprc != ibat_stat_ok) CERROR("Arp "LPX64"@%u.%u.%u.%u failed: %d\n", diff --git a/lnet/libcfs/linux/linux-tcpip.c b/lnet/libcfs/linux/linux-tcpip.c index 1c996b2..7ba18cf 100644 --- a/lnet/libcfs/linux/linux-tcpip.c +++ b/lnet/libcfs/linux/linux-tcpip.c @@ -45,7 +45,7 @@ libcfs_ipif_query (char *name, int *up, __u32 *ip, __u32 *mask) nob = strnlen(name, IFNAMSIZ); if (nob == IFNAMSIZ) { CERROR("Interface name %s too long\n", name); - rc -EINVAL; + rc = -EINVAL; goto out; } @@ -68,6 +68,8 @@ libcfs_ipif_query (char *name, int *up, __u32 *ip, __u32 *mask) goto out; } + *up = 1; + strcpy(ifr.ifr_name, name); ifr.ifr_addr.sa_family = AF_INET; set_fs(KERNEL_DS); @@ -275,7 +277,9 @@ libcfs_sock_write (struct socket *sock, void *buffer, int nob, int timeout) } set_fs (KERNEL_DS); + then = jiffies; rc = sock_sendmsg (sock, &msg, iov.iov_len); + ticks -= then - jiffies; set_fs (oldmm); if (rc == nob) @@ -289,7 +293,7 @@ libcfs_sock_write (struct socket *sock, void *buffer, int nob, int timeout) return (-ECONNABORTED); } - if (timeout == 0) + if (ticks <= 0) return -EAGAIN; buffer = ((char *)buffer) + rc; @@ -468,6 +472,32 @@ libcfs_sock_setbuf (struct socket *sock, int txbufsize, int rxbufsize) EXPORT_SYMBOL(libcfs_sock_setbuf); int +libcfs_sock_getaddr (struct socket *sock, int remote, __u32 *ip, int *port) +{ + struct sockaddr_in sin; + int len = sizeof (sin); + int rc; + + rc = sock->ops->getname (sock, (struct sockaddr *)&sin, &len, + remote ? 2 : 0); + if (rc != 0) { + CERROR ("Error %d getting sock %s IP/port\n", + rc, remote ? "peer" : "local"); + return rc; + } + + if (ip != NULL) + *ip = ntohl (sin.sin_addr.s_addr); + + if (port != NULL) + *port = ntohs (sin.sin_port); + + return 0; +} + +EXPORT_SYMBOL(libcfs_sock_getaddr); + +int libcfs_sock_getbuf (struct socket *sock, int *txbufsize, int *rxbufsize) { mm_segment_t oldmm = get_fs(); diff --git a/lnet/libcfs/nidstrings.c b/lnet/libcfs/nidstrings.c index 9b84b66..db73ccd 100644 --- a/lnet/libcfs/nidstrings.c +++ b/lnet/libcfs/nidstrings.c @@ -403,11 +403,10 @@ libcfs_str2net(char *str) ptl_nid_t libcfs_str2nid(char *str) { - char *sep = strchr(str, '@'); + char *sep = strchr(str, '@'); struct nalstrfns *nf; - int nob; - __u32 net; - __u32 addr; + __u32 net; + __u32 addr; if (sep != NULL) { nf = libcfs_str2net_internal(sep + 1, &net); diff --git a/lnet/lnet/Makefile.in b/lnet/lnet/Makefile.in index 851e6c9..926dc82 100644 --- a/lnet/lnet/Makefile.in +++ b/lnet/lnet/Makefile.in @@ -1,18 +1,10 @@ MODULES := portals -router_objs := $(ROUTER)r_router.o -router_objs += $(ROUTER)r_proc.o - portals-objs := api-errno.o api-ni.o config.o portals-objs += lib-me.o lib-msg.o lib-eq.o lib-md.o portals-objs += lib-move.o module.o lo.o -portals-objs += $(router_objs) +portals-objs += router.o router_proc.o default: all -r_%.c: ../router/%.c - ln -sf $< $@ - -EXTRA_PRE_CFLAGS := -I../router - @INCLUDE_RULES@ diff --git a/lnet/lnet/api-ni.c b/lnet/lnet/api-ni.c index 89e075d..eeb7ab8 100644 --- a/lnet/lnet/api-ni.c +++ b/lnet/lnet/api-ni.c @@ -589,9 +589,6 @@ ptl_shutdown_nalnis (void) { int i; ptl_ni_t *ni; - ptl_nal_t *nal; - struct list_head *tmp; - struct list_head *nxt; unsigned long flags; /* NB called holding the global mutex */ @@ -652,9 +649,11 @@ ptl_shutdown_nalnis (void) } PTL_UNLOCK(flags); - LASSERT (ptl_apini.apini_network_tokens != NULL); - PORTAL_FREE(ptl_apini.apini_network_tokens, - ptl_apini.apini_network_tokens_nob); + if (ptl_apini.apini_network_tokens != NULL) { + PORTAL_FREE(ptl_apini.apini_network_tokens, + ptl_apini.apini_network_tokens_nob); + ptl_apini.apini_network_tokens = NULL; + } } ptl_err_t @@ -664,7 +663,6 @@ ptl_startup_nalnis (void) ptl_ni_t *ni; struct list_head nilist; ptl_err_t rc = PTL_OK; - char *interface = NULL; unsigned long flags; int nal_type; int retry; @@ -696,6 +694,9 @@ ptl_startup_nalnis (void) goto failed; } + CDEBUG(D_WARNING,"Requesting module %s\n", + libcfs_nal2modname(nal_type)); + request_module(libcfs_nal2modname(nal_type)); PTL_MUTEX_DOWN(&ptl_apini.apini_nal_mutex); diff --git a/lnet/lnet/autoMakefile.am b/lnet/lnet/autoMakefile.am index 224b868..5952047 100644 --- a/lnet/lnet/autoMakefile.am +++ b/lnet/lnet/autoMakefile.am @@ -1,7 +1,7 @@ my_sources = api-errno.c api-ni.c config.c \ lib-me.c lib-msg.c lib-eq.c \ lib-md.c lib-move.c lo.c \ - $(top_srcdir)/portals/router/router.c + router.c if !CRAY_PORTALS @@ -24,6 +24,7 @@ macos_PROGRAMS := portals portals_SOURCES := api-errno.c api-ni.c config.c portals_SOURCES += lib-me.c lib-msg.c lib-eq.c lib-md.c portals_SOURCES += lib-move.c module.c lo.c +portals_SOURCES += router.c portals_CFLAGS := $(EXTRA_KCFLAGS) portals_LDFLAGS := $(EXTRA_KLDFLAGS) diff --git a/lnet/lnet/config.c b/lnet/lnet/config.c index f2d014b..0706bc0 100644 --- a/lnet/lnet/config.c +++ b/lnet/lnet/config.c @@ -106,9 +106,6 @@ ptl_trimwhite(char *str) int ptl_nis_conflict(ptl_ni_t *ni1, ptl_ni_t *ni2) { - int i; - int j; - if (PTL_NETNAL(PTL_NIDNET(ni1->ni_nid)) != /* different NALs */ PTL_NETNAL(PTL_NIDNET(ni2->ni_nid))) return 0; @@ -189,7 +186,6 @@ ptl_parse_networks(struct list_head *nilist, char *networks) char *str; ptl_ni_t *ni = NULL; __u32 net; - int rc; if (strlen(networks) > PAGE_SIZE) { /* _WAY_ conservative */ @@ -579,7 +575,6 @@ ptl_parse_route (char *str) __u32 net; ptl_nid_t nid; ptl_text_buf_t *ptb; - ptl_text_buf_t *tb2; int rc; char *sep; char *token = str; diff --git a/lnet/lnet/lib-move.c b/lnet/lnet/lib-move.c index b30e396..6297c51 100644 --- a/lnet/lnet/lib-move.c +++ b/lnet/lnet/lib-move.c @@ -634,7 +634,7 @@ ptl_send (ptl_ni_t *ni, void *private, ptl_msg_t *msg, type, target, routing, md->md_niov, md->md_iov.kiov, offset, len); - out: + ptl_ni_decref(ni); /* lose ref from kpr_lookup */ return rc; } diff --git a/lnet/lnet/module.c b/lnet/lnet/module.c index a435fa2..329ae9c 100644 --- a/lnet/lnet/module.c +++ b/lnet/lnet/module.c @@ -25,7 +25,7 @@ #define DEBUG_SUBSYSTEM S_PORTALS #include -static int config_on_load = 1; +static int config_on_load = 0; CFS_MODULE_PARM(config_on_load, "i", int, 0444, "configure network at module load"); @@ -83,6 +83,8 @@ static int init_kportals_module(void) RETURN(rc); } + CDEBUG(D_WARNING, "Startup: config_on_load=%d\n", config_on_load); + if (config_on_load) { ptl_handle_ni_t nih; diff --git a/lnet/router/router.c b/lnet/lnet/router.c similarity index 100% rename from lnet/router/router.c rename to lnet/lnet/router.c diff --git a/lnet/router/router.h b/lnet/lnet/router.h similarity index 100% rename from lnet/router/router.h rename to lnet/lnet/router.h diff --git a/lnet/router/proc.c b/lnet/lnet/router_proc.c similarity index 98% rename from lnet/router/proc.c rename to lnet/lnet/router_proc.c index dd54b59..7c6b07e 100644 --- a/lnet/router/proc.c +++ b/lnet/lnet/router_proc.c @@ -150,7 +150,6 @@ static void * kpr_seq_routes_start (struct seq_file *s, loff_t *pos) { kpr_seq_route_iterator_t *sri; - unsigned long flags; int rc; PORTAL_ALLOC(sri, sizeof(*sri)); @@ -179,7 +178,6 @@ static void * kpr_seq_routes_next (struct seq_file *s, void *iter, loff_t *pos) { kpr_seq_route_iterator_t *sri = iter; - unsigned long flags; int rc; loff_t next = *pos + 1; @@ -201,7 +199,6 @@ kpr_seq_routes_show (struct seq_file *s, void *iter) __u32 net; ptl_nid_t nid; int alive; - int stale; read_lock_irqsave(&kpr_state.kpr_rwlock, flags); diff --git a/lnet/router/Makefile.in b/lnet/router/Makefile.in deleted file mode 100644 index c43bc9f..0000000 --- a/lnet/router/Makefile.in +++ /dev/null @@ -1,2 +0,0 @@ - -@INCLUDE_RULES@ diff --git a/lnet/router/autoMakefile.am b/lnet/router/autoMakefile.am deleted file mode 100644 index 7fb9ed9..0000000 --- a/lnet/router/autoMakefile.am +++ /dev/null @@ -1,11 +0,0 @@ -# Copyright (C) 2001 Cluster File Systems, Inc. -# -# This code is issued under the GNU General Public License. -# See the file COPYING in this distribution - -# -# router is built into portals -# - -MOSTLYCLEANFILES = *.o *.ko *.mod.c -DIST_SOURCES = router.c proc.c router.h -- 1.8.3.1