From f5662ee637b57ad2ce0b63075f4548c5c8f6aef5 Mon Sep 17 00:00:00 2001 From: eeb Date: Thu, 5 May 2005 06:06:00 +0000 Subject: [PATCH] * placeholder --- lnet/include/lnet/lib-lnet.h | 8 +- lnet/include/lnet/lib-p30.h | 8 +- lnet/include/lnet/lib-types.h | 3 - lnet/klnds/qswlnd/qswlnd_cb.c | 51 ++-- lnet/klnds/socklnd/socklnd.c | 59 ++-- lnet/klnds/socklnd/socklnd_cb.c | 158 +++++----- lnet/lnet/Makefile.in | 2 +- lnet/lnet/api-ni.c | 295 +----------------- lnet/lnet/autoMakefile.am | 4 +- lnet/lnet/config.c | 645 ++++++++++++++++++++++++++++++++++++++++ lnet/router/proc.c | 357 ++++++++++++---------- lnet/router/router.c | 131 ++++---- lnet/router/router.h | 26 +- lnet/utils/routerstat.c | 26 +- lnet/utils/wirecheck.c | 4 +- 15 files changed, 1117 insertions(+), 660 deletions(-) create mode 100644 lnet/lnet/config.c diff --git a/lnet/include/lnet/lib-lnet.h b/lnet/include/lnet/lib-lnet.h index 2a83b6c..d02bcdd 100644 --- a/lnet/include/lnet/lib-lnet.h +++ b/lnet/include/lnet/lib-lnet.h @@ -364,7 +364,7 @@ ptl_handle2me (ptl_handle_me_t *handle) /* Portals Router */ /* NI APIs */ -int kpr_routing(void); +int kpr_forwarding(void); ptl_nid_t kpr_lookup(ptl_ni_t **ni, ptl_nid_t nid, int nob); void kpr_fwd_start(ptl_ni_t *ni, kpr_fwd_desc_t *fwd); void kpr_fwd_done(ptl_ni_t *ni, kpr_fwd_desc_t *fwd, int error); @@ -372,7 +372,8 @@ int kpr_notify(ptl_ni_t *ni, ptl_nid_t peer, int alive, time_t when); /* internal APIs */ int kpr_ctl(unsigned int cmd, void *arg); -void kpr_initialise(void); +int kpr_add_route(__u32 net, ptl_nid_t gateway_nid); +int kpr_initialise(void); void kpr_finalise(void); static inline void @@ -455,4 +456,7 @@ extern void ptl_register_nal(ptl_nal_t *nal); extern void ptl_unregister_nal(ptl_nal_t *nal); #endif +extern ptl_err_t ptl_parse_routes (char *route_str); +extern ptl_err_t ptl_parse_networks (struct list_head *nilist, char *networks); + #endif diff --git a/lnet/include/lnet/lib-p30.h b/lnet/include/lnet/lib-p30.h index 2a83b6c..d02bcdd 100644 --- a/lnet/include/lnet/lib-p30.h +++ b/lnet/include/lnet/lib-p30.h @@ -364,7 +364,7 @@ ptl_handle2me (ptl_handle_me_t *handle) /* Portals Router */ /* NI APIs */ -int kpr_routing(void); +int kpr_forwarding(void); ptl_nid_t kpr_lookup(ptl_ni_t **ni, ptl_nid_t nid, int nob); void kpr_fwd_start(ptl_ni_t *ni, kpr_fwd_desc_t *fwd); void kpr_fwd_done(ptl_ni_t *ni, kpr_fwd_desc_t *fwd, int error); @@ -372,7 +372,8 @@ int kpr_notify(ptl_ni_t *ni, ptl_nid_t peer, int alive, time_t when); /* internal APIs */ int kpr_ctl(unsigned int cmd, void *arg); -void kpr_initialise(void); +int kpr_add_route(__u32 net, ptl_nid_t gateway_nid); +int kpr_initialise(void); void kpr_finalise(void); static inline void @@ -455,4 +456,7 @@ extern void ptl_register_nal(ptl_nal_t *nal); extern void ptl_unregister_nal(ptl_nal_t *nal); #endif +extern ptl_err_t ptl_parse_routes (char *route_str); +extern ptl_err_t ptl_parse_networks (struct list_head *nilist, char *networks); + #endif diff --git a/lnet/include/lnet/lib-types.h b/lnet/include/lnet/lib-types.h index 3638141..cc91093 100644 --- a/lnet/include/lnet/lib-types.h +++ b/lnet/include/lnet/lib-types.h @@ -368,9 +368,6 @@ typedef struct ptl_pid_t apini_pid; /* requested pid */ ptl_ni_limits_t apini_actual_limits; - char *apini_net_tokens; /* tokenized 'networks' */ - int apini_net_tokens_nob; - struct list_head apini_nis; /* NAL instances */ struct list_head apini_zombie_nis; /* dying NAL instances */ int apini_nzombie_nis; /* # of NIS to wait for */ diff --git a/lnet/klnds/qswlnd/qswlnd_cb.c b/lnet/klnds/qswlnd/qswlnd_cb.c index b951939..c8c435c7 100644 --- a/lnet/klnds/qswlnd/qswlnd_cb.c +++ b/lnet/klnds/qswlnd/qswlnd_cb.c @@ -88,7 +88,7 @@ kqswnal_map_tx_kiov (kqswnal_tx_t *ktx, int offset, int nob, int niov, ptl_kiov_ kqswnal_nid2elanid(ktx->ktx_nid)); rail = ktx->ktx_rail; if (rail < 0) { - CERROR("No rails available for "LPX64"\n", ktx->ktx_nid); + CERROR("No rails available for %s\n", libcfs_nid2str(ktx->ktx_nid)); return (-ENETDOWN); } railmask = 1 << rail; @@ -209,7 +209,7 @@ kqswnal_map_tx_iov (kqswnal_tx_t *ktx, int offset, int nob, kqswnal_nid2elanid(ktx->ktx_nid)); rail = ktx->ktx_rail; if (rail < 0) { - CERROR("No rails available for "LPX64"\n", ktx->ktx_nid); + CERROR("No rails available for %s\n", libcfs_nid2str(ktx->ktx_nid)); return (-ENETDOWN); } railmask = 1 << rail; @@ -466,8 +466,8 @@ kqswnal_txhandler(EP_TXD *txd, void *arg, int status) if (status != EP_SUCCESS) { - CERROR ("Tx completion to "LPX64" failed: %d\n", - ktx->ktx_nid, status); + CERROR ("Tx completion to %s failed: %d\n", + libcfs_nid2str(ktx->ktx_nid), status); kqswnal_notify_peer_down(ktx); status = -EHOSTDOWN; @@ -565,7 +565,7 @@ kqswnal_launch (kqswnal_tx_t *ktx) return (0); default: /* fatal error */ - CERROR ("Tx to "LPX64" failed: %d\n", ktx->ktx_nid, rc); + CERROR ("Tx to %s failed: %d\n", libcfs_nid2str(ktx->ktx_nid), rc); kqswnal_notify_peer_down(ktx); return (-EHOSTUNREACH); } @@ -744,13 +744,13 @@ kqswnal_parse_rmd (kqswnal_rx_t *krx, int type, ptl_nid_t expected_nid) if (hdr->type != type) { CERROR ("Unexpected optimized get/put type %d (%d expected)" - "from "LPX64"\n", hdr->type, type, nid); + "from %s\n", hdr->type, type, libcfs_nid2str(nid)); return (NULL); } if (hdr->src_nid != nid) { - CERROR ("Unexpected optimized get/put source NID " - LPX64" from "LPX64"\n", hdr->src_nid, nid); + CERROR ("Unexpected optimized get/put source NID %s from %s\n", + libcfs_nid2str(hdr->src_nid), libcfs_nid2str(nid)); return (NULL); } @@ -879,8 +879,8 @@ kqswnal_rdma (kqswnal_rx_t *krx, ptl_msg_t *ptlmsg, int type, actually sending a portals message with it */ ktx = kqswnal_get_idle_tx(NULL, 0); if (ktx == NULL) { - CERROR ("Can't get txd for RDMA with "LPX64"\n", - ptlmsg->msg_ev.initiator.nid); + CERROR ("Can't get txd for RDMA with %s\n", + libcfs_nid2str(ptlmsg->msg_ev.initiator.nid)); return (-ENOMEM); } @@ -1305,8 +1305,8 @@ kqswnal_fwd_packet (ptl_ni_t *ni, kpr_fwd_desc_t *fwd) LBUG (); #endif /* The router wants this NAL to forward a packet */ - CDEBUG (D_NET, "forwarding [%p] to "LPX64", payload: %d frags %d bytes\n", - fwd, nid, niov, nob); + CDEBUG (D_NET, "forwarding [%p] to %s, payload: %d frags %d bytes\n", + fwd, libcfs_nid2str(nid), niov, nob); ktx = kqswnal_get_idle_tx (fwd, 0); if (ktx == NULL) /* can't get txd right now */ @@ -1326,7 +1326,8 @@ kqswnal_fwd_packet (ptl_ni_t *ni, kpr_fwd_desc_t *fwd) ktx->ktx_nfrag = ktx->ktx_firsttmpfrag = 1; if (kqswnal_nid2elanid (nid) < 0) { - CERROR("Can't forward [%p] to "LPX64": not a peer\n", fwd, nid); + CERROR("Can't forward [%p] to %s: not a peer\n", + fwd, libcfs_nid2str(nid)); rc = -EHOSTUNREACH; goto out; } @@ -1363,7 +1364,8 @@ kqswnal_fwd_packet (ptl_ni_t *ni, kpr_fwd_desc_t *fwd) rc = kqswnal_launch (ktx); out: if (rc != 0) { - CERROR ("Failed to forward [%p] to "LPX64": %d\n", fwd, nid, rc); + CERROR ("Failed to forward [%p] to %s: %d\n", + fwd, libcfs_nid2str(nid), rc); /* complete now (with failure) */ kqswnal_tx_done (ktx, rc); @@ -1383,8 +1385,9 @@ kqswnal_fwd_callback (ptl_ni_t *ni, void *arg, int error) { ptl_hdr_t *hdr = (ptl_hdr_t *)page_address (krx->krx_kiov[0].kiov_page); - CERROR("Failed to route packet from "LPX64" to "LPX64": %d\n", - le64_to_cpu(hdr->src_nid), le64_to_cpu(hdr->dest_nid),error); + CERROR("Failed to route packet from %s to %s: %d\n", + libcfs_nid2str(le64_to_cpu(hdr->src_nid)), + libcfs_nid2str(le64_to_cpu(hdr->dest_nid)), error); } LASSERT (atomic_read(&krx->krx_refcount) == 1); @@ -1512,8 +1515,9 @@ kqswnal_parse (kqswnal_rx_t *krx) if (kqswnal_nid2elanid (dest_nid) >= 0) /* should have gone direct to peer */ { - CERROR("dropping packet from "LPX64" for "LPX64 - ": target is peer\n", le64_to_cpu(hdr->src_nid), dest_nid); + CERROR("dropping packet from %s for %s: target is peer\n", + libcfs_nid2str(le64_to_cpu(hdr->src_nid)), + libcfs_nid2str(dest_nid)); kqswnal_rx_decref (krx); return; @@ -1610,10 +1614,11 @@ kqswnal_csum_error (kqswnal_rx_t *krx, int ishdr) { ptl_hdr_t *hdr = (ptl_hdr_t *)page_address (krx->krx_kiov[0].kiov_page); - CERROR ("%s checksum mismatch %p: dnid "LPX64", snid "LPX64 - ", dpid %d, spid %d, type %d\n", + CERROR ("%s checksum mismatch %p: dnid %s, snid %s, " + "dpid %d, spid %d, type %d\n", ishdr ? "Header" : "Payload", krx, - le64_to_cpu(hdr->dest_nid), le64_to_cpu(hdr->src_nid) + libcfs_nid2str(le64_to_cpu(hdr->dest_nid)), + libcfs_nid2str(le64_to_cpu(hdr->src_nid)), le32_to_cpu(hdr->dest_pid), le32_to_cpu(hdr->src_pid), le32_to_cpu(hdr->type)); @@ -1920,8 +1925,8 @@ kqswnal_scheduler (void *arg) rc = kqswnal_launch (ktx); if (rc != 0) { - CERROR("Failed delayed transmit to "LPX64 - ": %d\n", ktx->ktx_nid, rc); + CERROR("Failed delayed transmit to %s: %d\n", + libcfs_nid2str(ktx->ktx_nid), rc); kqswnal_tx_done (ktx, rc); } atomic_dec (&kqswnal_data.kqn_pending_txs); diff --git a/lnet/klnds/socklnd/socklnd.c b/lnet/klnds/socklnd/socklnd.c index 0afe554..46f31d9 100644 --- a/lnet/klnds/socklnd/socklnd.c +++ b/lnet/klnds/socklnd/socklnd.c @@ -51,8 +51,8 @@ ksocknal_set_mynid(ptl_nid_t nid) * are coming from. This is not a very graceful solution to this * problem. */ - CDEBUG(D_IOCTL, "setting mynid to "LPX64" (old nid="LPX64")\n", - nid, ni->ni_nid); + CDEBUG(D_IOCTL, "setting mynid to %s (old nid=%s)\n", + libcfs_nid2str(nid), libcfs_nid2str(ni->ni_nid)); LASSERT (PTL_NIDNET(nid) == PTL_NIDNET(ni->ni_nid)); ni->ni_nid = nid; @@ -139,7 +139,8 @@ ksocknal_create_peer (ptl_nid_t nid) void ksocknal_destroy_peer (ksock_peer_t *peer) { - CDEBUG (D_NET, "peer "LPX64" %p deleted\n", peer->ksnp_nid, peer); + CDEBUG (D_NET, "peer %s %p deleted\n", + libcfs_nid2str(peer->ksnp_nid), peer); LASSERT (atomic_read (&peer->ksnp_refcount) == 0); LASSERT (list_empty (&peer->ksnp_conns)); @@ -171,8 +172,9 @@ ksocknal_find_peer_locked (ptl_nid_t nid) if (peer->ksnp_nid != nid) continue; - CDEBUG(D_NET, "got peer [%p] -> "LPX64" (%d)\n", - peer, nid, atomic_read (&peer->ksnp_refcount)); + CDEBUG(D_NET, "got peer [%p] -> %s (%d)\n", + peer, libcfs_nid2str(nid), + atomic_read(&peer->ksnp_refcount)); return (peer); } return (NULL); @@ -299,14 +301,14 @@ ksocknal_associate_route_conn_locked(ksock_route_t *route, ksock_conn_t *conn) if (route->ksnr_myipaddr != conn->ksnc_myipaddr) { if (route->ksnr_myipaddr == 0) { /* route wasn't bound locally yet (the initial route) */ - CWARN("Binding "LPX64" %u.%u.%u.%u to %u.%u.%u.%u\n", - peer->ksnp_nid, + CWARN("Binding %s %u.%u.%u.%u to %u.%u.%u.%u\n", + libcfs_nid2str(peer->ksnp_nid), HIPQUAD(route->ksnr_ipaddr), HIPQUAD(conn->ksnc_myipaddr)); } else { - CWARN("Rebinding "LPX64" %u.%u.%u.%u from " + CWARN("Rebinding %s %u.%u.%u.%u from " "%u.%u.%u.%u to %u.%u.%u.%u\n", - peer->ksnp_nid, + libcfs_nid2str(peer->ksnp_nid), HIPQUAD(route->ksnr_ipaddr), HIPQUAD(route->ksnr_myipaddr), HIPQUAD(conn->ksnc_myipaddr)); @@ -348,8 +350,9 @@ ksocknal_add_route_locked (ksock_peer_t *peer, ksock_route_t *route) route2 = list_entry(tmp, ksock_route_t, ksnr_list); if (route2->ksnr_ipaddr == route->ksnr_ipaddr) { - CERROR ("Duplicate route "LPX64" %u.%u.%u.%u\n", - peer->ksnp_nid, HIPQUAD(route->ksnr_ipaddr)); + CERROR ("Duplicate route %s %u.%u.%u.%u\n", + libcfs_nid2str(peer->ksnp_nid), + HIPQUAD(route->ksnr_ipaddr)); LBUG(); } } @@ -1136,8 +1139,8 @@ ksocknal_create_conn (ksock_route_t *route, struct socket *sock, int type) * code below probably isn't going to work. */ if (route != NULL && route->ksnr_ipaddr != conn->ksnc_ipaddr) { - CERROR("Route "LPX64" %u.%u.%u.%u connected to %u.%u.%u.%u\n", - peer->ksnp_nid, + CERROR("Route %s %u.%u.%u.%u connected to %u.%u.%u.%u\n", + libcfs_nid2str(peer->ksnp_nid), HIPQUAD(route->ksnr_ipaddr), HIPQUAD(conn->ksnc_ipaddr)); } @@ -1189,8 +1192,8 @@ ksocknal_create_conn (ksock_route_t *route, struct socket *sock, int type) rc = ksocknal_close_stale_conns_locked(peer, incarnation); if (rc != 0) - CERROR ("Closed %d stale conns to nid "LPX64" ip %d.%d.%d.%d\n", - rc, conn->ksnc_peer->ksnp_nid, + CERROR ("Closed %d stale conns to nid %s ip %d.%d.%d.%d\n", + rc, libcfs_nid2str(conn->ksnc_peer->ksnp_nid), HIPQUAD(conn->ksnc_ipaddr)); write_unlock_irqrestore (global_lock, flags); @@ -1203,9 +1206,9 @@ ksocknal_create_conn (ksock_route_t *route, struct socket *sock, int type) ksocknal_connsock_decref(conn); } - CWARN("New conn nid:"LPX64" %u.%u.%u.%u -> %u.%u.%u.%u/%d" - " incarnation:"LPX64" sched[%d]/%d\n", - nid, HIPQUAD(conn->ksnc_myipaddr), + CWARN("New conn %s %u.%u.%u.%u -> %u.%u.%u.%u/%d" + " incarnation:"LPD64" sched[%d]/%d\n", + libcfs_nid2str(nid), HIPQUAD(conn->ksnc_myipaddr), HIPQUAD(conn->ksnc_ipaddr), conn->ksnc_port, incarnation, (int)(conn->ksnc_scheduler - ksocknal_data.ksnd_schedulers), irq); @@ -1399,9 +1402,9 @@ ksocknal_destroy_conn (ksock_conn_t *conn) /* complete current receive if any */ switch (conn->ksnc_rx_state) { case SOCKNAL_RX_BODY: - CERROR("Completing partial receive from "LPX64 + CERROR("Completing partial receive from %s" ", ip %d.%d.%d.%d:%d, with error\n", - conn->ksnc_peer->ksnp_nid, + libcfs_nid2str(conn->ksnc_peer->ksnp_nid), HIPQUAD(conn->ksnc_ipaddr), conn->ksnc_port); ptl_finalize (ksocknal_data.ksnd_ni, NULL, conn->ksnc_cookie, PTL_FAIL); @@ -1459,9 +1462,10 @@ ksocknal_close_stale_conns_locked (ksock_peer_t *peer, __u64 incarnation) if (conn->ksnc_incarnation == incarnation) continue; - CWARN("Closing stale conn nid:"LPX64" ip:%08x/%d " - "incarnation:"LPX64"("LPX64")\n", - peer->ksnp_nid, conn->ksnc_ipaddr, conn->ksnc_port, + CWARN("Closing stale conn %s ip:%08x/%d " + "incarnation:"LPD64"("LPD64")\n", + libcfs_nid2str(peer->ksnp_nid), + conn->ksnc_ipaddr, conn->ksnc_port, conn->ksnc_incarnation, incarnation); count++; @@ -1536,7 +1540,8 @@ ksocknal_notify (ptl_ni_t *ni, ptl_nid_t gw_nid, int alive) /* The router is telling me she's been notified of a change in * gateway state.... */ - CDEBUG (D_NET, "gw "LPX64" %s\n", gw_nid, alive ? "up" : "down"); + CDEBUG (D_NET, "gw %s %s\n", libcfs_nid2str(gw_nid), + alive ? "up" : "down"); if (!alive) { /* If the gateway crashed, close all open connections... */ @@ -2166,8 +2171,8 @@ ksocknal_startup (ptl_ni_t *ni) return (rc); } - if (kpr_routing()) { - /* Only allocate forwarding buffers if there's a router */ + if (kpr_forwarding()) { + /* Only allocate forwarding buffers if we're a gateway */ for (i = 0; i < (SOCKNAL_SMALL_FWD_NMSGS + SOCKNAL_LARGE_FWD_NMSGS); i++) { @@ -2215,7 +2220,7 @@ ksocknal_startup (ptl_ni_t *ni) ksocknal_data.ksnd_init = SOCKNAL_INIT_ALL; printk(KERN_INFO "Lustre: Routing socket NAL loaded " - "(initial mem %d, incarnation "LPX64")\n", + "(initial mem %d, incarnation "LPD64")\n", pkmem, ksocknal_data.ksnd_incarnation); return (0); diff --git a/lnet/klnds/socklnd/socklnd_cb.c b/lnet/klnds/socklnd/socklnd_cb.c index 627b0c1..8beaa7c 100644 --- a/lnet/klnds/socklnd/socklnd_cb.c +++ b/lnet/klnds/socklnd/socklnd_cb.c @@ -480,9 +480,9 @@ ksocknal_process_transmit (ksock_conn_t *conn, ksock_tx_t *tx) HIPQUAD(conn->ksnc_ipaddr), rc); break; } - CERROR("[%p] Error %d on write to "LPX64 + CERROR("[%p] Error %d on write to %s" " ip %d.%d.%d.%d:%d\n", conn, rc, - conn->ksnc_peer->ksnp_nid, + libcfs_nid2str(conn->ksnc_peer->ksnp_nid), HIPQUAD(conn->ksnc_ipaddr), conn->ksnc_port); } @@ -591,8 +591,8 @@ ksocknal_queue_tx_locked (ksock_tx_t *tx, ksock_conn_t *conn) LASSERT(!conn->ksnc_closing); LASSERT(tx->tx_resid == tx->tx_nob); - CDEBUG (D_NET, "Sending to "LPX64" ip %d.%d.%d.%d:%d\n", - conn->ksnc_peer->ksnp_nid, + CDEBUG (D_NET, "Sending to %s ip %d.%d.%d.%d:%d\n", + libcfs_nid2str(conn->ksnc_peer->ksnp_nid), HIPQUAD(conn->ksnc_ipaddr), conn->ksnc_port); @@ -718,8 +718,8 @@ ksocknal_launch_packet (ksock_tx_t *tx, ptl_nid_t nid, int routing) rc = ksocknal_add_peer(nid, (__u32)nid, *ksocknal_tunables.ksnd_port); if (rc != 0) { - CERROR("Can't add peer "LPX64": %d\n", - nid, rc); + CERROR("Can't add peer %s: %d\n", + libcfs_nid2str(nid), rc); return rc; } } @@ -751,7 +751,7 @@ ksocknal_launch_packet (ksock_tx_t *tx, ptl_nid_t nid, int routing) write_unlock_irqrestore(g_lock, flags); if (create_peer) { - CERROR("Can't find peer "LPX64"\n", nid); + CERROR("Can't find peer %s\n", libcfs_nid2str(nid)); return -ENOENT; } @@ -787,7 +787,7 @@ ksocknal_launch_packet (ksock_tx_t *tx, ptl_nid_t nid, int routing) write_unlock_irqrestore (g_lock, flags); - CERROR("Peer entry with no routes: "LPX64"\n", nid); + CERROR("Peer entry with no routes: %s\n", libcfs_nid2str(nid)); return (-EHOSTUNREACH); } @@ -920,8 +920,9 @@ ksocknal_fwd_packet (ptl_ni_t *ni, kpr_fwd_desc_t *fwd) int routing; int rc; - CDEBUG (D_NET, "Forwarding [%p] -> "LPX64" ("LPX64"))\n", fwd, - fwd->kprfd_gateway_nid, fwd->kprfd_target_nid); + CDEBUG (D_NET, "Forwarding [%p] -> %s (%s))\n", fwd, + libcfs_nid2str(fwd->kprfd_gateway_nid), + libcfs_nid2str(fwd->kprfd_target_nid)); routing = (nid != ksocknal_data.ksnd_ni->ni_nid); if (!routing) @@ -979,16 +980,14 @@ ksocknal_fmb_callback (ptl_ni_t *ni, void *arg, int error) unsigned long flags; if (error != 0) - CERROR("Failed to route packet from " - LPX64" %s to "LPX64" %s: %d\n", - le64_to_cpu(hdr->src_nid), + CERROR("Failed to route packet from %s to %s: %d\n", libcfs_nid2str(le64_to_cpu(hdr->src_nid)), - le64_to_cpu(hdr->dest_nid), libcfs_nid2str(le64_to_cpu(hdr->dest_nid)), error); else - CDEBUG (D_NET, "routed packet from "LPX64" to "LPX64": OK\n", - le64_to_cpu(hdr->src_nid), le64_to_cpu(hdr->dest_nid)); + CDEBUG (D_NET, "routed packet from %s to %s: OK\n", + libcfs_nid2str(le64_to_cpu(hdr->src_nid)), + libcfs_nid2str(le64_to_cpu(hdr->dest_nid))); /* drop peer ref taken on init */ ksocknal_peer_decref(fmb->fmb_peer); @@ -1103,8 +1102,9 @@ ksocknal_init_fmb (ksock_conn_t *conn, ksock_fmb_t *fmb) ksocknal_fmb_callback, fmb); if (payload_nob == 0) { /* got complete packet already */ - CDEBUG (D_NET, "%p "LPX64"->"LPX64" fwd_start (immediate)\n", - conn, le64_to_cpu(conn->ksnc_hdr.src_nid), dest_nid); + CDEBUG (D_NET, "%p %s->%s fwd_start (immediate)\n", conn, + libcfs_nid2str(le64_to_cpu(conn->ksnc_hdr.src_nid)), + libcfs_nid2str(dest_nid)); kpr_fwd_start (ksocknal_data.ksnd_ni, &fmb->fmb_fwd); @@ -1124,8 +1124,9 @@ ksocknal_init_fmb (ksock_conn_t *conn, ksock_fmb_t *fmb) conn->ksnc_rx_kiov = conn->ksnc_rx_iov_space.kiov; memcpy(conn->ksnc_rx_kiov, fmb->fmb_kiov, niov * sizeof(ptl_kiov_t)); - CDEBUG (D_NET, "%p "LPX64"->"LPX64" %d reading body\n", conn, - le64_to_cpu(conn->ksnc_hdr.src_nid), dest_nid, payload_nob); + CDEBUG (D_NET, "%p %s->%s %d reading body\n", conn, + libcfs_nid2str(le64_to_cpu(conn->ksnc_hdr.src_nid)), + libcfs_nid2str(dest_nid), payload_nob); return (0); } @@ -1137,17 +1138,18 @@ ksocknal_fwd_parse (ksock_conn_t *conn) ptl_nid_t src_nid = le64_to_cpu(conn->ksnc_hdr.src_nid); int body_len = le32_to_cpu(conn->ksnc_hdr.payload_length); - CDEBUG (D_NET, "%p "LPX64"->"LPX64" %d parsing header\n", conn, - src_nid, dest_nid, conn->ksnc_rx_nob_left); + CDEBUG (D_NET, "%p %s->%s %d parsing header\n", conn, + libcfs_nid2str(src_nid), + libcfs_nid2str(dest_nid), conn->ksnc_rx_nob_left); LASSERT (conn->ksnc_rx_state == SOCKNAL_RX_HEADER); LASSERT (conn->ksnc_rx_scheduled); if (body_len < 0) { /* length corrupt (overflow) */ - CERROR("dropping packet from "LPX64" (%s) for "LPX64" (%s): " + CERROR("dropping packet from %s for %s: " "packet size %d illegal\n", - src_nid, libcfs_nid2str(src_nid), - dest_nid, libcfs_nid2str(dest_nid), + libcfs_nid2str(src_nid), + libcfs_nid2str(dest_nid), body_len); ksocknal_new_packet (conn, 0); /* on to new packet */ @@ -1155,23 +1157,33 @@ ksocknal_fwd_parse (ksock_conn_t *conn) } if (body_len > PTL_MTU) { /* too big to forward */ - CERROR ("dropping packet from "LPX64" (%s) for "LPX64 - "(%s): packet size %d too big\n", - src_nid, libcfs_nid2str(src_nid), - dest_nid, libcfs_nid2str(dest_nid), + CERROR ("dropping packet from %s for %s: " + "packet size %d too big\n", + libcfs_nid2str(src_nid), + libcfs_nid2str(dest_nid), body_len); /* on to new packet (skip this one's body) */ ksocknal_new_packet (conn, body_len); return; } + if (!kpr_forwarding()) { + CERROR("dropping packet from %s for %s: " + "I'm not a router\n", + libcfs_nid2str(src_nid), + libcfs_nid2str(dest_nid)); + + ksocknal_new_packet (conn, body_len); /* on to new packet */ + return; + } + /* should have gone direct */ peer = ksocknal_find_peer (conn->ksnc_hdr.dest_nid); if (peer != NULL) { - CERROR ("dropping packet from "LPX64" (%s) for "LPX64 - "(%s): target is a peer\n", - src_nid, libcfs_nid2str(src_nid), - dest_nid, libcfs_nid2str(dest_nid)); + CERROR ("dropping packet from %s for %s: " + "target is a peer\n", + libcfs_nid2str(src_nid), + libcfs_nid2str(dest_nid)); ksocknal_peer_decref(peer); /* drop ref from get above */ /* on to next packet (skip this one's body) */ @@ -1278,14 +1290,16 @@ ksocknal_process_receive (ksock_conn_t *conn) LASSERT (rc != -EAGAIN); if (rc == 0) - CWARN ("[%p] EOF from "LPX64" ip %d.%d.%d.%d:%d\n", - conn, conn->ksnc_peer->ksnp_nid, + CWARN ("[%p] EOF from %s ip %d.%d.%d.%d:%d\n", + conn, + libcfs_nid2str(conn->ksnc_peer->ksnp_nid), HIPQUAD(conn->ksnc_ipaddr), conn->ksnc_port); else if (!conn->ksnc_closing) - CERROR ("[%p] Error %d on read from "LPX64 + CERROR ("[%p] Error %d on read from %s" " ip %d.%d.%d.%d:%d\n", - conn, rc, conn->ksnc_peer->ksnp_nid, + conn, rc, + libcfs_nid2str(conn->ksnc_peer->ksnp_nid), HIPQUAD(conn->ksnc_ipaddr), conn->ksnc_port); @@ -1346,9 +1360,9 @@ ksocknal_process_receive (ksock_conn_t *conn) case SOCKNAL_RX_BODY_FWD: /* payload all received */ - CDEBUG (D_NET, "%p "LPX64"->"LPX64" %d fwd_start (got body)\n", - conn, le64_to_cpu(conn->ksnc_hdr.src_nid), - le64_to_cpu(conn->ksnc_hdr.dest_nid), + CDEBUG (D_NET, "%p %s->%s %d fwd_start (got body)\n", conn, + libcfs_nid2str(le64_to_cpu(conn->ksnc_hdr.src_nid)), + libcfs_nid2str(le64_to_cpu(conn->ksnc_hdr.dest_nid)), conn->ksnc_rx_nob_left); /* forward the packet. NB ksocknal_init_fmb() put fmb into @@ -1815,18 +1829,18 @@ ksocknal_recv_hello (ksock_conn_t *conn, ptl_nid_t *nid, if (*nid == PTL_NID_ANY) { /* don't know peer's nid yet */ *nid = le64_to_cpu(hdr.src_nid); } else if (*nid != le64_to_cpu (hdr.src_nid)) { - LCONSOLE_ERROR("Connected successfully to nid "LPX64" on host " - "%u.%u.%u.%u, but they claimed they were nid " - LPX64" (%s); please check your Lustre " + LCONSOLE_ERROR("Connected successfully to nid %s on host " + "%u.%u.%u.%u, but they claimed they were " + "nid %s; please check your Lustre " "configuration.\n", - *nid, HIPQUAD(conn->ksnc_ipaddr), - le64_to_cpu(hdr.src_nid), + libcfs_nid2str(*nid), HIPQUAD(conn->ksnc_ipaddr), libcfs_nid2str(le64_to_cpu(hdr.src_nid))); - CERROR ("Connected to nid "LPX64"@%u.%u.%u.%u " - "but expecting "LPX64"\n", - le64_to_cpu (hdr.src_nid), - HIPQUAD(conn->ksnc_ipaddr), *nid); + CERROR ("Connected to nid %s ip %u.%u.%u.%u " + "but expecting %s\n", + libcfs_nid2str(le64_to_cpu (hdr.src_nid)), + HIPQUAD(conn->ksnc_ipaddr), + libcfs_nid2str(*nid)); return (-EPROTO); } @@ -1836,13 +1850,15 @@ ksocknal_recv_hello (ksock_conn_t *conn, ptl_nid_t *nid, /* I've accepted this connection; peer determines type */ conn->ksnc_type = ksocknal_invert_type(type); if (conn->ksnc_type == SOCKNAL_CONN_NONE) { - CERROR ("Unexpected type %d from "LPX64"@%u.%u.%u.%u\n", - type, *nid, HIPQUAD(conn->ksnc_ipaddr)); + CERROR ("Unexpected type %d from %s ip %u.%u.%u.%u\n", + type, libcfs_nid2str(*nid), + HIPQUAD(conn->ksnc_ipaddr)); return (-EPROTO); } } else if (ksocknal_invert_type(type) != conn->ksnc_type) { - CERROR ("Mismatched types: me %d, "LPX64"@%u.%u.%u.%u %d\n", - conn->ksnc_type, *nid, HIPQUAD(conn->ksnc_ipaddr), + CERROR ("Mismatched types: me %d, %s ip %u.%u.%u.%u %d\n", + conn->ksnc_type, libcfs_nid2str(*nid), + HIPQUAD(conn->ksnc_ipaddr), le32_to_cpu(hdr.msg.hello.type)); return (-EPROTO); } @@ -1853,9 +1869,9 @@ ksocknal_recv_hello (ksock_conn_t *conn, ptl_nid_t *nid, if (nips > SOCKNAL_MAX_INTERFACES || nips * sizeof(__u32) != __le32_to_cpu (hdr.payload_length)) { - CERROR("Bad payload length %d from "LPX64"@%u.%u.%u.%u\n", + CERROR("Bad payload length %d from %s ip %u.%u.%u.%u\n", __le32_to_cpu (hdr.payload_length), - *nid, HIPQUAD(conn->ksnc_ipaddr)); + libcfs_nid2str(*nid), HIPQUAD(conn->ksnc_ipaddr)); } if (nips == 0) @@ -1863,8 +1879,8 @@ ksocknal_recv_hello (ksock_conn_t *conn, ptl_nid_t *nid, rc = ksocknal_lib_sock_read (sock, ipaddrs, nips * sizeof(*ipaddrs)); if (rc != 0) { - CERROR ("Error %d reading IPs from "LPX64"@%u.%u.%u.%u\n", - rc, *nid, HIPQUAD(conn->ksnc_ipaddr)); + CERROR ("Error %d reading IPs from %s ip %u.%u.%u.%u\n", + rc, libcfs_nid2str(*nid), HIPQUAD(conn->ksnc_ipaddr)); return (rc); } @@ -1872,8 +1888,8 @@ ksocknal_recv_hello (ksock_conn_t *conn, ptl_nid_t *nid, ipaddrs[i] = __le32_to_cpu(ipaddrs[i]); if (ipaddrs[i] == 0) { - CERROR("Zero IP[%d] from "LPX64"@%u.%u.%u.%u\n", - i, *nid, HIPQUAD(conn->ksnc_ipaddr)); + CERROR("Zero IP[%d] from %s ip %u.%u.%u.%u\n", + i, libcfs_nid2str(*nid), HIPQUAD(conn->ksnc_ipaddr)); return (-EPROTO); } } @@ -2035,13 +2051,11 @@ ksocknal_connect (ksock_route_t *route) while (!list_empty (&zombies)) { tx = list_entry (zombies.next, ksock_tx_t, tx_list); - CERROR ("Deleting packet type %d len %d ("LPX64" %s->"LPX64" %s)\n", + CERROR ("Deleting packet type %d len %d %s->%s\n", le32_to_cpu (tx->tx_hdr->type), le32_to_cpu (tx->tx_hdr->payload_length), - le64_to_cpu (tx->tx_hdr->src_nid), libcfs_nid2str(le64_to_cpu(tx->tx_hdr->src_nid)), - le64_to_cpu (tx->tx_hdr->dest_nid), - libcfs_nid2str(le64_to_cpu(tx->tx_hdr->src_nid))); + libcfs_nid2str(le64_to_cpu (tx->tx_hdr->dest_nid))); list_del (&tx->tx_list); /* complete now */ @@ -2161,8 +2175,9 @@ ksocknal_find_timed_out_conn (ksock_peer_t *peer) } /* Something (e.g. failed keepalive) set the socket error */ - CERROR ("Socket error %d: "LPX64" %p %d.%d.%d.%d\n", - SOCK_ERROR(conn->ksnc_sock), peer->ksnp_nid, + CERROR ("Socket error %d: %s %p %d.%d.%d.%d\n", + SOCK_ERROR(conn->ksnc_sock), + libcfs_nid2str(peer->ksnp_nid), conn, HIPQUAD(conn->ksnc_ipaddr)); return (conn); @@ -2177,8 +2192,9 @@ ksocknal_find_timed_out_conn (ksock_peer_t *peer) "%u.%u.%u.%u; the network or that node " "may be down.\n", HIPQUAD(conn->ksnc_ipaddr)); - CERROR ("Timed out RX from "LPX64" %p %d.%d.%d.%d\n", - peer->ksnp_nid,conn,HIPQUAD(conn->ksnc_ipaddr)); + CERROR ("Timed out RX from %s %p %d.%d.%d.%d\n", + libcfs_nid2str(peer->ksnp_nid), + conn, HIPQUAD(conn->ksnc_ipaddr)); return (conn); } @@ -2193,8 +2209,8 @@ ksocknal_find_timed_out_conn (ksock_peer_t *peer) "%u.%u.%u.%u; the network or that node " "may be down.\n", HIPQUAD(conn->ksnc_ipaddr)); - CERROR ("Timed out TX to "LPX64" %s%d %p %d.%d.%d.%d\n", - peer->ksnp_nid, + CERROR ("Timed out TX to %s %s%d %p %d.%d.%d.%d\n", + libcfs_nid2str(peer->ksnp_nid), list_empty (&conn->ksnc_tx_queue) ? "" : "Q ", SOCK_WMEM_QUEUED(conn->ksnc_sock), conn, HIPQUAD(conn->ksnc_ipaddr)); @@ -2226,8 +2242,8 @@ ksocknal_check_peer_timeouts (int idx) if (conn != NULL) { read_unlock (&ksocknal_data.ksnd_global_lock); - CERROR ("Timeout out conn->"LPX64" ip %d.%d.%d.%d:%d\n", - peer->ksnp_nid, + CERROR ("Timeout out conn->%s ip %d.%d.%d.%d:%d\n", + libcfs_nid2str(peer->ksnp_nid), HIPQUAD(conn->ksnc_ipaddr), conn->ksnc_port); ksocknal_close_conn_and_siblings (conn, -ETIMEDOUT); diff --git a/lnet/lnet/Makefile.in b/lnet/lnet/Makefile.in index 3c00d77..e298027 100644 --- a/lnet/lnet/Makefile.in +++ b/lnet/lnet/Makefile.in @@ -3,7 +3,7 @@ MODULES := portals router_objs := $(ROUTER)r_router.o router_objs += $(ROUTER)r_proc.o -portals-objs := api-errno.o api-ni.o +portals-objs := api-errno.o api-ni.o config.o portals-objs += lib-me.o lib-msg.o lib-eq.o lib-md.o portals-objs += lib-move.o module.o portals-objs += $(router_objs) diff --git a/lnet/lnet/api-ni.c b/lnet/lnet/api-ni.c index 5b53f54..f31f2ca 100644 --- a/lnet/lnet/api-ni.c +++ b/lnet/lnet/api-ni.c @@ -22,19 +22,19 @@ #define DEBUG_SUBSYSTEM S_PORTALS #include -ptl_apini_t ptl_apini; /* THE network interface (at the API) */ - #define DEFAULT_NETWORKS "tcp" static char *networks = DEFAULT_NETWORKS; CFS_MODULE_PARM(networks, "s", charp, 0444, "local networks (default='"DEFAULT_NETWORKS"')"); +ptl_apini_t ptl_apini; /* THE network interface (at the API) */ + void ptl_assert_wire_constants (void) { /* Wire protocol assertions generated by 'wirecheck' - * running on Linux mdevi 2.4.21-p4smp-55chaos #1 SMP Tue Jun 8 14:38:44 PDT 2004 i686 i686 i - * with gcc version 3.2.3 20030502 (Red Hat Linux 3.2.3-34) */ - + * running on Linux robert.bartonsoftware.com 2.6.8-1.521 + * #1 Mon Aug 16 09:01:18 EDT 2004 i686 athlon i386 GNU/Linux + * with gcc version 3.3.3 20040412 (Red Hat Linux 3.3.3-7) */ /* Constants... */ CLASSERT (PORTALS_PROTO_MAGIC == 0xeebc0ded); @@ -400,14 +400,6 @@ ptl_apini_init(ptl_pid_t requested_pid, LASSERT (ptl_apini.apini_refcount == 0); - ptl_apini.apini_net_tokens_nob = strlen(networks) + 1; - PORTAL_ALLOC(ptl_apini.apini_net_tokens, - ptl_apini.apini_net_tokens_nob); - if (ptl_apini.apini_net_tokens == NULL) { - CERROR("Can't allocate net tokens\n"); - goto out; - } - ptl_apini.apini_pid = requested_pid; rc = ptl_descriptor_setup (requested_limits, @@ -465,9 +457,6 @@ ptl_apini_init(ptl_pid_t requested_pid, if (rc != PTL_OK) { ptl_cleanup_handle_hash (); ptl_descriptor_cleanup (); - if (ptl_apini.apini_net_tokens != NULL) - PORTAL_FREE(ptl_apini.apini_net_tokens, - ptl_apini.apini_net_tokens_nob); } RETURN (rc); @@ -533,7 +522,6 @@ ptl_apini_fini (void) ptl_cleanup_handle_hash (); ptl_descriptor_cleanup (); - PORTAL_FREE(ptl_apini.apini_net_tokens, ptl_apini.apini_net_tokens_nob); #ifndef __KERNEL__ pthread_mutex_destroy(&ptl_apini.apini_mutex); @@ -646,270 +634,6 @@ ptl_shutdown_nalnis (void) PTL_UNLOCK(flags); } -void ptl_syntax(char *name, char *str, int offset, int width) -{ - const char *dots = "................................" - "................................" - "................................" - "................................" - "................................" - "................................" - "................................" - "................................"; - const char *dashes = "--------------------------------" - "--------------------------------" - "--------------------------------" - "--------------------------------" - "--------------------------------" - "--------------------------------" - "--------------------------------" - "--------------------------------"; - - LCONSOLE_ERROR("Error parsing '%s=\"%s\"'\n", name, str); - LCONSOLE_ERROR("here...........%.*s..%.*s|%.*s|\n", - strlen(name), dots, offset, dots, - (width < 1) ? 0 : width - 1, dashes); -} - -int -ptl_nis_conflict(ptl_ni_t *ni1, ptl_ni_t *ni2) -{ - int i; - int j; - - if (PTL_NETNAL(PTL_NIDNET(ni1->ni_nid)) != /* different NALs */ - PTL_NETNAL(PTL_NIDNET(ni2->ni_nid))) - return 0; - - if (ni1 != ni2 && - PTL_NIDNET(ni1->ni_nid) == PTL_NIDNET(ni2->ni_nid)) { - CERROR("Duplicate network: %s\n", - libcfs_net2str(PTL_NIDNET(ni1->ni_nid))); - return 1; - } - - if (ni1->ni_interfaces[0] == NULL || - ni2->ni_interfaces[0] == NULL) { - /* one (or both) using all available interfaces */ - if (ni1 != ni2) { - CERROR("Interface conflict: %s, %s\n", - libcfs_net2str(PTL_NIDNET(ni1->ni_nid)), - libcfs_net2str(PTL_NIDNET(ni2->ni_nid))); - return 1; - } - return 0; - } - - for (i = 0; i < PTL_MAX_INTERFACES; i++) { - if (ni1->ni_interfaces[i] == NULL) - break; - - for (j = 0; j < PTL_MAX_INTERFACES; j++) { - if (ni2->ni_interfaces[j] == NULL) - break; - - if (ni1 == ni2 && i == j) - continue; - - if (strcmp(ni1->ni_interfaces[i], - ni2->ni_interfaces[j])) - continue; - - CERROR("Duplicate interface: %s(%s), %s(%s)\n", - libcfs_net2str(PTL_NIDNET(ni1->ni_nid)), - ni1->ni_interfaces[i], - libcfs_net2str(PTL_NIDNET(ni2->ni_nid)), - ni2->ni_interfaces[i]); - return 1; - } - } - - return 0; -} - -ptl_err_t -ptl_check_ni_conflicts(ptl_ni_t *ni, struct list_head *nilist) -{ - struct list_head *tmp; - ptl_ni_t *ni2; - - /* Yes! ni just added to this list. - * Check its network is unique and its interfaces don't conflict */ - LASSERT (ni == list_entry(nilist->prev, ptl_ni_t, ni_list)); - - list_for_each (tmp, nilist) { - ni2 = list_entry(tmp, ptl_ni_t, ni_list); - - if (ptl_nis_conflict(ni, ni2)) - return PTL_FAIL; - } - - return PTL_OK; -} - -int -ptl_iswhite (char c) -{ - switch (c) { - case ' ': - case '\t': - case '\n': - case '\r': - return 1; - default: - return 0; - } -} - -char * -ptl_trimwhite(char *str) -{ - char *end; - - while (ptl_iswhite(*str)) - str++; - - end = str + strlen(str); - while (end > str) { - if (!ptl_iswhite(end[-1])) - break; - end--; - } - - *end = 0; - return str; -} - -ptl_err_t -ptl_parse_networks(struct list_head *nilist) -{ - char *tokens = ptl_apini.apini_net_tokens; - char *str = tokens; - ptl_ni_t *ni = NULL; - __u32 net; - int rc; - - LASSERT (ptl_apini.apini_net_tokens_nob == strlen(networks) + 1); - memcpy (tokens, networks, ptl_apini.apini_net_tokens_nob); - - while (str != NULL && *str != 0) { - char *comma = strchr(str, ','); - char *bracket = strchr(str, '('); - int niface; - char *iface; - - PORTAL_ALLOC(ni, sizeof(*ni)); - if (ni == NULL) { - CERROR ("ENOMEM parsing 'networks=\"%s\"'\n", networks); - goto failed; - } - /* zero counters/flags, NULL pointers... */ - memset(ni, 0, sizeof(*ni)); - list_add_tail(&ni->ni_list, nilist); - - if (bracket == NULL || - (comma != NULL && comma < bracket)) { - if (comma != NULL) - *comma++ = 0; - net = libcfs_str2net(ptl_trimwhite(str)); - - if (net == PTL_NIDNET(PTL_NID_ANY)) { - ptl_syntax("networks", networks, - str - tokens, strlen(str)); - goto failed; - } - - ni->ni_nid = PTL_MKNID(net, 0); - if (ptl_check_ni_conflicts(ni, nilist) != PTL_OK) - goto failed; - - str = comma; - continue; - } - - *bracket = 0; - net = libcfs_str2net(ptl_trimwhite(str)); - if (net == PTL_NIDNET(PTL_NID_ANY)) { - ptl_syntax("networks", networks, - str - tokens, strlen(str)); - goto failed; - } - - ni->ni_nid = PTL_MKNID(net, 0); - - niface = 0; - iface = bracket + 1; - - bracket = strchr(iface, ')'); - if (bracket == NULL) { - ptl_syntax ("networks", networks, - iface - tokens, strlen(iface)); - goto failed; - } - - *bracket = 0; - do { - comma = strchr(iface, ','); - if (comma != NULL) - *comma++ = 0; - - iface = ptl_trimwhite(iface); - if (*iface == 0) { - ptl_syntax("networks", networks, - iface - tokens, strlen(iface)); - goto failed; - } - - if (niface == PTL_MAX_INTERFACES) { - LCONSOLE_ERROR("Too many interfaces for %s\n", - libcfs_net2str(PTL_NIDNET(ni->ni_nid))); - goto failed; - } - - ni->ni_interfaces[niface++] = iface; - iface = comma; - } while (iface != NULL); - - if (ptl_check_ni_conflicts(ni, nilist) != PTL_OK) - goto failed; - - str = bracket + 1; - comma = strchr(bracket + 1, ','); - if (comma != NULL) { - *comma = 0; - str = ptl_trimwhite(str); - if (*str != 0) { - ptl_syntax ("networks", networks, - str - tokens, strlen(str)); - goto failed; - } - str = comma + 1; - continue; - } - - str = ptl_trimwhite(str); - if (*str != 0) { - ptl_syntax ("networks", networks, - str - tokens, strlen(str)); - goto failed; - } - } - - if (list_empty(nilist)) { - LCONSOLE_ERROR("No networks specified\n"); - goto failed; - } - return PTL_OK; - - failed: - while (!list_empty(nilist)) { - ni = list_entry(nilist->next, ptl_ni_t, ni_list); - - list_del(&ni->ni_list); - PORTAL_FREE(ni, sizeof(*ni)); - } - return PTL_FAIL; -} ptl_err_t ptl_load_nal (int type) @@ -932,7 +656,7 @@ ptl_startup_nalnis (void) int retry; INIT_LIST_HEAD(&nilist); - rc = ptl_parse_networks(&nilist); + rc = ptl_parse_networks(&nilist, networks); if (rc != PTL_OK) return rc; @@ -1076,10 +800,15 @@ PtlNIInit(ptl_interface_t interface, ptl_pid_t requested_pid, if (rc != PTL_OK) goto out; - kpr_initialise(); + rc = kpr_initialise(); + if (rc != 0) { + ptl_apini_fini(); + goto out; + } rc = ptl_startup_nalnis(); if (rc != PTL_OK) { + kpr_finalise(); ptl_apini_fini(); goto out; } diff --git a/lnet/lnet/autoMakefile.am b/lnet/lnet/autoMakefile.am index 2eb95de..a82dfe3 100644 --- a/lnet/lnet/autoMakefile.am +++ b/lnet/lnet/autoMakefile.am @@ -1,4 +1,4 @@ -my_sources = api-errno.c api-ni.c \ +my_sources = api-errno.c api-ni.c config.c \ lib-me.c lib-msg.c lib-eq.c \ lib-md.c lib-move. \ $(top_srcdir)/portals/router/router.c @@ -21,7 +21,7 @@ endif # LINUX if DARWIN macos_PROGRAMS := portals -portals_SOURCES := api-errno.c api-ni.c +portals_SOURCES := api-errno.c api-ni.c config.c portals_SOURCES += lib-me.c lib-msg.c lib-eq.c lib-md.c portals_SOURCES += lib-move.c module.c diff --git a/lnet/lnet/config.c b/lnet/lnet/config.c new file mode 100644 index 0000000..3359f68 --- /dev/null +++ b/lnet/lnet/config.c @@ -0,0 +1,645 @@ +/* -*- mode: c; c-basic-offset: 8; indent-tabs-mode: nil; -*- + * vim:expandtab:shiftwidth=8:tabstop=8: + * + * Copyright (c) 2005 Cluster File Systems, Inc. + * + * This file is part of Lustre, http://www.sf.net/projects/lustre/ + * + * Lustre is free software; you can redistribute it and/or + * modify it under the terms of version 2 of the GNU General Public + * License as published by the Free Software Foundation. + * + * Lustre is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with Lustre; if not, write to the Free Software + * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA. + */ + +#define DEBUG_SUBSYSTEM S_PORTALS +#include + +typedef struct { /* tmp struct for parsing routes */ + struct list_head ptb_list; /* stash on lists */ + int ptb_size; /* allocated size */ + char ptb_text[0]; /* text buffer */ +} ptl_text_buf_t; + +static int ptl_tbnob = 0; /* track text buf allocation */ +#define PTL_MAX_TEXTBUF_NOB (64<<10) /* bound allocation */ + +void +ptl_syntax(char *name, char *str, int offset, int width) +{ + const char *dots = "................................" + "................................" + "................................" + "................................" + "................................" + "................................" + "................................" + "................................"; + const char *dashes = "--------------------------------" + "--------------------------------" + "--------------------------------" + "--------------------------------" + "--------------------------------" + "--------------------------------" + "--------------------------------" + "--------------------------------"; + + LCONSOLE_ERROR("Error parsing '%s=\"%s\"'\n", name, str); + LCONSOLE_ERROR("here...........%.*s..%.*s|%.*s|\n", + strlen(name), dots, offset, dots, + (width < 1) ? 0 : width - 1, dashes); +} + +int +ptl_issep (char c) +{ + switch (c) { + case '\n': + case '\r': + case ';': + return 1; + default: + return 0; + } +} + +int +ptl_iswhite (char c) +{ + switch (c) { + case ' ': + case '\t': + case '\n': + case '\r': + return 1; + default: + return 0; + } +} + +char * +ptl_trimwhite(char *str) +{ + char *end; + + while (ptl_iswhite(*str)) + str++; + + end = str + strlen(str); + while (end > str) { + if (!ptl_iswhite(end[-1])) + break; + end--; + } + + *end = 0; + return str; +} + +int +ptl_nis_conflict(ptl_ni_t *ni1, ptl_ni_t *ni2) +{ + int i; + int j; + + if (PTL_NETNAL(PTL_NIDNET(ni1->ni_nid)) != /* different NALs */ + PTL_NETNAL(PTL_NIDNET(ni2->ni_nid))) + return 0; + + if (ni1 != ni2 && + PTL_NIDNET(ni1->ni_nid) == PTL_NIDNET(ni2->ni_nid)) { + CERROR("Duplicate network: %s\n", + libcfs_net2str(PTL_NIDNET(ni1->ni_nid))); + return 1; + } + + if (ni1->ni_interfaces[0] == NULL || + ni2->ni_interfaces[0] == NULL) { + /* one (or both) using all available interfaces */ + if (ni1 != ni2) { + CERROR("Interface conflict: %s, %s\n", + libcfs_net2str(PTL_NIDNET(ni1->ni_nid)), + libcfs_net2str(PTL_NIDNET(ni2->ni_nid))); + return 1; + } + return 0; + } + + for (i = 0; i < PTL_MAX_INTERFACES; i++) { + if (ni1->ni_interfaces[i] == NULL) + break; + + for (j = 0; j < PTL_MAX_INTERFACES; j++) { + if (ni2->ni_interfaces[j] == NULL) + break; + + if (ni1 == ni2 && i == j) + continue; + + if (strcmp(ni1->ni_interfaces[i], + ni2->ni_interfaces[j])) + continue; + + CERROR("Duplicate interface: %s(%s), %s(%s)\n", + libcfs_net2str(PTL_NIDNET(ni1->ni_nid)), + ni1->ni_interfaces[i], + libcfs_net2str(PTL_NIDNET(ni2->ni_nid)), + ni2->ni_interfaces[i]); + return 1; + } + } + + return 0; +} + +ptl_err_t +ptl_check_ni_conflicts(ptl_ni_t *ni, struct list_head *nilist) +{ + struct list_head *tmp; + ptl_ni_t *ni2; + + /* Yes! ni just added to this list. + * Check its network is unique and its interfaces don't conflict */ + LASSERT (ni == list_entry(nilist->prev, ptl_ni_t, ni_list)); + + list_for_each (tmp, nilist) { + ni2 = list_entry(tmp, ptl_ni_t, ni_list); + + if (ptl_nis_conflict(ni, ni2)) + return PTL_FAIL; + } + + return PTL_OK; +} + +ptl_err_t +ptl_parse_networks(struct list_head *nilist, char *networks) +{ + int tokensize = strlen(networks) + 1; + char *tokens; + char *str; + ptl_ni_t *ni = NULL; + __u32 net; + int rc; + + if (strlen(networks) > PAGE_SIZE) { + /* _WAY_ conservative */ + CERROR("Can't parse networks; string too long\n"); + return PTL_FAIL; + } + + PORTAL_ALLOC(tokens, tokensize); + if (tokens == NULL) { + CERROR("Can't allocate net tokens\n"); + return PTL_FAIL; + } + + memcpy (tokens, networks, tokensize); + str = tokens; + + while (str != NULL && *str != 0) { + char *comma = strchr(str, ','); + char *bracket = strchr(str, '('); + int niface; + char *iface; + + PORTAL_ALLOC(ni, sizeof(*ni)); + if (ni == NULL) { + CERROR ("ENOMEM parsing 'networks=\"%s\"'\n", networks); + goto failed; + } + /* zero counters/flags, NULL pointers... */ + memset(ni, 0, sizeof(*ni)); + list_add_tail(&ni->ni_list, nilist); + + if (bracket == NULL || + (comma != NULL && comma < bracket)) { + if (comma != NULL) + *comma++ = 0; + net = libcfs_str2net(ptl_trimwhite(str)); + + if (net == PTL_NIDNET(PTL_NID_ANY)) { + ptl_syntax("networks", networks, + str - tokens, strlen(str)); + goto failed; + } + + ni->ni_nid = PTL_MKNID(net, 0); + if (ptl_check_ni_conflicts(ni, nilist) != PTL_OK) + goto failed; + + str = comma; + continue; + } + + *bracket = 0; + net = libcfs_str2net(ptl_trimwhite(str)); + if (net == PTL_NIDNET(PTL_NID_ANY)) { + ptl_syntax("networks", networks, + str - tokens, strlen(str)); + goto failed; + } + + ni->ni_nid = PTL_MKNID(net, 0); + + niface = 0; + iface = bracket + 1; + + bracket = strchr(iface, ')'); + if (bracket == NULL) { + ptl_syntax ("networks", networks, + iface - tokens, strlen(iface)); + goto failed; + } + + *bracket = 0; + do { + comma = strchr(iface, ','); + if (comma != NULL) + *comma++ = 0; + + iface = ptl_trimwhite(iface); + if (*iface == 0) { + ptl_syntax("networks", networks, + iface - tokens, strlen(iface)); + goto failed; + } + + if (niface == PTL_MAX_INTERFACES) { + LCONSOLE_ERROR("Too many interfaces for %s\n", + libcfs_net2str(PTL_NIDNET(ni->ni_nid))); + goto failed; + } + + ni->ni_interfaces[niface++] = iface; + iface = comma; + } while (iface != NULL); + + if (ptl_check_ni_conflicts(ni, nilist) != PTL_OK) + goto failed; + + str = bracket + 1; + comma = strchr(bracket + 1, ','); + if (comma != NULL) { + *comma = 0; + str = ptl_trimwhite(str); + if (*str != 0) { + ptl_syntax ("networks", networks, + str - tokens, strlen(str)); + goto failed; + } + str = comma + 1; + continue; + } + + str = ptl_trimwhite(str); + if (*str != 0) { + ptl_syntax ("networks", networks, + str - tokens, strlen(str)); + goto failed; + } + } + + if (list_empty(nilist)) { + LCONSOLE_ERROR("No networks specified\n"); + goto failed; + } + PORTAL_FREE(tokens, tokensize); + return PTL_OK; + + failed: + while (!list_empty(nilist)) { + ni = list_entry(nilist->next, ptl_ni_t, ni_list); + + list_del(&ni->ni_list); + PORTAL_FREE(ni, sizeof(*ni)); + } + PORTAL_FREE(tokens, tokensize); + return PTL_FAIL; +} + +ptl_text_buf_t * +ptl_new_text_buf (int str_len) +{ + ptl_text_buf_t *ptb; + int nob; + + nob = offsetof(ptl_text_buf_t, ptb_text[str_len + 1]); + if (nob > PAGE_SIZE) { + /* _way_ conservative for "route net gateway..." */ + CERROR("text buffer too big\n"); + return NULL; + } + + if (ptl_tbnob + nob > PTL_MAX_TEXTBUF_NOB) { + CERROR("Too many text buffers\n"); + return NULL; + } + + PORTAL_ALLOC(ptb, nob); + if (ptb == NULL) + return NULL; + + ptb->ptb_size = nob; + ptl_tbnob += nob; + return ptb; +} + +void +ptl_free_text_buf (ptl_text_buf_t *ptb) +{ + PORTAL_FREE(ptb, ptb->ptb_size); + ptl_tbnob -= ptb->ptb_size; +} + +void +ptl_free_text_bufs(struct list_head *tbs) +{ + ptl_text_buf_t *ptb; + + while (!list_empty(tbs)) { + ptb = list_entry(tbs->next, ptl_text_buf_t, ptb_list); + + list_del(&ptb->ptb_list); + ptl_free_text_buf(ptb); + } +} + +void +ptl_print_text_bufs(struct list_head *tbs) +{ + struct list_head *tmp; + ptl_text_buf_t *ptb; + + list_for_each (tmp, tbs) { + ptb = list_entry(tmp, ptl_text_buf_t, ptb_list); + + CDEBUG(D_WARNING, "%s\n", ptb->ptb_text); + } + + CDEBUG(D_WARNING, "%d allocated\n", ptl_tbnob); +} + +int +ptl_str2tbs_sep (struct list_head *tbs, char *str) +{ + struct list_head pending; + char *sep; + int nob; + ptl_text_buf_t *ptb; + + INIT_LIST_HEAD(&pending); + + /* Split 'str' into separate commands */ + for (;;) { + /* scan for separator or comment */ + for (sep = str; *sep != 0; sep++) + if (ptl_issep(*sep) || *sep == '#') + break; + + nob = sep - str; + if (nob > 0) { + ptb = ptl_new_text_buf(nob + 1); + if (ptb == NULL) { + ptl_free_text_bufs(&pending); + return -1; + } + + memcpy(ptb->ptb_text, str, nob); + ptb->ptb_text[nob] = 0; + + list_add_tail(&ptb->ptb_list, &pending); + } + + if (*sep == '#') { + /* scan for separator */ + do { + sep++; + } while (*sep != 0 && !ptl_issep(*sep)); + } + + if (*sep == 0) + break; + + str = sep + 1; + } + + list_splice(&pending, tbs->prev); + return 0; +} + +int +ptl_str2tbs_expand (struct list_head *tbs, char *str) +{ + struct list_head pending; + char *sep; + int nob; + ptl_text_buf_t *ptb; + int lo; + int hi; + int stride; + int i; + int scanned; + int scanned2; + + INIT_LIST_HEAD(&pending); + + sep = strchr(str, '['); + if (sep == NULL) /* nothing to expand */ + return 0; + + /* check it's a valid range... */ + nob = sep - str; + + if (sscanf(sep + 1, "%d-%d%n", &lo, &hi, &scanned) < 2) + goto failed; + + if (sep[1 + scanned] != '/') + stride = 1; + else if (sscanf(sep + 1 + scanned + 1, "%d%n", &stride, &scanned2) < 1) + goto failed; + else + scanned += 1 + scanned2; + + if (sep[scanned + 1] != ']') + goto failed; + + if (hi < 0 || lo < 0 || stride < 0 || hi < lo || + (hi - lo) % stride != 0) + goto failed; + + /* ...and expand it */ + for (i = lo; i <= hi; i += stride) { + char num[16]; + ptl_text_buf_t *ptb; + + snprintf(num, sizeof(num), "%d", i); + if (strlen(num) == sizeof(num) - 1) + goto failed; + + ptb = ptl_new_text_buf(nob + strlen(num) + + strlen(sep + 1 + scanned + 1)); + if (ptb == NULL) + goto failed; + + memcpy(ptb->ptb_text, str, sep - str); + strcpy(&ptb->ptb_text[nob], num); + strcat(&ptb->ptb_text[nob], &sep[1 + scanned + 1]); + + list_add_tail(&ptb->ptb_list, &pending); + } + + list_splice(&pending, tbs->prev); + return (hi + 1 - lo); + + failed: + ptl_free_text_bufs(&pending); + return -1; +} + +int +ptl_parse_route (char *str) +{ + /* static scratch buffer OK (single threaded) */ + static char cmd[PAGE_SIZE]; + + struct list_head gateways; + __u32 net = 0; /* avoid a warning */ + ptl_nid_t nid; + ptl_text_buf_t *ptb; + ptl_text_buf_t *tb2; + int rc; + char *sep; + char *token; + int ntokens = 0; + + INIT_LIST_HEAD(&gateways); + + /* save a copy of the string for error messages */ + strncpy(cmd, str, sizeof(cmd) - 1); + cmd[sizeof(cmd) - 1] = 0; + + sep = str; + for (;;) { + /* scan for token start */ + while (ptl_iswhite(*sep)) + sep++; + if (*sep == 0) { + if (ntokens < 3) { + ptl_syntax("routes", cmd, sep - str, -1); + return -1; + } + return 0; + } + + ntokens++; + token = sep++; + + /* scan for token end */ + while (*sep != 0 && !ptl_iswhite(*sep)) + sep++; + if (*sep != 0) + *sep++ = 0; + + if (ntokens == 1) { + if (!strcmp(token, "route")) + continue; + goto token_error; + } + + if (ntokens == 2) { + net = libcfs_str2net(token); + if (net != PTL_NIDNET(PTL_NID_ANY)) + continue; + goto token_error; + } + + ptb = ptl_new_text_buf(strlen(token)); + if (ptb == NULL) { + CERROR ("Error parsing routes\n"); + return -1; + } + + strcpy(ptb->ptb_text, token); + + list_add (&ptb->ptb_list, &gateways); + while (!list_empty(&gateways)) { + ptb = list_entry(gateways.next, + ptl_text_buf_t, ptb_list); + + /* Add ptb's expansions right after it */ + rc = ptl_str2tbs_expand(ptb->ptb_list.next, + ptb->ptb_text); + if (rc < 0) + goto token_error; + + if (rc == 0) { + /* no expansions: check gateway nid */ + nid = libcfs_str2nid(ptb->ptb_text); + if (nid == PTL_NID_ANY) + goto token_error; + + rc = kpr_add_route (net, nid); + if (rc != 0) { + CERROR("Can't create route " + "to %s via %s\n", + libcfs_net2str(net), + libcfs_nid2str(nid)); + goto error; + } + } + + list_del(&ptb->ptb_list); + ptl_free_text_buf(ptb); + } + } + + token_error: + ptl_syntax("routes", cmd, token - str, strlen(token)); + error: + ptl_free_text_bufs(&gateways); + return -1; +} + +ptl_err_t +ptl_parse_routes (char *routes) +{ + struct list_head tbs; + ptl_text_buf_t *ptb; + int rc = PTL_OK; + + INIT_LIST_HEAD(&tbs); + + if (ptl_str2tbs_sep(&tbs, routes) < 0) { + CERROR("Error parsing routes\n"); + rc = PTL_FAIL; + goto out; + } + + /* Parse expanded cmds */ + while (!list_empty(&tbs)) { + ptb = list_entry(tbs.next, ptl_text_buf_t, ptb_list); + + if (ptl_parse_route(ptb->ptb_text) < 0) { + ptl_free_text_bufs(&tbs); + rc = PTL_FAIL; + goto out; + } + + list_del(&ptb->ptb_list); + ptl_free_text_buf(ptb); + } + + out: + LASSERT (ptl_tbnob == 0); + return rc; +} + + diff --git a/lnet/router/proc.c b/lnet/router/proc.c index d44ae07..9bec9c4 100644 --- a/lnet/router/proc.c +++ b/lnet/router/proc.c @@ -22,220 +22,247 @@ */ #include "router.h" +#include #define KPR_PROC_ROUTER "sys/portals/router" #define KPR_PROC_ROUTES "sys/portals/routes" -/* Used for multi-page route list book keeping */ -struct proc_route_data { - struct list_head *curr; - unsigned int generation; - off_t skip; - rwlock_t proc_route_rwlock; -} kpr_read_routes_data; - -/* nal2name support re-used from utils/portals.c */ -struct name2num { - char *name; - int num; -} nalnames[] = { - { "any", 0}, - { "elan", QSWNAL}, - { "tcp", SOCKNAL}, - { "gm", GMNAL}, - { "ib", OPENIBNAL}, - { "iib", IIBNAL}, - { "lo", LONAL}, - { NULL, -1} -}; - -static struct name2num *name2num_lookup_num(struct name2num *table, int num) -{ - while (table->name != NULL) - if (num == table->num) - return (table); - else - table++; - return (NULL); -} - -static char *nal2name(int nal) -{ - struct name2num *e = name2num_lookup_num(nalnames, nal); - return ((e == NULL) ? "???" : e->name); -} - - -static int kpr_proc_router_read(char *page, char **start, off_t off, - int count, int *eof, void *data) +static int +kpr_proc_stats_read (char *page, char **start, off_t off, + int count, int *eof, void *data) { - unsigned long long bytes = kpr_fwd_bytes; - unsigned long packets = kpr_fwd_packets; - unsigned long errors = kpr_fwd_errors; - unsigned int qdepth = atomic_read (&kpr_queue_depth); - int len; + unsigned long long bytes; + unsigned long long packets; + unsigned long long errors; + unsigned int qdepth; + unsigned long flags; + *start = page; *eof = 1; if (off != 0) - return (0); - - len = sprintf(page, "%Ld %ld %ld %d\n", bytes, packets, errors, qdepth); + return 0; + + spin_lock_irqsave(&kpr_state.kpr_stats_lock, flags); - *start = page; - return (len); -} + bytes = kpr_state.kpr_fwd_bytes; + packets = kpr_state.kpr_fwd_packets; + errors = kpr_state.kpr_fwd_errors; + qdepth = atomic_read(&kpr_state.kpr_queue_depth); -static int kpr_proc_router_write(struct file *file, const char *ubuffer, - unsigned long count, void *data) -{ - /* Ignore what we've been asked to write, and just zero the stats */ - kpr_fwd_bytes = 0; - kpr_fwd_packets = 0; - kpr_fwd_errors = 0; + spin_unlock_irqrestore(&kpr_state.kpr_stats_lock, flags); - return (count); + return sprintf(page, "%Ld %Ld %Ld %d\n", bytes, packets, errors, qdepth); } -static int kpr_proc_routes_read(char *page, char **start, off_t off, - int count, int *eof, void *data) +static int +kpr_proc_stats_write(struct file *file, const char *ubuffer, + unsigned long count, void *data) { - struct proc_route_data *prd = data; - kpr_route_entry_t *re; - kpr_gateway_entry_t *ge; - int chunk_len = 0; - int line_len = 0; - int user_len = 0; - int rc = 0; + unsigned long flags; - *eof = 1; - *start = page; + spin_lock_irqsave(&kpr_state.kpr_stats_lock, flags); + + /* just zero the stats */ + kpr_state.kpr_fwd_bytes = 0; + kpr_state.kpr_fwd_packets = 0; + kpr_state.kpr_fwd_errors = 0; - write_lock(&(prd->proc_route_rwlock)); + spin_unlock_irqrestore(&kpr_state.kpr_stats_lock, flags); + return (count); +} - if (prd->curr == NULL) { - if (off != 0) - goto routes_read_exit; +typedef struct { + unsigned long long sri_generation; + kpr_route_entry_t *sri_route; + loff_t sri_off; +} kpr_seq_route_iterator_t; - /* First pass, initialize our private data */ - prd->curr = kpr_routes.next; - prd->generation = kpr_routes_generation; - prd->skip = 0; +int +kpr_seq_routes_seek (kpr_seq_route_iterator_t *sri, loff_t off) +{ + struct list_head *tmp; + int rc; + unsigned long flags; + loff_t here; + + read_lock_irqsave(&kpr_state.kpr_rwlock, flags); + + if (sri->sri_route != NULL && + sri->sri_generation != kpr_state.kpr_generation) { + /* tables have changed */ + rc = -ESTALE; } else { - /* Abort route list generation change */ - if (prd->generation != kpr_routes_generation) { - prd->curr = NULL; - rc = sprintf(page, "\nError: Routes Changed\n"); - goto routes_read_exit; + if (sri->sri_route == NULL || sri->sri_off > off) { + /* search from start */ + tmp = kpr_state.kpr_routes.next; + here = 0; + } else { + /* continue search */ + tmp = &sri->sri_route->kpre_list; + here = sri->sri_off; } - /* All the routes have been walked */ - if (prd->curr == &kpr_routes) { - prd->curr = NULL; - goto routes_read_exit; + sri->sri_generation = kpr_state.kpr_generation; + sri->sri_off = off; + sri->sri_route = NULL; + rc = -ENOENT; + + while (tmp != &kpr_state.kpr_routes) { + if (here == off) { + sri->sri_route = + list_entry(tmp, kpr_route_entry_t, + kpre_list); + rc = 0; + break; + } + tmp = tmp->next; + here++; } - } + } + + read_unlock_irqrestore(&kpr_state.kpr_rwlock, flags); + return rc; +} - read_lock(&kpr_rwlock); - *start = page + prd->skip; - user_len = -prd->skip; - - while ((prd->curr != NULL) && (prd->curr != &kpr_routes)) { - re = list_entry(prd->curr, kpr_route_entry_t, kpre_list); - ge = re->kpre_gateway; - - line_len = sprintf(page + chunk_len, - "net %12s: gateway %s %s\n", - libcfs_net2str(re->kpre_net), - libcfs_nid2str(ge->kpge_nid), - ge->kpge_alive ? "up" : "down"); - chunk_len += line_len; - user_len += line_len; - - /* Abort the route list changed */ - if (prd->curr->next == NULL) { - prd->curr = NULL; - read_unlock(&kpr_rwlock); - rc = sprintf(page, "\nError: Routes Changed\n"); - goto routes_read_exit; - } +static void * +kpr_seq_routes_start (struct seq_file *s, loff_t *pos) +{ + kpr_seq_route_iterator_t *sri; + unsigned long flags; + int rc; + + PORTAL_ALLOC(sri, sizeof(*sri)); + if (sri == NULL) + return NULL; + + sri->sri_route = NULL; + rc = kpr_seq_routes_seek(sri, *pos); + if (rc == 0) + return sri; + + PORTAL_FREE(sri, sizeof(*sri)); + return NULL; +} - prd->curr = prd->curr->next; +static void +kpr_seq_routes_stop (struct seq_file *s, void *iter) +{ + kpr_seq_route_iterator_t *sri = iter; + + if (sri != NULL) + PORTAL_FREE(sri, sizeof(*sri)); +} - /* The route table will exceed one page, break the while loop - * so the function can be re-called with a new page. - */ - if ((chunk_len > (PAGE_SIZE - 80)) || (user_len > count)) - break; +static void * +kpr_seq_routes_next (struct seq_file *s, void *iter, loff_t *pos) +{ + kpr_seq_route_iterator_t *sri = iter; + unsigned long flags; + int rc; + loff_t next = *pos + 1; + + rc = kpr_seq_routes_seek(sri, next); + if (rc != 0) { + PORTAL_FREE(sri, sizeof(*sri)); + return NULL; } + + *pos = next; + return sri; +} + +static int +kpr_seq_routes_show (struct seq_file *s, void *iter) +{ + kpr_seq_route_iterator_t *sri = iter; + unsigned long flags; + __u32 net; + ptl_nid_t nid; + int alive; + int stale; - *eof = 0; - - /* Caller received only a portion of the last entry, the - * remaining will be delivered in the next page if asked for. - */ - if (user_len > count) { - prd->curr = prd->curr->prev; - prd->skip = line_len - (user_len - count); - read_unlock(&kpr_rwlock); - rc = count; - goto routes_read_exit; + read_lock_irqsave(&kpr_state.kpr_rwlock, flags); + + LASSERT (sri->sri_route != NULL); + + if (sri->sri_generation != kpr_state.kpr_generation) { + read_unlock_irqrestore(&kpr_state.kpr_rwlock, flags); + return -ESTALE; } - /* Not enough data to entirely satify callers request */ - prd->skip = 0; - read_unlock(&kpr_rwlock); - rc = user_len; + net = sri->sri_route->kpre_net; + nid = sri->sri_route->kpre_gateway->kpge_nid; + alive = sri->sri_route->kpre_gateway->kpge_alive; -routes_read_exit: - write_unlock(&(prd->proc_route_rwlock)); - return rc; + read_unlock_irqrestore(&kpr_state.kpr_rwlock, flags); + + seq_printf(s, "net %12s: gateway %s %s\n", + libcfs_net2str(net), libcfs_nid2str(nid), + alive ? "up" : "down"); + return 0; } -static int kpr_proc_routes_write(struct file *file, const char *ubuffer, - unsigned long count, void *data) +static struct seq_operations kpr_routes_sops = { + .start = kpr_seq_routes_start, + .stop = kpr_seq_routes_stop, + .next = kpr_seq_routes_next, + .show = kpr_seq_routes_show, +}; + +static int +kpr_seq_routes_open(struct inode *inode, struct file *file) { - /* no-op; lctl should be used to adjust the routes */ - return (count); + struct proc_dir_entry *dp = PDE(inode); + struct seq_file *sf; + int rc; + + rc = seq_open(file, &kpr_routes_sops); + if (rc == 0) { + sf = file->private_data; + sf->private = dp->data; + } + + return rc; } -void kpr_proc_init(void) +static struct file_operations kpr_routes_fops = { + .owner = THIS_MODULE, + .open = kpr_seq_routes_open, + .read = seq_read, + .llseek = seq_lseek, + .release = seq_release, +}; + +void +kpr_proc_init(void) { - struct proc_dir_entry *router_entry; - struct proc_dir_entry *routes_entry; + struct proc_dir_entry *stats; + struct proc_dir_entry *routes; /* Initialize KPR_PROC_ROUTER */ - router_entry = create_proc_entry (KPR_PROC_ROUTER, - S_IFREG | S_IRUGO | S_IWUSR, NULL); - - if (router_entry == NULL) { + stats = create_proc_entry (KPR_PROC_ROUTER, 0644, NULL); + if (stats == NULL) { CERROR("couldn't create proc entry %s\n", KPR_PROC_ROUTER); return; } - router_entry->data = NULL; - router_entry->read_proc = kpr_proc_router_read; - router_entry->write_proc = kpr_proc_router_write; + stats->data = NULL; + stats->read_proc = kpr_proc_stats_read; + stats->write_proc = kpr_proc_stats_write; /* Initialize KPR_PROC_ROUTES */ - routes_entry = create_proc_entry (KPR_PROC_ROUTES, - S_IFREG | S_IRUGO | S_IWUSR, NULL); - - if (routes_entry == NULL) { + routes = create_proc_entry (KPR_PROC_ROUTES, 0444, NULL); + if (routes == NULL) { CERROR("couldn't create proc entry %s\n", KPR_PROC_ROUTES); return; } - - kpr_read_routes_data.curr = NULL; - kpr_read_routes_data.generation = 0; - kpr_read_routes_data.skip = 0; - kpr_read_routes_data.proc_route_rwlock = RW_LOCK_UNLOCKED; - - routes_entry->data = &kpr_read_routes_data; - routes_entry->read_proc = kpr_proc_routes_read; - routes_entry->write_proc = kpr_proc_routes_write; + + routes->proc_fops = &kpr_routes_fops; + routes->data = NULL; } -void kpr_proc_fini(void) +void +kpr_proc_fini(void) { remove_proc_entry(KPR_PROC_ROUTER, 0); remove_proc_entry(KPR_PROC_ROUTES, 0); diff --git a/lnet/router/router.c b/lnet/router/router.c index d3f3faf..82a28f3 100644 --- a/lnet/router/router.c +++ b/lnet/router/router.c @@ -23,25 +23,20 @@ #include "router.h" -LIST_HEAD(kpr_routes); -LIST_HEAD(kpr_gateways); +struct kpr_state; -unsigned int kpr_routes_generation; -unsigned long long kpr_fwd_bytes; -unsigned long kpr_fwd_packets; -unsigned long kpr_fwd_errors; -atomic_t kpr_queue_depth; +static int forwarding = 0; +CFS_MODULE_PARM(forwarding, "i", int, 0444, + "Boolean: set non-zero to forward between networks"); -/* Mostly the tables are read-only (thread and interrupt context) - * - * Once in a blue moon we register/deregister NALs and add/remove routing - * entries (thread context only)... */ -rwlock_t kpr_rwlock = RW_LOCK_UNLOCKED; +static char *routes = ""; +CFS_MODULE_PARM(routes, "s", charp, 0444, + "routes to non-local networks"); int -kpr_routing () +kpr_forwarding () { - return 1; + return forwarding; } void @@ -122,10 +117,10 @@ kpr_notify (ptl_ni_t *ni, ptl_nid_t gateway_nid, int alive, time_t when) } /* Serialise with lookups (i.e. write lock) */ - write_lock_irqsave(&kpr_rwlock, flags); + write_lock_irqsave(&kpr_state.kpr_rwlock, flags); found = 0; - list_for_each_safe (e, n, &kpr_gateways) { + list_for_each_safe (e, n, &kpr_state.kpr_gateways) { ge = list_entry(e, kpr_gateway_entry_t, kpge_list); if (ge->kpge_nid != gateway_nid) @@ -137,14 +132,14 @@ kpr_notify (ptl_ni_t *ni, ptl_nid_t gateway_nid, int alive, time_t when) if (!found) { /* gateway not found */ - write_unlock_irqrestore(&kpr_rwlock, flags); + write_unlock_irqrestore(&kpr_state.kpr_rwlock, flags); CDEBUG (D_NET, "Gateway not found\n"); return (0); } if (when < ge->kpge_timestamp) { /* out of date information */ - write_unlock_irqrestore (&kpr_rwlock, flags); + write_unlock_irqrestore (&kpr_state.kpr_rwlock, flags); CDEBUG (D_NET, "Out of date\n"); return (0); } @@ -154,7 +149,7 @@ kpr_notify (ptl_ni_t *ni, ptl_nid_t gateway_nid, int alive, time_t when) if ((!ge->kpge_alive) == (!alive)) { /* new date for old news */ - write_unlock_irqrestore (&kpr_rwlock, flags); + write_unlock_irqrestore (&kpr_state.kpr_rwlock, flags); CDEBUG (D_NET, "Old news\n"); return (0); } @@ -166,14 +161,14 @@ kpr_notify (ptl_ni_t *ni, ptl_nid_t gateway_nid, int alive, time_t when) if (alive) { /* Reset all gateway weights so the newly-enabled gateway * doesn't have to play catch-up */ - list_for_each_safe (e, n, &kpr_gateways) { + list_for_each_safe (e, n, &kpr_state.kpr_gateways) { kpr_gateway_entry_t *ge = list_entry(e, kpr_gateway_entry_t, kpge_list); atomic_set (&ge->kpge_weight, 0); } } - write_unlock_irqrestore(&kpr_rwlock, flags); + write_unlock_irqrestore(&kpr_state.kpr_rwlock, flags); if (ni == NULL) { /* userland notified me: notify NAL? */ @@ -261,16 +256,16 @@ kpr_lookup (ptl_ni_t **nip, ptl_nid_t target_nid, int nob) return ni->ni_nid; } - read_lock_irqsave(&kpr_rwlock, flags); + read_lock_irqsave(&kpr_state.kpr_rwlock, flags); if (ni != NULL && ni->ni_shutdown) { /* pre-determined ni is shutting down */ - read_unlock_irqrestore(&kpr_rwlock, flags); + read_unlock_irqrestore(&kpr_state.kpr_rwlock, flags); return PTL_NID_ANY; } /* Search routes for one that has a gateway to target_nid on the callers network */ - list_for_each (e, &kpr_routes) { + list_for_each (e, &kpr_state.kpr_routes) { re = list_entry (e, kpr_route_entry_t, kpre_list); if (re->kpre_net != target_net) /* incorrect target net */ @@ -303,7 +298,7 @@ kpr_lookup (ptl_ni_t **nip, ptl_nid_t target_nid, int nob) } if (ge == NULL) { - read_unlock_irqrestore(&kpr_rwlock, flags); + read_unlock_irqrestore(&kpr_state.kpr_rwlock, flags); LASSERT (gwni == NULL); return PTL_NID_ANY; @@ -311,7 +306,7 @@ kpr_lookup (ptl_ni_t **nip, ptl_nid_t target_nid, int nob) kpr_update_weight (ge, nob); gwnid = ge->kpge_nid; - read_unlock_irqrestore(&kpr_rwlock, flags); + read_unlock_irqrestore(&kpr_state.kpr_rwlock, flags); /* NB can't deref 're/ge' after lock released! */ CDEBUG (D_NET, "lookup %s from %s: %s\n", @@ -351,10 +346,18 @@ kpr_fwd_start (ptl_ni_t *src_ni, kpr_fwd_desc_t *fwd) fwd->kprfd_src_ni = src_ni; /* stash calling ni */ - read_lock_irqsave(&kpr_rwlock, flags); + read_lock_irqsave(&kpr_state.kpr_rwlock, flags); + + spin_lock(&kpr_state.kpr_stats_lock); + kpr_state.kpr_fwd_packets++; + kpr_state.kpr_fwd_bytes += nob + sizeof(ptl_hdr_t); + spin_unlock(&kpr_state.kpr_stats_lock); - kpr_fwd_packets++; /* (loose) stats accounting */ - kpr_fwd_bytes += nob + sizeof(ptl_hdr_t); + if (!kpr_forwarding()) { + /* I'm not a router */ + rc = -EHOSTUNREACH; + goto out; + } if (src_ni->ni_shutdown) { /* caller is shutting down */ rc = -ESHUTDOWN; @@ -363,7 +366,7 @@ kpr_fwd_start (ptl_ni_t *src_ni, kpr_fwd_desc_t *fwd) /* Search routes for one that has a gateway to target_nid NOT on the caller's network */ - list_for_each (e, &kpr_routes) { + list_for_each (e, &kpr_state.kpr_routes) { re = list_entry (e, kpr_route_entry_t, kpre_list); if (re->kpre_net != target_net) /* no match */ @@ -400,9 +403,9 @@ kpr_fwd_start (ptl_ni_t *src_ni, kpr_fwd_desc_t *fwd) kpr_update_weight (ge, nob); fwd->kprfd_gateway_nid = ge->kpge_nid; - atomic_inc (&kpr_queue_depth); + atomic_inc (&kpr_state.kpr_queue_depth); - read_unlock_irqrestore(&kpr_rwlock, flags); + read_unlock_irqrestore(&kpr_state.kpr_rwlock, flags); CDEBUG (D_NET, "forward [%p] %s: src ni %s dst ni %s gw %s\n", fwd, libcfs_nid2str(target_nid), @@ -417,14 +420,16 @@ kpr_fwd_start (ptl_ni_t *src_ni, kpr_fwd_desc_t *fwd) rc = -EHOSTUNREACH; out: - kpr_fwd_errors++; + spin_lock_irqsave(&kpr_state.kpr_stats_lock, flags); + kpr_state.kpr_fwd_errors++; + spin_unlock_irqrestore(&kpr_state.kpr_stats_lock, flags); CDEBUG (D_NET, "Failed to forward [%p] %s from %s\n", fwd, libcfs_nid2str(target_nid), libcfs_nid2str(src_ni->ni_nid)); (fwd->kprfd_callback)(src_ni, fwd->kprfd_callback_arg, rc); - read_unlock_irqrestore(&kpr_rwlock, flags); + read_unlock_irqrestore(&kpr_state.kpr_rwlock, flags); } void @@ -438,7 +443,7 @@ kpr_fwd_done (ptl_ni_t *dst_ni, kpr_fwd_desc_t *fwd, int error) (fwd->kprfd_callback)(src_ni, fwd->kprfd_callback_arg, error); - atomic_dec (&kpr_queue_depth); + atomic_dec (&kpr_state.kpr_queue_depth); } int @@ -475,9 +480,9 @@ kpr_add_route (__u32 net, ptl_nid_t gateway_nid) re->kpre_net = net; LASSERT(!in_interrupt()); - write_lock_irqsave(&kpr_rwlock, flags); + write_lock_irqsave(&kpr_state.kpr_rwlock, flags); - list_for_each (e, &kpr_gateways) { + list_for_each (e, &kpr_state.kpr_gateways) { kpr_gateway_entry_t *ge2 = list_entry(e, kpr_gateway_entry_t, kpge_list); @@ -491,12 +496,12 @@ kpr_add_route (__u32 net, ptl_nid_t gateway_nid) if (!dup) { /* Adding a new gateway... */ - list_add (&ge->kpge_list, &kpr_gateways); + list_add (&ge->kpge_list, &kpr_state.kpr_gateways); /* ...zero all gateway weights so this one doesn't have to * play catch-up */ - list_for_each (e, &kpr_gateways) { + list_for_each (e, &kpr_state.kpr_gateways) { kpr_gateway_entry_t *ge2 = list_entry(e, kpr_gateway_entry_t, kpge_list); atomic_set (&ge2->kpge_weight, 0); @@ -505,10 +510,10 @@ kpr_add_route (__u32 net, ptl_nid_t gateway_nid) re->kpre_gateway = ge; ge->kpge_refcount++; - list_add (&re->kpre_list, &kpr_routes); - kpr_routes_generation++; + list_add (&re->kpre_list, &kpr_state.kpr_routes); + kpr_state.kpr_generation++; - write_unlock_irqrestore(&kpr_rwlock, flags); + write_unlock_irqrestore(&kpr_state.kpr_rwlock, flags); return (0); } @@ -527,9 +532,9 @@ kpr_del_route (__u32 net, ptl_nid_t gw_nid) /* NB Caller may specify either all routes via the given gateway * or a specific route entry actual NIDs) */ - write_lock_irqsave(&kpr_rwlock, flags); + write_lock_irqsave(&kpr_state.kpr_rwlock, flags); - list_for_each_safe (e, n, &kpr_routes) { + list_for_each_safe (e, n, &kpr_state.kpr_routes) { kpr_route_entry_t *re = list_entry(e, kpr_route_entry_t, kpre_list); kpr_gateway_entry_t *ge = re->kpre_gateway; @@ -553,8 +558,8 @@ kpr_del_route (__u32 net, ptl_nid_t gw_nid) PORTAL_FREE(re, sizeof (*re)); } - kpr_routes_generation++; - write_unlock_irqrestore(&kpr_rwlock, flags); + kpr_state.kpr_generation++; + write_unlock_irqrestore(&kpr_state.kpr_rwlock, flags); return (rc); } @@ -566,9 +571,9 @@ kpr_get_route (int idx, __u32 *net, ptl_nid_t *gateway_nid, __u32 *alive) unsigned long flags; LASSERT (!in_interrupt()); - read_lock_irqsave(&kpr_rwlock, flags); + read_lock_irqsave(&kpr_state.kpr_rwlock, flags); - for (e = kpr_routes.next; e != &kpr_routes; e = e->next) { + for (e = kpr_state.kpr_routes.next; e != &kpr_state.kpr_routes; e = e->next) { kpr_route_entry_t *re = list_entry(e, kpr_route_entry_t, kpre_list); kpr_gateway_entry_t *ge = re->kpre_gateway; @@ -578,12 +583,12 @@ kpr_get_route (int idx, __u32 *net, ptl_nid_t *gateway_nid, __u32 *alive) *gateway_nid = ge->kpge_nid; *alive = ge->kpge_alive; - read_unlock_irqrestore(&kpr_rwlock, flags); + read_unlock_irqrestore(&kpr_state.kpr_rwlock, flags); return (0); } } - read_unlock_irqrestore(&kpr_rwlock, flags); + read_unlock_irqrestore(&kpr_state.kpr_rwlock, flags); return (-ENOENT); } @@ -619,8 +624,8 @@ kpr_finalise (void) #ifdef __KERNEL__ kpr_proc_fini(); #endif - while (!list_empty (&kpr_routes)) { - kpr_route_entry_t *re = list_entry(kpr_routes.next, + while (!list_empty (&kpr_state.kpr_routes)) { + kpr_route_entry_t *re = list_entry(kpr_state.kpr_routes.next, kpr_route_entry_t, kpre_list); @@ -628,8 +633,8 @@ kpr_finalise (void) PORTAL_FREE(re, sizeof (*re)); } - while (!list_empty (&kpr_gateways)) { - kpr_gateway_entry_t *ge = list_entry(kpr_gateways.next, + while (!list_empty (&kpr_state.kpr_gateways)) { + kpr_gateway_entry_t *ge = list_entry(kpr_state.kpr_gateways.next, kpr_gateway_entry_t, kpge_list); @@ -641,7 +646,7 @@ kpr_finalise (void) atomic_read(&portal_kmemory)); } -void +int kpr_initialise (void) { int rc; @@ -649,14 +654,24 @@ kpr_initialise (void) CDEBUG(D_MALLOC, "kpr_initialise: kmem %d\n", atomic_read(&portal_kmemory)); - kpr_routes_generation = 0; + memset(&kpr_state, 0, sizeof(kpr_state)); + + INIT_LIST_HEAD(&kpr_state.kpr_routes); + INIT_LIST_HEAD(&kpr_state.kpr_gateways); + rwlock_init(&kpr_state.kpr_rwlock); + spin_lock_init(&kpr_state.kpr_stats_lock); + + rc = ptl_parse_routes(routes); #ifdef __KERNEL__ - kpr_proc_init(); + if (rc == 0) + kpr_proc_init(); #endif + + return rc; } -EXPORT_SYMBOL(kpr_routing); +EXPORT_SYMBOL(kpr_forwarding); EXPORT_SYMBOL(kpr_lookup); EXPORT_SYMBOL(kpr_fwd_start); EXPORT_SYMBOL(kpr_fwd_done); diff --git a/lnet/router/router.h b/lnet/router/router.h index 5e5fa78..208b7fe 100644 --- a/lnet/router/router.h +++ b/lnet/router/router.h @@ -65,16 +65,24 @@ typedef struct time_t kpru_when; } kpr_upcall_t; -extern void kpr_proc_init (void); -extern void kpr_proc_fini (void); +typedef struct{ + struct list_head kpr_routes; /* net -> gateways lookup */ + struct list_head kpr_gateways; /* known gateways */ + unsigned long long kpr_generation; /* validity stamp */ + rwlock_t kpr_rwlock; /* stabilize */ + + atomic_t kpr_queue_depth; /* packets being forwarded */ -extern unsigned int kpr_routes_generation; -extern unsigned long long kpr_fwd_bytes; -extern unsigned long kpr_fwd_packets; -extern unsigned long kpr_fwd_errors; -extern atomic_t kpr_queue_depth; + unsigned long long kpr_fwd_bytes; /* counters */ + unsigned long long kpr_fwd_packets; + unsigned long long kpr_fwd_errors; + spinlock_t kpr_stats_lock; /* serialise */ + +} kpr_state_t; -extern struct list_head kpr_routes; -extern rwlock_t kpr_rwlock; +extern kpr_state_t kpr_state; + +extern void kpr_proc_init (void); +extern void kpr_proc_fini (void); #endif /* _KPLROUTER_H */ diff --git a/lnet/utils/routerstat.c b/lnet/utils/routerstat.c index 99bc59b..54acfc3 100644 --- a/lnet/utils/routerstat.c +++ b/lnet/utils/routerstat.c @@ -22,14 +22,14 @@ do_stat (int fd) static char buffer[1024]; static double last = 0.0; static unsigned long long old_bytes; - static unsigned long old_packets; - static unsigned long old_errors; + static unsigned long long old_packets; + static unsigned long long old_errors; double now; double t; unsigned long long new_bytes, bytes; - unsigned long new_packets, packets; - unsigned long new_errors, errors; - unsigned long depth; + unsigned long long new_packets, packets; + unsigned long long new_errors, errors; + unsigned long long depth; int n; lseek (fd, 0, SEEK_SET); @@ -42,7 +42,7 @@ do_stat (int fd) } buffer[n] = 0; - n = sscanf (buffer, "%Lu %lu %lu %lu", + n = sscanf (buffer, "%Lu %Lu %Lu %Lu", &new_bytes, &new_packets, &new_errors, &depth); if (n < 3) @@ -52,9 +52,9 @@ do_stat (int fd) } if (last == 0.0) - printf ("%llu bytes, %lu packets (sz %lld), %lu errors", + printf ("%llu bytes, %llu packets (sz %lld), %llu errors", new_bytes, new_packets, - (long long)((new_packets == 0) ? 0LL : new_bytes/new_packets), + ((new_packets == 0) ? 0LL : new_bytes/new_packets), new_errors); else { @@ -64,26 +64,28 @@ do_stat (int fd) bytes = -1ULL - old_bytes + new_bytes + 1; else bytes = new_bytes - old_bytes; + if (new_packets < old_packets) packets = -1UL - old_packets + new_packets + 1; else packets = new_packets - old_packets; + if (new_errors < old_errors) errors = -1UL - old_errors + new_errors + 1; else errors = new_errors - old_errors; - printf ("%9llu bytes (%7.2fMb/s), %7lu packets (sz %5lld, %5ld/s), %lu errors (%ld/s)", + printf ("%9llu bytes (%7.2fMb/s), %7llu packets (sz %5lld, %5lld/s), %llu errors (%lld/s)", bytes, ((double)bytes)/((1<<20) * t), - packets, (long long)((packets == 0) ? 0LL : bytes/packets), (long)(packets/t), - errors, (long)(errors/t)); + packets, (packets == 0) ? 0LL : bytes/packets, packets/t, + errors, errors/t); } old_bytes = new_bytes; old_packets = new_packets; old_errors = new_errors; if (n == 4) - printf (", depth (%ld)\n", depth); + printf (", depth (%lld)\n", depth); else printf ("\n"); diff --git a/lnet/utils/wirecheck.c b/lnet/utils/wirecheck.c index f97a268..e179f40 100644 --- a/lnet/utils/wirecheck.c +++ b/lnet/utils/wirecheck.c @@ -181,8 +181,8 @@ system_string (char *cmdline, char *str, int len) int main (int argc, char **argv) { - char unameinfo[80]; - char gccinfo[80]; + char unameinfo[256]; + char gccinfo[256]; system_string("uname -a", unameinfo, sizeof(unameinfo)); system_string("gcc -v 2>&1 | tail -1", gccinfo, sizeof(gccinfo)); -- 1.8.3.1