From dce7bf045bacb836ce16ad50cd7f8949a3f68c4d Mon Sep 17 00:00:00 2001 From: eeb Date: Thu, 19 May 2005 14:05:33 +0000 Subject: [PATCH] * changed router lookups to always return the local destination socknal + qswnal changes to match simplified "will this be forwarded" predicate in ptl_send() * fixed bug in multi-instance socknal to pass the correct NI to the tx teardown function to allow for the case where the tx hasn't even had a conn assigned yet. * fixed routerstat stats formatting * removed osc::check_elan_limit() and replaced with max_unfragmented_pages() which actually works. NB it depend on the assumption that file and page offsets are aligned! * fixed lconf to stay away from portals, except to 'modprobe' for it first in 'load_module()', and to rmmod all the NALs + portals when ptlrpc gets unloaded. --- lnet/klnds/qswlnd/qswlnd_cb.c | 3 --- lnet/klnds/socklnd/socklnd.h | 2 +- lnet/klnds/socklnd/socklnd_cb.c | 28 +++++++++++----------------- lnet/lnet/lib-move.c | 24 +++++++++++++----------- lnet/router/router.c | 21 +++++++++++---------- lnet/utils/routerstat.c | 4 ++-- 6 files changed, 38 insertions(+), 44 deletions(-) diff --git a/lnet/klnds/qswlnd/qswlnd_cb.c b/lnet/klnds/qswlnd/qswlnd_cb.c index 9a71e80..bbb778c 100644 --- a/lnet/klnds/qswlnd/qswlnd_cb.c +++ b/lnet/klnds/qswlnd/qswlnd_cb.c @@ -1312,9 +1312,6 @@ kqswnal_fwd_packet (ptl_ni_t *ni, kpr_fwd_desc_t *fwd) if (ktx == NULL) /* can't get txd right now */ return; /* fwd will be scheduled when tx desc freed */ - if (nid == ni->ni_nid) /* gateway is me */ - nid = fwd->kprfd_target_nid; /* target is final dest */ - /* copy hdr into pre-mapped buffer */ memcpy(ktx->ktx_buffer, fwd->kprfd_hdr, sizeof(ptl_hdr_t)); diff --git a/lnet/klnds/socklnd/socklnd.h b/lnet/klnds/socklnd/socklnd.h index 5936445..0014402 100644 --- a/lnet/klnds/socklnd/socklnd.h +++ b/lnet/klnds/socklnd/socklnd.h @@ -518,7 +518,7 @@ extern int ksocknal_close_conn_and_siblings (ksock_conn_t *conn, int why); extern int ksocknal_close_matching_conns (ptl_nid_t nid, __u32 ipaddr); extern void ksocknal_queue_tx_locked (ksock_tx_t *tx, ksock_conn_t *conn); -extern void ksocknal_tx_done (ksock_tx_t *tx, int asynch); +extern void ksocknal_tx_done (ksock_peer_t *peer, ksock_tx_t *tx, int asynch); extern void ksocknal_fwd_packet (ptl_ni_t *ni, kpr_fwd_desc_t *fwd); extern void ksocknal_fmb_callback (ptl_ni_t *ni, void *arg, int error); extern void ksocknal_notify (ptl_ni_t *ni, ptl_nid_t gw_nid, int alive); diff --git a/lnet/klnds/socklnd/socklnd_cb.c b/lnet/klnds/socklnd/socklnd_cb.c index 1ff7bd4..25af875 100644 --- a/lnet/klnds/socklnd/socklnd_cb.c +++ b/lnet/klnds/socklnd/socklnd_cb.c @@ -360,7 +360,7 @@ ksocknal_zc_callback (zccd_t *zcd) #endif void -ksocknal_tx_done (ksock_tx_t *tx, int asynch) +ksocknal_tx_done (ksock_peer_t *peer, ksock_tx_t *tx, int asynch) { ksock_ltx_t *ltx; ENTRY; @@ -378,7 +378,7 @@ ksocknal_tx_done (ksock_tx_t *tx, int asynch) } if (tx->tx_isfwd) { /* was a forwarded packet? */ - kpr_fwd_done (tx->tx_conn->ksnc_peer->ksnp_ni, + kpr_fwd_done (peer->ksnp_ni, KSOCK_TX_2_KPR_FWD_DESC (tx), (tx->tx_resid == 0) ? 0 : -ECONNABORTED); EXIT; @@ -388,7 +388,7 @@ ksocknal_tx_done (ksock_tx_t *tx, int asynch) /* local send */ ltx = KSOCK_TX_2_KSOCK_LTX (tx); - ptl_finalize (tx->tx_conn->ksnc_peer->ksnp_ni, + ptl_finalize (peer->ksnp_ni, ltx->ltx_private, ltx->ltx_cookie, (tx->tx_resid == 0) ? PTL_OK : PTL_FAIL); @@ -415,7 +415,7 @@ ksocknal_tx_launched (ksock_tx_t *tx) #endif /* Any zero-copy-ness (if any) has completed; I can complete the * transmit now, avoiding an extra schedule */ - ksocknal_tx_done (tx, 0); + ksocknal_tx_done (tx->tx_conn->ksnc_peer, tx, 0); } int @@ -682,7 +682,7 @@ ksocknal_find_connecting_route_locked (ksock_peer_t *peer) } int -ksocknal_launch_packet (ptl_ni_t *ni, ksock_tx_t *tx, ptl_nid_t nid, int routing) +ksocknal_launch_packet (ptl_ni_t *ni, ksock_tx_t *tx, ptl_nid_t nid) { unsigned long flags; ksock_peer_t *peer; @@ -798,7 +798,6 @@ ksocknal_sendmsg(ptl_ni_t *ni, ptl_hdr_t *hdr, int type, ptl_process_id_t target, - int routing, unsigned int payload_niov, struct iovec *payload_iov, ptl_kiov_t *payload_kiov, @@ -880,7 +879,7 @@ ksocknal_sendmsg(ptl_ni_t *ni, payload_offset, payload_nob); } - rc = ksocknal_launch_packet(ni, <x->ltx_tx, target.nid, routing); + rc = ksocknal_launch_packet(ni, <x->ltx_tx, target.nid); if (rc == 0) return (PTL_OK); @@ -895,7 +894,7 @@ ksocknal_send (ptl_ni_t *ni, void *private, ptl_msg_t *cookie, size_t payload_offset, size_t payload_len) { return (ksocknal_sendmsg(ni, private, cookie, - hdr, type, tgt, routing, + hdr, type, tgt, payload_niov, payload_iov, NULL, payload_offset, payload_len)); } @@ -907,7 +906,7 @@ ksocknal_send_pages (ptl_ni_t *ni, void *private, ptl_msg_t *cookie, size_t payload_offset, size_t payload_len) { return (ksocknal_sendmsg(ni, private, cookie, - hdr, type, tgt, routing, + hdr, type, tgt, payload_niov, NULL, payload_kiov, payload_offset, payload_len)); } @@ -917,17 +916,12 @@ ksocknal_fwd_packet (ptl_ni_t *ni, kpr_fwd_desc_t *fwd) { ptl_nid_t nid = fwd->kprfd_gateway_nid; ksock_ftx_t *ftx = (ksock_ftx_t *)&fwd->kprfd_scratch; - int routing; int rc; CDEBUG (D_NET, "Forwarding [%p] -> %s (%s))\n", fwd, libcfs_nid2str(fwd->kprfd_gateway_nid), libcfs_nid2str(fwd->kprfd_target_nid)); - routing = (nid != ni->ni_nid); - if (!routing) - nid = fwd->kprfd_target_nid; - /* setup iov for hdr */ ftx->ftx_iov.iov_base = fwd->kprfd_hdr; ftx->ftx_iov.iov_len = sizeof(ptl_hdr_t); @@ -939,7 +933,7 @@ ksocknal_fwd_packet (ptl_ni_t *ni, kpr_fwd_desc_t *fwd) ftx->ftx_tx.tx_nkiov = fwd->kprfd_niov; ftx->ftx_tx.tx_kiov = fwd->kprfd_kiov; - rc = ksocknal_launch_packet (ni, &ftx->ftx_tx, nid, routing); + rc = ksocknal_launch_packet (ni, &ftx->ftx_tx, nid); if (rc != 0) kpr_fwd_done (ni, fwd, rc); } @@ -1593,7 +1587,7 @@ int ksocknal_scheduler (void *arg) list_del (&tx->tx_list); spin_unlock_irqrestore (&sched->kss_lock, flags); - ksocknal_tx_done (tx, 1); + ksocknal_tx_done (tx->tx_conn->ksnc_peer, tx, 1); spin_lock_irqsave (&sched->kss_lock, flags); } @@ -2055,7 +2049,7 @@ ksocknal_connect (ksock_route_t *route) list_del (&tx->tx_list); /* complete now */ - ksocknal_tx_done (tx, 0); + ksocknal_tx_done (peer, tx, 0); } } diff --git a/lnet/lnet/lib-move.c b/lnet/lnet/lib-move.c index 6dd787d..b30e396 100644 --- a/lnet/lnet/lib-move.c +++ b/lnet/lnet/lib-move.c @@ -30,6 +30,10 @@ static int allow_destination_aliases = 0; CFS_MODULE_PARM(allow_destination_aliases, "i", int, 0644, "Boolean: don't require strict destination NIDs"); +static int implicit_loopback = 1; +CFS_MODULE_PARM(implicit_loopback, "i", int, 0644, + "Boolean: allow destination aliases when sending to yourself"); + /* forward ref */ static void ptl_commit_md (ptl_libmd_t *md, ptl_msg_t *msg); @@ -587,16 +591,15 @@ ptl_send (ptl_ni_t *ni, void *private, ptl_msg_t *msg, return PTL_FAIL; } - if (PTL_NETNAL(PTL_NIDNET(ni->ni_nid)) == LONAL) { - if (target.nid != ni->ni_nid) { - /* will gateway have to forward? */ - routing = (gw_nid != ni->ni_nid); - } else if (allow_destination_aliases) { - /* it's for me (force lonal) */ + if (PTL_NETNAL(PTL_NIDNET(ni->ni_nid)) != LONAL) { + if (gw_nid != ni->ni_nid) { /* it's not for me */ + routing = gw_nid != target.nid; /* will gateway have to forward? */ + } else if (allow_destination_aliases || /* force lonal? */ + implicit_loopback) { ptl_ni_addref(ptl_loni); ptl_ni_decref(ni); ni = ptl_loni; - } else { + } else { /* barf */ ptl_ni_decref(ni); CERROR("Attempt to send to self via %s, not LONAL\n", libcfs_nid2str(target.nid)); @@ -615,8 +618,7 @@ ptl_send (ptl_ni_t *ni, void *private, ptl_msg_t *msg, if (type == PTL_MSG_PUT || type == PTL_MSG_GET) msg->msg_ev.initiator.nid = ni->ni_nid; - if (routing) - target.nid = gw_nid; + target.nid = gw_nid; if (len == 0) rc = (ni->ni_nal->nal_send)(ni, private, msg, hdr, @@ -1045,8 +1047,8 @@ ptl_parse(ptl_ni_t *ni, ptl_hdr_t *hdr, void *private) dest_nid = le64_to_cpu(hdr->dest_nid); if (dest_nid != ni->ni_nid) { - if (!ptl_islocalnid(dest_nid)) - return PTL_IFACE_DUP; + if (!ptl_islocalnid(dest_nid)) /* tell NAL to use the router */ + return PTL_IFACE_DUP; /* to forward */ /* dest_nid is one of my NIs */ diff --git a/lnet/router/router.c b/lnet/router/router.c index e072434..edd9669 100644 --- a/lnet/router/router.c +++ b/lnet/router/router.c @@ -238,8 +238,9 @@ kpr_lookup (ptl_ni_t **nip, ptl_nid_t target_nid, int nob) ptl_ni_t *gwni = NULL; ptl_ni_t *tmpni = NULL; kpr_gateway_entry_t *ge = NULL; - int rc = -ENOENT; __u32 target_net = PTL_NIDNET(target_nid); + + /* Return the NID I must send to, to reach 'target_nid' */ CDEBUG (D_NET, "lookup "LPX64" from %s\n", target_nid, (ni == NULL) ? "<>" : libcfs_nid2str(ni->ni_nid)); @@ -248,15 +249,15 @@ kpr_lookup (ptl_ni_t **nip, ptl_nid_t target_nid, int nob) gwni = ptl_net2ni(target_net); /* is it a local network? */ if (gwni != NULL) { *nip = gwni; - return gwni->ni_nid; + return target_nid; + } + } else { /* ni already determined */ + if (target_net == PTL_NIDNET(ni->ni_nid)) { + ptl_ni_addref(ni); /* extra ref so caller can drop blindly */ + return target_nid; } - } else if (target_net == PTL_NIDNET(ni->ni_nid)) { - /* Caller wants to know if 'target_nid' can be reached via a - * gateway ON HER OWN NETWORK */ - ptl_ni_addref(ni); /* extra ref so caller can drop blindly */ - return ni->ni_nid; } - + CDEBUG(D_NET, "%s from %s\n", libcfs_nid2str(target_nid), (ni == NULL) ? "" : libcfs_nid2str(ni->ni_nid)); @@ -347,7 +348,7 @@ kpr_lookup (ptl_ni_t **nip, ptl_nid_t target_nid, int nob) LASSERT ((gwni == NULL) != (ni == NULL)); - if (gwni == NULL) + if (ni != NULL) ptl_ni_addref(ni); /* extra ref so caller can drop blindly */ else *nip = gwni; /* already got a ref */ @@ -401,7 +402,7 @@ kpr_fwd_start (ptl_ni_t *src_ni, kpr_fwd_desc_t *fwd) if (dst_ni->ni_nal->nal_fwd == NULL) goto failed; - fwd->kprfd_gateway_nid = dst_ni->ni_nid; + fwd->kprfd_gateway_nid = target_nid; atomic_inc (&kpr_state.kpr_queue_depth); read_unlock_irqrestore(&kpr_state.kpr_rwlock, flags); diff --git a/lnet/utils/routerstat.c b/lnet/utils/routerstat.c index 54acfc3..8d53ff0 100644 --- a/lnet/utils/routerstat.c +++ b/lnet/utils/routerstat.c @@ -75,7 +75,7 @@ do_stat (int fd) else errors = new_errors - old_errors; - printf ("%9llu bytes (%7.2fMb/s), %7llu packets (sz %5lld, %5lld/s), %llu errors (%lld/s)", + printf ("%9llu bytes (%7.2fMb/s), %7llu packets (sz %5lld, %5.0f/s), %llu errors (%0.0f/s)", bytes, ((double)bytes)/((1<<20) * t), packets, (packets == 0) ? 0LL : bytes/packets, packets/t, errors, errors/t); @@ -103,7 +103,7 @@ int main (int argc, char **argv) if (argc > 1) interval = atoi (argv[1]); - fd = open ("/proc/sys/portals/router", O_RDONLY); + fd = open ("/proc/sys/portals/router_stats", O_RDONLY); if (fd < 0) { fprintf (stderr, "Can't open stat: %s\n", strerror (errno)); -- 1.8.3.1