From d49945183129a7cd8987b6bf36438f5115d9d447 Mon Sep 17 00:00:00 2001 From: eeb Date: Sat, 7 May 2005 03:57:57 +0000 Subject: [PATCH] * changed router to do "obvious" routing automatically if enabled * tested routing between elan and tcp networks --- lnet/klnds/qswlnd/qswlnd_cb.c | 6 +-- lnet/klnds/socklnd/socklnd_cb.c | 27 ++++++------ lnet/lnet/api-ni.c | 4 -- lnet/lnet/module.c | 21 +++++++++- lnet/router/proc.c | 10 ++--- lnet/router/router.c | 91 ++++++++++++++++++++++++++++++----------- 6 files changed, 109 insertions(+), 50 deletions(-) diff --git a/lnet/klnds/qswlnd/qswlnd_cb.c b/lnet/klnds/qswlnd/qswlnd_cb.c index c8c435c7..f0c2793 100644 --- a/lnet/klnds/qswlnd/qswlnd_cb.c +++ b/lnet/klnds/qswlnd/qswlnd_cb.c @@ -1513,8 +1513,8 @@ kqswnal_parse (kqswnal_rx_t *krx) LASSERTF (0, "checksums for forwarded packets not implemented\n"); #endif - if (kqswnal_nid2elanid (dest_nid) >= 0) /* should have gone direct to peer */ - { + if (PTL_NIDNET(dest_nid) == PTL_NIDNET(kqswnal_data.kqn_ni->ni_nid)) { + /* should have gone direct to peer */ CERROR("dropping packet from %s for %s: target is peer\n", libcfs_nid2str(le64_to_cpu(hdr->src_nid)), libcfs_nid2str(dest_nid)); @@ -1942,7 +1942,7 @@ kqswnal_scheduler (void *arg) spin_unlock_irqrestore (&kqswnal_data.kqn_sched_lock, flags); /* If we're shutting down, this will just requeue fwd on kqn_idletxd_fwdq */ - kqswnal_fwd_packet (NULL, fwd); + kqswnal_fwd_packet (kqswnal_data.kqn_ni, fwd); did_something = 1; spin_lock_irqsave (&kqswnal_data.kqn_sched_lock, flags); diff --git a/lnet/klnds/socklnd/socklnd_cb.c b/lnet/klnds/socklnd/socklnd_cb.c index 7a1be2b..e9fb82a 100644 --- a/lnet/klnds/socklnd/socklnd_cb.c +++ b/lnet/klnds/socklnd/socklnd_cb.c @@ -1156,13 +1156,15 @@ ksocknal_fwd_parse (ksock_conn_t *conn) return; } - if (body_len > PTL_MTU) { /* too big to forward */ + if (PTL_NIDNET(conn->ksnc_hdr.dest_nid) == + PTL_NIDNET(ksocknal_data.ksnd_ni->ni_nid)) { + /* should have gone direct */ CERROR ("dropping packet from %s for %s: " - "packet size %d too big\n", + "target is a peer\n", libcfs_nid2str(src_nid), - libcfs_nid2str(dest_nid), - body_len); - /* on to new packet (skip this one's body) */ + libcfs_nid2str(dest_nid)); + + /* on to next packet (skip this one's body) */ ksocknal_new_packet (conn, body_len); return; } @@ -1176,17 +1178,14 @@ ksocknal_fwd_parse (ksock_conn_t *conn) ksocknal_new_packet (conn, body_len); /* on to new packet */ return; } - - /* should have gone direct */ - peer = ksocknal_find_peer (conn->ksnc_hdr.dest_nid); - if (peer != NULL) { + + if (body_len > PTL_MTU) { /* too big to forward */ CERROR ("dropping packet from %s for %s: " - "target is a peer\n", + "packet size %d too big\n", libcfs_nid2str(src_nid), - libcfs_nid2str(dest_nid)); - ksocknal_peer_decref(peer); /* drop ref from get above */ - - /* on to next packet (skip this one's body) */ + libcfs_nid2str(dest_nid), + body_len); + /* on to new packet (skip this one's body) */ ksocknal_new_packet (conn, body_len); return; } diff --git a/lnet/lnet/api-ni.c b/lnet/lnet/api-ni.c index d2c6549..bc4ef60 100644 --- a/lnet/lnet/api-ni.c +++ b/lnet/lnet/api-ni.c @@ -27,10 +27,6 @@ static char *networks = DEFAULT_NETWORKS; CFS_MODULE_PARM(networks, "s", charp, 0444, "local networks (default='"DEFAULT_NETWORKS"')"); -static int nal_load_timeout = 10; -CFS_MODULE_PARM(nal_load_timeout, "i", int, 0444, - "seconds to wait for a NAL to load"); - ptl_apini_t ptl_apini; /* THE network interface (at the API) */ void ptl_assert_wire_constants (void) diff --git a/lnet/lnet/module.c b/lnet/lnet/module.c index 9943b3e..10057b9 100644 --- a/lnet/lnet/module.c +++ b/lnet/lnet/module.c @@ -25,6 +25,10 @@ #define DEBUG_SUBSYSTEM S_PORTALS #include +static int config_on_load = 1; +CFS_MODULE_PARM(config_on_load, "i", int, 0444, + "configure network at module load"); + static int kportal_ioctl(unsigned int cmd, struct portal_ioctl_data *data) { ptl_err_t initrc; @@ -50,7 +54,7 @@ static int kportal_ioctl(unsigned int cmd, struct portal_ioctl_data *data) initrc = PtlNIInit(PTL_IFACE_DEFAULT, LUSTRE_SRV_PTL_PID, NULL, NULL, &nih); if (!(initrc == PTL_OK || initrc == PTL_IFACE_DUP)) - RETURN (-EINVAL); + RETURN (-ENETDOWN); rc = PtlNICtl(nih, cmd, data); @@ -79,6 +83,21 @@ static int init_kportals_module(void) RETURN(rc); } + if (config_on_load) { + ptl_handle_ni_t nih; + + PTL_MUTEX_DOWN(&ptl_apini.apini_api_mutex); + ptl_apini.apini_niinit_self = 1; + PTL_MUTEX_UP(&ptl_apini.apini_api_mutex); + + rc = PtlNIInit(PTL_IFACE_DEFAULT, LUSTRE_SRV_PTL_PID, + NULL, NULL, &nih); + if (rc != PTL_OK) { + PtlFini(); + return -ENETDOWN; + } + } + rc = libcfs_register_ioctl(&kportal_ioctl_handler); LASSERT (rc == 0); diff --git a/lnet/router/proc.c b/lnet/router/proc.c index 9bec9c4..5358492 100644 --- a/lnet/router/proc.c +++ b/lnet/router/proc.c @@ -24,7 +24,7 @@ #include "router.h" #include -#define KPR_PROC_ROUTER "sys/portals/router" +#define KPR_PROC_STATS "sys/portals/router_stats" #define KPR_PROC_ROUTES "sys/portals/routes" static int @@ -239,10 +239,10 @@ kpr_proc_init(void) struct proc_dir_entry *stats; struct proc_dir_entry *routes; - /* Initialize KPR_PROC_ROUTER */ - stats = create_proc_entry (KPR_PROC_ROUTER, 0644, NULL); + /* Initialize KPR_PROC_STATS */ + stats = create_proc_entry (KPR_PROC_STATS, 0644, NULL); if (stats == NULL) { - CERROR("couldn't create proc entry %s\n", KPR_PROC_ROUTER); + CERROR("couldn't create proc entry %s\n", KPR_PROC_STATS); return; } @@ -264,6 +264,6 @@ kpr_proc_init(void) void kpr_proc_fini(void) { - remove_proc_entry(KPR_PROC_ROUTER, 0); + remove_proc_entry(KPR_PROC_STATS, 0); remove_proc_entry(KPR_PROC_ROUTES, 0); } diff --git a/lnet/router/router.c b/lnet/router/router.c index fff6c1a..47bb078 100644 --- a/lnet/router/router.c +++ b/lnet/router/router.c @@ -243,9 +243,6 @@ kpr_lookup (ptl_ni_t **nip, ptl_nid_t target_nid, int nob) int rc = -ENOENT; __u32 target_net = PTL_NIDNET(target_nid); - /* Caller wants to know if 'target_nid' can be reached via a gateway - * ON HER OWN NETWORK */ - CDEBUG (D_NET, "lookup "LPX64" from %s\n", target_nid, (ni == NULL) ? "<>" : libcfs_nid2str(ni->ni_nid)); @@ -256,10 +253,15 @@ kpr_lookup (ptl_ni_t **nip, ptl_nid_t target_nid, int nob) return gwni->ni_nid; } } else if (target_net == PTL_NIDNET(ni->ni_nid)) { - ptl_ni_addref(ni); /* extra ref so caller can drop blindly */ + /* Caller wants to know if 'target_nid' can be reached via a + * gateway ON HER OWN NETWORK */ + ptl_ni_addref(ni); /* extra ref so caller can drop blindly */ return ni->ni_nid; } + CDEBUG(D_NET, "%s from %s\n", libcfs_nid2str(target_nid), + (ni == NULL) ? "" : libcfs_nid2str(ni->ni_nid)); + read_lock_irqsave(&kpr_state.kpr_rwlock, flags); if (ni != NULL && ni->ni_shutdown) { @@ -283,22 +285,37 @@ kpr_lookup (ptl_ni_t **nip, ptl_nid_t target_nid, int nob) if (PTL_NIDNET(ni->ni_nid) != /* gateway not on ni's net */ PTL_NIDNET(re->kpre_gateway->kpge_nid)) continue; - tmpni = NULL; + + if (ge != NULL && + kpr_ge_isbetter (ge, re->kpre_gateway)) + continue; + + } else if (gwni != NULL && + PTL_NIDNET(gwni->ni_nid) == + PTL_NIDNET(ge->kpge_nid)) { + /* another gateway on the same net */ + + if (kpr_ge_isbetter(ge, re->kpre_gateway)) + continue; } else { - tmpni = ptl_net2ni(PTL_NIDNET(ge->kpge_nid)); + /* another gateway on a new/different net */ + + tmpni = ptl_net2ni(PTL_NIDNET(re->kpre_gateway->kpge_nid)); if (tmpni == NULL) /* gateway not on a local net */ continue; - } - if (ge == NULL || - kpr_ge_isbetter (re->kpre_gateway, ge)) { + if (ge != NULL && + kpr_ge_isbetter(ge, re->kpre_gateway)) { + ptl_ni_decref(tmpni); + continue; + } + if (gwni != NULL) ptl_ni_decref(gwni); - ge = re->kpre_gateway; gwni = tmpni; - } else if (tmpni != NULL) { - ptl_ni_decref(tmpni); } + + ge = re->kpre_gateway; } if (ge == NULL) { @@ -308,7 +325,7 @@ kpr_lookup (ptl_ni_t **nip, ptl_nid_t target_nid, int nob) return PTL_NID_ANY; } - kpr_update_weight (ge, nob); + kpr_update_weight(ge, nob); gwnid = ge->kpge_nid; read_unlock_irqrestore(&kpr_state.kpr_rwlock, flags); @@ -335,8 +352,8 @@ kpr_fwd_start (ptl_ni_t *src_ni, kpr_fwd_desc_t *fwd) __u32 target_net = PTL_NIDNET(target_nid); __u32 source_net = PTL_NIDNET(src_ni->ni_nid); int nob = fwd->kprfd_nob; - kpr_gateway_entry_t *ge = NULL; - ptl_ni_t *dst_ni = NULL; + kpr_gateway_entry_t *ge; + ptl_ni_t *dst_ni; ptl_ni_t *tmp_ni; unsigned long flags; struct list_head *e; @@ -346,6 +363,7 @@ kpr_fwd_start (ptl_ni_t *src_ni, kpr_fwd_desc_t *fwd) CDEBUG (D_NET, "forward [%p] %s from %s\n", fwd, libcfs_nid2str(target_nid), libcfs_nid2str(src_ni->ni_nid)); + LASSERT (target_net != source_net); LASSERT (nob == ptl_kiov_nob (fwd->kprfd_niov, fwd->kprfd_kiov)); fwd->kprfd_src_ni = src_ni; /* stash calling ni */ @@ -368,8 +386,32 @@ kpr_fwd_start (ptl_ni_t *src_ni, kpr_fwd_desc_t *fwd) goto out; } - /* Search routes for one that has a gateway to target_nid NOT on the caller's network */ + /* Is the target_nid on a local network? */ + dst_ni = ptl_net2ni(target_net); + if (dst_ni != NULL) { + if (dst_ni->ni_nal->nal_fwd == NULL) { + rc = -EHOSTUNREACH; + goto out; + } + + fwd->kprfd_gateway_nid = dst_ni->ni_nid; + atomic_inc (&kpr_state.kpr_queue_depth); + + read_unlock_irqrestore(&kpr_state.kpr_rwlock, flags); + + CDEBUG (D_NET, "forward [%p] %s: src ni %s dst ni %s\n", + fwd, libcfs_nid2str(target_nid), + libcfs_nid2str(src_ni->ni_nid), + libcfs_nid2str(dst_ni->ni_nid)); + dst_ni->ni_nal->nal_fwd(dst_ni, fwd); + ptl_ni_decref(dst_ni); + return; + } + + /* Search routes for one that has a gateway to target_nid NOT on the caller's network */ + dst_ni = NULL; + ge = NULL; list_for_each (e, &kpr_state.kpr_routes) { re = list_entry (e, kpr_route_entry_t, kpre_list); @@ -391,14 +433,17 @@ kpr_fwd_start (ptl_ni_t *src_ni, kpr_fwd_desc_t *fwd) continue; } - if (ge == NULL || - kpr_ge_isbetter (re->kpre_gateway, ge)) { - if (dst_ni != NULL) - ptl_ni_decref(dst_ni); - - dst_ni = tmp_ni; - ge = re->kpre_gateway; + if (ge != NULL && + kpr_ge_isbetter(ge, re->kpre_gateway)) { + ptl_ni_decref(tmp_ni); + continue; } + + if (dst_ni != NULL) + ptl_ni_decref(dst_ni); + + dst_ni = tmp_ni; + ge = re->kpre_gateway; } if (ge != NULL) { -- 1.8.3.1