From 54cb140831d1915b1f122ee64d588ab573431380 Mon Sep 17 00:00:00 2001 From: eeb Date: Fri, 29 Apr 2005 19:34:49 +0000 Subject: [PATCH] * libcfs/nidstrings.c does nid parsing/printing. It's included in libptlctl.a for userspace utils too. * addr@net - style nids working with > 1 local network * router part of portals; ptl_send() uses it to chose the NI, but NALs still do am-i-there-yet forwarding. NAL interface changed to match. * reworked nal commands into PtlCtl(ni, cmd, arg) and rewired portals ioctl handling. PtlFailNid, now just another PtlCtl. NAL interface includes ctl handler. * commented out all lconf portals config, except setting the NID (pending socknal working out its own interfaces) * removed all portals config from zeroconf and llmount * "lctl network" on its own lists all local NIDs (replaces shownid). Most (all?) portals config commands still work from lctl. * simplified NAL setup/teardown (no separate nal_cmd and router init) --- lnet/include/lnet/lib-lnet.h | 1 + lnet/include/lnet/lib-p30.h | 1 + lnet/klnds/lolnd/lolnd.h | 4 ++-- lnet/klnds/lolnd/lolnd_cb.c | 50 +++++++++++++++++++++++++------------------- lnet/libcfs/nidstrings.c | 24 +++++++++++++++------ lnet/lnet/api-ni.c | 15 +++++++------ lnet/lnet/lib-move.c | 1 - lnet/router/autoMakefile.am | 6 +++++- lnet/router/router.c | 34 +++++++++++++++++------------- 9 files changed, 83 insertions(+), 53 deletions(-) diff --git a/lnet/include/lnet/lib-lnet.h b/lnet/include/lnet/lib-lnet.h index 2647e2e..2b02a1c 100644 --- a/lnet/include/lnet/lib-lnet.h +++ b/lnet/include/lnet/lib-lnet.h @@ -401,6 +401,7 @@ extern void ptl_queue_zombie_ni (ptl_ni_t *ni); static inline void ptl_ni_decref(ptl_ni_t *ni) { + /* CAVEAT EMPTOR! must NOT be holding PTL_LOCK() (deadlock) */ LASSERT (atomic_read(&ni->ni_refcount) > 0); if (atomic_dec_and_test(&ni->ni_refcount)) ptl_queue_zombie_ni(ni); diff --git a/lnet/include/lnet/lib-p30.h b/lnet/include/lnet/lib-p30.h index 2647e2e..2b02a1c 100644 --- a/lnet/include/lnet/lib-p30.h +++ b/lnet/include/lnet/lib-p30.h @@ -401,6 +401,7 @@ extern void ptl_queue_zombie_ni (ptl_ni_t *ni); static inline void ptl_ni_decref(ptl_ni_t *ni) { + /* CAVEAT EMPTOR! must NOT be holding PTL_LOCK() (deadlock) */ LASSERT (atomic_read(&ni->ni_refcount) > 0); if (atomic_dec_and_test(&ni->ni_refcount)) ptl_queue_zombie_ni(ni); diff --git a/lnet/klnds/lolnd/lolnd.h b/lnet/klnds/lolnd/lolnd.h index db5a2c5..092af64 100644 --- a/lnet/klnds/lolnd/lolnd.h +++ b/lnet/klnds/lolnd/lolnd.h @@ -58,13 +58,13 @@ ptl_err_t klonal_startup (ptl_ni_t *ni, char **interfaces); void klonal_shutdown (ptl_ni_t *ni); ptl_err_t klonal_send (ptl_ni_t *ni, void *private, ptl_msg_t *ptlmsg, ptl_hdr_t *hdr, - int type, ptl_nid_t nid, ptl_pid_t pid, + int type, ptl_process_id_t tgt, int routing, unsigned int payload_niov, struct iovec *payload_iov, size_t payload_offset, size_t payload_nob); ptl_err_t klonal_send_pages (ptl_ni_t *ni, void *private, ptl_msg_t *ptlmsg, ptl_hdr_t *hdr, - int type, ptl_nid_t nid, ptl_pid_t pid, + int type, ptl_process_id_t tgt, int routing, unsigned int payload_niov, ptl_kiov_t *payload_kiov, size_t payload_offset, size_t payload_nob); diff --git a/lnet/klnds/lolnd/lolnd_cb.c b/lnet/klnds/lolnd/lolnd_cb.c index 5ff0528..8a24a5c 100644 --- a/lnet/klnds/lolnd/lolnd_cb.c +++ b/lnet/klnds/lolnd/lolnd_cb.c @@ -22,17 +22,17 @@ #include "lonal.h" ptl_err_t -klonal_send (ptl_ni_t *ni, - void *private, - ptl_msg_t *ptlmsg, - ptl_hdr_t *hdr, - int type, - ptl_nid_t nid, - ptl_pid_t pid, - unsigned int payload_niov, - struct iovec *payload_iov, - size_t payload_offset, - size_t payload_nob) +klonal_send (ptl_ni_t *ni, + void *private, + ptl_msg_t *ptlmsg, + ptl_hdr_t *hdr, + int type, + ptl_process_id_t target, + int routing, + unsigned int payload_niov, + struct iovec *payload_iov, + size_t payload_offset, + size_t payload_nob) { klo_desc_t klod = { .klod_type = KLOD_IOV, @@ -42,6 +42,9 @@ klonal_send (ptl_ni_t *ni, .klod_iov = { .iov = payload_iov } }; ptl_err_t rc; + LASSERT (!routing); + LASSERT (target.nid == ni->ni_nid); + rc = ptl_parse(ni, hdr, &klod); if (rc == PTL_OK) ptl_finalize(ni, private, ptlmsg, PTL_OK); @@ -50,17 +53,17 @@ klonal_send (ptl_ni_t *ni, } ptl_err_t -klonal_send_pages (ptl_ni_t *ni, - void *private, - ptl_msg_t *ptlmsg, - ptl_hdr_t *hdr, - int type, - ptl_nid_t nid, - ptl_pid_t pid, - unsigned int payload_niov, - ptl_kiov_t *payload_kiov, - size_t payload_offset, - size_t payload_nob) +klonal_send_pages (ptl_ni_t *ni, + void *private, + ptl_msg_t *ptlmsg, + ptl_hdr_t *hdr, + int type, + ptl_process_id_t target, + int routing, + unsigned int payload_niov, + ptl_kiov_t *payload_kiov, + size_t payload_offset, + size_t payload_nob) { klo_desc_t klod = { .klod_type = KLOD_KIOV, @@ -70,6 +73,9 @@ klonal_send_pages (ptl_ni_t *ni, .klod_iov = { .kiov = payload_kiov } }; ptl_err_t rc; + LASSERT (!routing); + LASSERT (target.nid == ni->ni_nid); + rc = ptl_parse(ni, hdr, &klod); if (rc == PTL_OK) ptl_finalize(ni, private, ptlmsg, PTL_OK); diff --git a/lnet/libcfs/nidstrings.c b/lnet/libcfs/nidstrings.c index 71a5dc5..fc64c99 100644 --- a/lnet/libcfs/nidstrings.c +++ b/lnet/libcfs/nidstrings.c @@ -163,13 +163,25 @@ libcfs_ip_str2addr(char *str, int nob, __u32 *addr) /* known hostname? */ if (('a' <= str[0] && str[0] <= 'z') || ('A' <= str[0] && str[0] <= 'Z')) { - struct hostent *he = gethostbyname(str); - - if (he != NULL) { - __u32 ip = *(__u32 *)he->h_addr; + char *tmp; - *addr = ntohl(ip); - return 1; + PORTAL_ALLOC(tmp, nob + 1); + if (tmp != NULL) { + struct hostent *he; + + memcpy(tmp, str, nob); + tmp[nob] = 0; + + he = gethostbyname(tmp); + + PORTAL_FREE(tmp, nob); + + if (he != NULL) { + __u32 ip = *(__u32 *)he->h_addr; + + *addr = ntohl(ip); + return 1; + } } } #endif diff --git a/lnet/lnet/api-ni.c b/lnet/lnet/api-ni.c index c73068a..20c96b3 100644 --- a/lnet/lnet/api-ni.c +++ b/lnet/lnet/api-ni.c @@ -621,12 +621,13 @@ ptl_shutdown_nalnis (void) list_del (&ni->ni_list); ni->ni_shutdown = 1; - ptl_ni_decref(ni); /* drop apini's ref (shutdown on last ref) */ ptl_apini.apini_nzombie_nis++; - } - PTL_UNLOCK(flags); + PTL_UNLOCK(flags); - PTL_LOCK(flags); + ptl_ni_decref(ni); /* drop apini's ref (shutdown on last ref) */ + + PTL_LOCK(flags); + } /* Now wait for the NI's I just nuked to show up on apini_zombie_nis * and shut them down in guaranteed thread context */ @@ -646,6 +647,7 @@ ptl_shutdown_nalnis (void) ni = list_entry(ptl_apini.apini_zombie_nis.next, ptl_ni_t, ni_list); + list_del(&ni->ni_list); PTL_UNLOCK(flags); @@ -657,6 +659,7 @@ ptl_shutdown_nalnis (void) PTL_LOCK(flags); ptl_apini.apini_nzombie_nis--; } + PTL_UNLOCK(flags); } ptl_err_t @@ -682,12 +685,12 @@ ptl_startup_nalnis (void) break; } + atomic_set(&ni->ni_refcount, 1); + atomic_inc(&nal->nal_refcount); ni->ni_nal = nal; ni->ni_nid = PTL_MKNID(PTL_MKNET(nal->nal_type, 0), 0); /* for now */ - atomic_inc(&nal->nal_refcount); - rc = (nal->nal_startup)(ni, &interface); if (rc != PTL_OK) { CERROR("Error %d staring up NI %s\n", diff --git a/lnet/lnet/lib-move.c b/lnet/lnet/lib-move.c index 6c48234..5658166 100644 --- a/lnet/lnet/lib-move.c +++ b/lnet/lnet/lib-move.c @@ -168,7 +168,6 @@ ptl_fail_nid (ptl_nid_t nid, unsigned int threshold) struct list_head cull; LASSERT (ptl_init); - LASSERT (ptl_apini.apini_refcount > 0); if (threshold != 0) { /* Adding a new entry */ diff --git a/lnet/router/autoMakefile.am b/lnet/router/autoMakefile.am index e00d06e..7fb9ed9 100644 --- a/lnet/router/autoMakefile.am +++ b/lnet/router/autoMakefile.am @@ -3,5 +3,9 @@ # This code is issued under the GNU General Public License. # See the file COPYING in this distribution +# +# router is built into portals +# + MOSTLYCLEANFILES = *.o *.ko *.mod.c -DIST_SOURCES = $(kptlrouter-objs:%.o=%.c) router.h +DIST_SOURCES = router.c proc.c router.h diff --git a/lnet/router/router.c b/lnet/router/router.c index aabd9a0..d3f3faf 100644 --- a/lnet/router/router.c +++ b/lnet/router/router.c @@ -237,6 +237,7 @@ kpr_lookup (ptl_ni_t **nip, ptl_nid_t target_nid, int nob) ptl_nid_t gwnid; struct list_head *e; kpr_route_entry_t *re; + unsigned long flags; ptl_ni_t *gwni = NULL; ptl_ni_t *tmpni = NULL; kpr_gateway_entry_t *ge = NULL; @@ -248,21 +249,23 @@ kpr_lookup (ptl_ni_t **nip, ptl_nid_t target_nid, int nob) CDEBUG (D_NET, "lookup "LPX64" from %s\n", target_nid, (ni == NULL) ? "<>" : libcfs_nid2str(ni->ni_nid)); - LASSERT (!in_interrupt()); if (ni == NULL) { /* ni not determined yet */ gwni = ptl_net2ni(target_net); /* is it a local network? */ if (gwni != NULL) { *nip = gwni; - return ni->ni_nid; + return gwni->ni_nid; } + } else if (target_net == PTL_NIDNET(ni->ni_nid)) { + ptl_ni_addref(ni); /* extra ref so caller can drop blindly */ + return ni->ni_nid; } - read_lock (&kpr_rwlock); + read_lock_irqsave(&kpr_rwlock, flags); if (ni != NULL && ni->ni_shutdown) { /* pre-determined ni is shutting down */ - read_unlock (&kpr_rwlock); + read_unlock_irqrestore(&kpr_rwlock, flags); return PTL_NID_ANY; } @@ -300,7 +303,7 @@ kpr_lookup (ptl_ni_t **nip, ptl_nid_t target_nid, int nob) } if (ge == NULL) { - read_unlock (&kpr_rwlock); + read_unlock_irqrestore(&kpr_rwlock, flags); LASSERT (gwni == NULL); return PTL_NID_ANY; @@ -308,7 +311,7 @@ kpr_lookup (ptl_ni_t **nip, ptl_nid_t target_nid, int nob) kpr_update_weight (ge, nob); gwnid = ge->kpge_nid; - read_unlock (&kpr_rwlock); + read_unlock_irqrestore(&kpr_rwlock, flags); /* NB can't deref 're/ge' after lock released! */ CDEBUG (D_NET, "lookup %s from %s: %s\n", @@ -336,6 +339,7 @@ kpr_fwd_start (ptl_ni_t *src_ni, kpr_fwd_desc_t *fwd) kpr_gateway_entry_t *ge = NULL; ptl_ni_t *dst_ni = NULL; ptl_ni_t *tmp_ni; + unsigned long flags; struct list_head *e; kpr_route_entry_t *re; int rc; @@ -344,11 +348,10 @@ kpr_fwd_start (ptl_ni_t *src_ni, kpr_fwd_desc_t *fwd) libcfs_nid2str(target_nid), libcfs_nid2str(src_ni->ni_nid)); LASSERT (nob == ptl_kiov_nob (fwd->kprfd_niov, fwd->kprfd_kiov)); - LASSERT (!in_interrupt()); fwd->kprfd_src_ni = src_ni; /* stash calling ni */ - read_lock (&kpr_rwlock); + read_lock_irqsave(&kpr_rwlock, flags); kpr_fwd_packets++; /* (loose) stats accounting */ kpr_fwd_bytes += nob + sizeof(ptl_hdr_t); @@ -399,7 +402,7 @@ kpr_fwd_start (ptl_ni_t *src_ni, kpr_fwd_desc_t *fwd) fwd->kprfd_gateway_nid = ge->kpge_nid; atomic_inc (&kpr_queue_depth); - read_unlock (&kpr_rwlock); + read_unlock_irqrestore(&kpr_rwlock, flags); CDEBUG (D_NET, "forward [%p] %s: src ni %s dst ni %s gw %s\n", fwd, libcfs_nid2str(target_nid), @@ -421,7 +424,7 @@ kpr_fwd_start (ptl_ni_t *src_ni, kpr_fwd_desc_t *fwd) (fwd->kprfd_callback)(src_ni, fwd->kprfd_callback_arg, rc); - read_unlock (&kpr_rwlock); + read_unlock_irqrestore(&kpr_rwlock, flags); } void @@ -472,7 +475,7 @@ kpr_add_route (__u32 net, ptl_nid_t gateway_nid) re->kpre_net = net; LASSERT(!in_interrupt()); - write_lock_irqsave (&kpr_rwlock, flags); + write_lock_irqsave(&kpr_rwlock, flags); list_for_each (e, &kpr_gateways) { kpr_gateway_entry_t *ge2 = list_entry(e, kpr_gateway_entry_t, @@ -505,7 +508,7 @@ kpr_add_route (__u32 net, ptl_nid_t gateway_nid) list_add (&re->kpre_list, &kpr_routes); kpr_routes_generation++; - write_unlock_irqrestore (&kpr_rwlock, flags); + write_unlock_irqrestore(&kpr_rwlock, flags); return (0); } @@ -560,9 +563,10 @@ int kpr_get_route (int idx, __u32 *net, ptl_nid_t *gateway_nid, __u32 *alive) { struct list_head *e; + unsigned long flags; LASSERT (!in_interrupt()); - read_lock(&kpr_rwlock); + read_lock_irqsave(&kpr_rwlock, flags); for (e = kpr_routes.next; e != &kpr_routes; e = e->next) { kpr_route_entry_t *re = list_entry(e, kpr_route_entry_t, @@ -574,12 +578,12 @@ kpr_get_route (int idx, __u32 *net, ptl_nid_t *gateway_nid, __u32 *alive) *gateway_nid = ge->kpge_nid; *alive = ge->kpge_alive; - read_unlock(&kpr_rwlock); + read_unlock_irqrestore(&kpr_rwlock, flags); return (0); } } - read_unlock (&kpr_rwlock); + read_unlock_irqrestore(&kpr_rwlock, flags); return (-ENOENT); } -- 1.8.3.1