From 9b5830c9989482e44d64ea76411150046948e51a Mon Sep 17 00:00:00 2001 From: pschwan Date: Tue, 10 Jun 2003 10:48:23 +0000 Subject: [PATCH] b_devel -> b_orphan merge for 20030610 --- lnet/archdep.m4 | 2 +- lnet/include/linux/kp30.h | 16 ++-- lnet/include/lnet/lib-lnet.h | 16 ++-- lnet/include/lnet/lib-p30.h | 16 ++-- lnet/include/lnet/lnetctl.h | 1 + lnet/include/lnet/ptlctl.h | 1 + lnet/klnds/qswlnd/qswlnd.c | 42 ++++++++-- lnet/klnds/qswlnd/qswlnd.h | 21 +++++ lnet/klnds/qswlnd/qswlnd_cb.c | 37 ++++----- lnet/klnds/socklnd/socklnd.c | 11 +-- lnet/klnds/socklnd/socklnd.h | 1 - lnet/klnds/toelnd/toenal.c | 2 +- lnet/klnds/toelnd/toenal_cb.c | 7 +- lnet/router/router.c | 6 +- lnet/ulnds/connection.c | 7 +- lnet/ulnds/connection.h | 12 +-- lnet/ulnds/socklnd/connection.c | 7 +- lnet/ulnds/socklnd/connection.h | 12 +-- lnet/ulnds/socklnd/tcplnd.c | 18 +++-- lnet/ulnds/tcplnd.c | 18 +++-- lnet/utils/portals.c | 130 +++++++++++++------------------ lnet/utils/ptlctl.c | 1 + lustre/portals/archdep.m4 | 2 +- lustre/portals/include/linux/kp30.h | 16 ++-- lustre/portals/include/portals/lib-p30.h | 16 ++-- lustre/portals/include/portals/ptlctl.h | 1 + lustre/portals/knals/qswnal/qswnal.c | 42 ++++++++-- lustre/portals/knals/qswnal/qswnal.h | 21 +++++ lustre/portals/knals/qswnal/qswnal_cb.c | 37 ++++----- lustre/portals/knals/socknal/socknal.c | 11 +-- lustre/portals/knals/socknal/socknal.h | 1 - lustre/portals/knals/toenal/toenal.c | 2 +- lustre/portals/knals/toenal/toenal_cb.c | 7 +- lustre/portals/router/router.c | 6 +- lustre/portals/unals/connection.c | 7 +- lustre/portals/unals/connection.h | 12 +-- lustre/portals/unals/tcpnal.c | 18 +++-- lustre/portals/utils/portals.c | 130 +++++++++++++------------------ lustre/portals/utils/ptlctl.c | 1 + 39 files changed, 380 insertions(+), 334 deletions(-) diff --git a/lnet/archdep.m4 b/lnet/archdep.m4 index 2c7fa59..a2a407c 100644 --- a/lnet/archdep.m4 +++ b/lnet/archdep.m4 @@ -219,7 +219,7 @@ else AC_MSG_RESULT(no) fi -AC_ARG_ENABLE(zerocopy, [ --enable-zerocopy enable socknal zerocopy],enable_zerocopy="-DSOCKNAL_ZC=1", enable_zercopy=$enable_zerocopy_temp) +AC_ARG_ENABLE(zerocopy, [ --enable-zerocopy enable socknal zerocopy],enable_zerocopy="-DSOCKNAL_ZC=1", enable_zerocopy=$enable_zerocopy_temp) AC_ARG_ENABLE(affinity, [ --enable-affinity enable process/irq affinity],enable_affinity="-DCPU_AFFINITY=1", enable_affinity=$enable_affinity_temp) ##################################### diff --git a/lnet/include/linux/kp30.h b/lnet/include/linux/kp30.h index b21167a..d0f3f09 100644 --- a/lnet/include/linux/kp30.h +++ b/lnet/include/linux/kp30.h @@ -83,8 +83,10 @@ extern unsigned int portal_printk; #ifndef __KERNEL__ #define THREAD_SIZE 8192 #endif -#ifdef __arch_ia64__ -#define CDEBUG_STACK() ((unsigned long)__builtin_dwarf_cfa()&(THREAD_SIZE - 1)) +#ifdef __ia64__ +#define CDEBUG_STACK() (THREAD_SIZE - \ + ((unsigned long)__builtin_dwarf_cfa() & \ + (THREAD_SIZE - 1))) #else #define CDEBUG_STACK() (THREAD_SIZE - \ ((unsigned long)__builtin_frame_address(0) & \ @@ -265,7 +267,7 @@ do { \ if (s > PORTAL_VMALLOC_SIZE) \ (ptr) = vmalloc(s); \ else \ - (ptr) = kmalloc(s, GFP_KERNEL); \ + (ptr) = kmalloc(s, GFP_NOFS); \ if ((ptr) == NULL) \ CERROR("PORTALS: out of memory at %s:%d (tried to alloc" \ " '" #ptr "' = %ld)\n", __FILE__, __LINE__, s); \ @@ -465,8 +467,8 @@ kpr_lookup (kpr_router_t *router, ptl_nid_t nid, ptl_nid_t *gateway_nid) } static inline void -kpr_fwd_init (kpr_fwd_desc_t *fwd, ptl_nid_t nid, - int nob, int niov, struct iovec *iov, +kpr_fwd_init (kpr_fwd_desc_t *fwd, ptl_nid_t nid, + int nob, int niov, struct iovec *iov, kpr_fwd_callback_t callback, void *callback_arg) { fwd->kprfd_target_nid = nid; @@ -610,7 +612,7 @@ extern void kportal_blockallsigs (void); # ifdef PORTAL_DEBUG # undef NDEBUG # include -# define LASSERT(e) assert(e) +# define LASSERT(e) assert(e) # else # define LASSERT(e) # endif @@ -914,7 +916,7 @@ void kportal_put_ni (int nal); #ifndef BITS_PER_LONG #if (~0UL) == 0xffffffffUL #define BITS_PER_LONG 32 -#else +#else #define BITS_PER_LONG 64 #endif #endif diff --git a/lnet/include/lnet/lib-lnet.h b/lnet/include/lnet/lib-lnet.h index ca92e5f..b623b93 100644 --- a/lnet/include/lnet/lib-lnet.h +++ b/lnet/include/lnet/lib-lnet.h @@ -183,28 +183,28 @@ static inline lib_eq_t * lib_eq_alloc (nal_cb_t *nal) { /* NEVER called with statelock held */ - lib_eq_t *eq = kmem_cache_alloc(ptl_eq_slab, GFP_KERNEL); - + lib_eq_t *eq = kmem_cache_alloc(ptl_eq_slab, GFP_NOFS); + if (eq == NULL) return (NULL); - + atomic_inc (&eq_in_use_count); return (eq); } -static inline void +static inline void lib_eq_free (nal_cb_t *nal, lib_eq_t *eq) { /* ALWAYS called with statelock held */ atomic_dec (&eq_in_use_count); - kmem_cache_free(ptl_eq_slab, eq); + kmem_cache_free(ptl_eq_slab, eq); } static inline lib_md_t * lib_md_alloc (nal_cb_t *nal) { /* NEVER called with statelock held */ - lib_md_t *md = kmem_cache_alloc(ptl_md_slab, GFP_KERNEL); + lib_md_t *md = kmem_cache_alloc(ptl_md_slab, GFP_NOFS); if (md == NULL) return (NULL); @@ -225,11 +225,11 @@ static inline lib_me_t * lib_me_alloc (nal_cb_t *nal) { /* NEVER called with statelock held */ - lib_me_t *me = kmem_cache_alloc(ptl_me_slab, GFP_KERNEL); + lib_me_t *me = kmem_cache_alloc(ptl_me_slab, GFP_NOFS); if (me == NULL) return (NULL); - + atomic_inc (&me_in_use_count); return (me); } diff --git a/lnet/include/lnet/lib-p30.h b/lnet/include/lnet/lib-p30.h index ca92e5f..b623b93 100644 --- a/lnet/include/lnet/lib-p30.h +++ b/lnet/include/lnet/lib-p30.h @@ -183,28 +183,28 @@ static inline lib_eq_t * lib_eq_alloc (nal_cb_t *nal) { /* NEVER called with statelock held */ - lib_eq_t *eq = kmem_cache_alloc(ptl_eq_slab, GFP_KERNEL); - + lib_eq_t *eq = kmem_cache_alloc(ptl_eq_slab, GFP_NOFS); + if (eq == NULL) return (NULL); - + atomic_inc (&eq_in_use_count); return (eq); } -static inline void +static inline void lib_eq_free (nal_cb_t *nal, lib_eq_t *eq) { /* ALWAYS called with statelock held */ atomic_dec (&eq_in_use_count); - kmem_cache_free(ptl_eq_slab, eq); + kmem_cache_free(ptl_eq_slab, eq); } static inline lib_md_t * lib_md_alloc (nal_cb_t *nal) { /* NEVER called with statelock held */ - lib_md_t *md = kmem_cache_alloc(ptl_md_slab, GFP_KERNEL); + lib_md_t *md = kmem_cache_alloc(ptl_md_slab, GFP_NOFS); if (md == NULL) return (NULL); @@ -225,11 +225,11 @@ static inline lib_me_t * lib_me_alloc (nal_cb_t *nal) { /* NEVER called with statelock held */ - lib_me_t *me = kmem_cache_alloc(ptl_me_slab, GFP_KERNEL); + lib_me_t *me = kmem_cache_alloc(ptl_me_slab, GFP_NOFS); if (me == NULL) return (NULL); - + atomic_inc (&me_in_use_count); return (me); } diff --git a/lnet/include/lnet/lnetctl.h b/lnet/include/lnet/lnetctl.h index fdaae69..dc02780 100644 --- a/lnet/include/lnet/lnetctl.h +++ b/lnet/include/lnet/lnetctl.h @@ -38,6 +38,7 @@ int jt_ptl_connect(int argc, char **argv); int jt_ptl_disconnect(int argc, char **argv); int jt_ptl_push_connection(int argc, char **argv); int jt_ptl_ping(int argc, char **argv); +int jt_ptl_shownid(int argc, char **argv); int jt_ptl_mynid(int argc, char **argv); int jt_ptl_add_uuid(int argc, char **argv); int jt_ptl_add_uuid_old(int argc, char **argv); /* backwards compatibility */ diff --git a/lnet/include/lnet/ptlctl.h b/lnet/include/lnet/ptlctl.h index fdaae69..dc02780 100644 --- a/lnet/include/lnet/ptlctl.h +++ b/lnet/include/lnet/ptlctl.h @@ -38,6 +38,7 @@ int jt_ptl_connect(int argc, char **argv); int jt_ptl_disconnect(int argc, char **argv); int jt_ptl_push_connection(int argc, char **argv); int jt_ptl_ping(int argc, char **argv); +int jt_ptl_shownid(int argc, char **argv); int jt_ptl_mynid(int argc, char **argv); int jt_ptl_add_uuid(int argc, char **argv); int jt_ptl_add_uuid_old(int argc, char **argv); /* backwards compatibility */ diff --git a/lnet/klnds/qswlnd/qswlnd.c b/lnet/klnds/qswlnd/qswlnd.c index d64b7ad..1a8fb74 100644 --- a/lnet/klnds/qswlnd/qswlnd.c +++ b/lnet/klnds/qswlnd/qswlnd.c @@ -101,16 +101,36 @@ static nal_t * kqswnal_init(int interface, ptl_pt_index_t ptl_size, ptl_ac_index_t ac_size, ptl_pid_t requested_pid) { - ptl_nid_t mynid = ep_nodeid (kqswnal_data.kqn_epdev); - int nnids = ep_numnodes (kqswnal_data.kqn_epdev); + ptl_nid_t mynid = kqswnal_elanid2nid (kqswnal_data.kqn_elanid); + int nnids = kqswnal_data.kqn_nnodes; - CDEBUG(D_NET, "calling lib_init with nid "LPX64" of %d\n", mynid,nnids); + CDEBUG(D_NET, "calling lib_init with nid "LPX64" of %d\n", mynid, nnids); lib_init(&kqswnal_lib, mynid, 0, nnids, ptl_size, ac_size); return (&kqswnal_api); } +int +kqswnal_cmd (struct portal_ioctl_data *data, void *private) +{ + LASSERT (data != NULL); + + switch (data->ioc_nal_cmd) { + case NAL_CMD_REGISTER_MYNID: + CDEBUG (D_IOCTL, "setting NID offset to "LPX64" (was "LPX64")\n", + data->ioc_nid - kqswnal_data.kqn_elanid, + kqswnal_data.kqn_nid_offset); + kqswnal_data.kqn_nid_offset = + data->ioc_nid - kqswnal_data.kqn_elanid; + kqswnal_lib.ni.nid = data->ioc_nid; + return (0); + + default: + return (-EINVAL); + } +} + void __exit kqswnal_finalise (void) { @@ -324,6 +344,10 @@ kqswnal_initialise (void) return (-ENOMEM); } + kqswnal_data.kqn_nid_offset = 0; + kqswnal_data.kqn_nnodes = ep_numnodes (kqswnal_data.kqn_epdev); + kqswnal_data.kqn_elanid = ep_nodeid (kqswnal_data.kqn_epdev); + /**********************************************************************/ /* Get the transmitter */ @@ -477,7 +501,7 @@ kqswnal_initialise (void) LASSERT (krx->krx_npages > 0); for (j = 0; j < krx->krx_npages; j++) { - krx->krx_pages[j] = alloc_page (GFP_KERNEL); + krx->krx_pages[j] = alloc_page(GFP_KERNEL); if (krx->krx_pages[j] == NULL) { kqswnal_finalise (); @@ -554,13 +578,19 @@ kqswnal_initialise (void) rc = kpr_register (&kqswnal_data.kqn_router, &kqswnal_router_interface); CDEBUG(D_NET, "Can't initialise routing interface (rc = %d): not routing\n",rc); + rc = kportal_nal_register (QSWNAL, &kqswnal_cmd, NULL); + if (rc != 0) { + CERROR ("Can't initialise command interface (rc = %d)\n", rc); + kqswnal_finalise (); + return (rc); + } + PORTAL_SYMBOL_REGISTER(kqswnal_ni); kqswnal_data.kqn_init = KQN_INIT_ALL; printk(KERN_INFO "Routing QSW NAL loaded on node %d of %d " "(Routing %s, initial mem %d)\n", - ep_nodeid (kqswnal_data.kqn_epdev), - ep_numnodes (kqswnal_data.kqn_epdev), + kqswnal_data.kqn_elanid, kqswnal_data.kqn_nnodes, kpr_routing (&kqswnal_data.kqn_router) ? "enabled" : "disabled", pkmem); diff --git a/lnet/klnds/qswlnd/qswlnd.h b/lnet/klnds/qswlnd/qswlnd.h index 657b02b..88ab74f 100644 --- a/lnet/klnds/qswlnd/qswlnd.h +++ b/lnet/klnds/qswlnd/qswlnd.h @@ -200,6 +200,10 @@ typedef struct ELAN3_DMA_HANDLE *kqn_eptxdmahandle; /* elan reserved tx vaddrs */ ELAN3_DMA_HANDLE *kqn_eprxdmahandle; /* elan reserved rx vaddrs */ kpr_router_t kqn_router; /* connection to Kernel Portals Router module */ + + ptl_nid_t kqn_nid_offset; /* this cluster's NID offset */ + int kqn_nnodes; /* this cluster's size */ + int kqn_elanid; /* this nodes's elan ID */ } kqswnal_data_t; /* kqn_init state */ @@ -217,6 +221,23 @@ extern void kqswnal_rxhandler(EP_RXD *rxd); extern int kqswnal_scheduler (void *); extern void kqswnal_fwd_packet (void *arg, kpr_fwd_desc_t *fwd); +static inline ptl_nid_t +kqswnal_elanid2nid (int elanid) +{ + return (kqswnal_data.kqn_nid_offset + elanid); +} + +static inline int +kqswnal_nid2elanid (ptl_nid_t nid) +{ + /* not in this cluster? */ + if (nid < kqswnal_data.kqn_nid_offset || + nid >= kqswnal_data.kqn_nid_offset + kqswnal_data.kqn_nnodes) + return (-1); + + return (nid - kqswnal_data.kqn_nid_offset); +} + static inline void kqswnal_requeue_rx (kqswnal_rx_t *krx) { diff --git a/lnet/klnds/qswlnd/qswlnd_cb.c b/lnet/klnds/qswlnd/qswlnd_cb.c index 5979885..3b47a25 100644 --- a/lnet/klnds/qswlnd/qswlnd_cb.c +++ b/lnet/klnds/qswlnd/qswlnd_cb.c @@ -109,21 +109,15 @@ kqswnal_sti(nal_cb_t *nal, unsigned long *flags) static int kqswnal_dist(nal_cb_t *nal, ptl_nid_t nid, unsigned long *dist) { - /* network distance doesn't mean much for this nal */ - *dist = (nid == nal->ni.nid) ? 0 : 1; + if (nid == nal->ni.nid) + *dist = 0; /* it's me */ + else if (kqswnal_nid2elanid (nid) >= 0) + *dist = 1; /* it's my peer */ + else + *dist = 2; /* via router */ return (0); } -int -kqswnal_ispeer (ptl_nid_t nid) -{ - unsigned int elanid = (unsigned int)nid; - - /* didn't lose high bits on conversion and it's in this machine? */ - return ((ptl_nid_t)elanid == nid && - elanid < ep_numnodes (kqswnal_data.kqn_epdev)); -} - void kqswnal_unmap_tx (kqswnal_tx_t *ktx) { @@ -453,11 +447,14 @@ kqswnal_launch (kqswnal_tx_t *ktx) { /* Don't block for transmit descriptor if we're in interrupt context */ int attr = in_interrupt() ? (EP_NO_SLEEP | EP_NO_ALLOC) : 0; - int rc = ep_transmit_large(kqswnal_data.kqn_eptx, ktx->ktx_nid, - ktx->ktx_port, attr, kqswnal_txhandler, - ktx, ktx->ktx_iov, ktx->ktx_niov); + int dest = kqswnal_nid2elanid (ktx->ktx_nid); long flags; - + int rc; + + LASSERT (dest >= 0); /* must be a peer */ + rc = ep_transmit_large(kqswnal_data.kqn_eptx, dest, + ktx->ktx_port, attr, kqswnal_txhandler, + ktx, ktx->ktx_iov, ktx->ktx_niov); if (rc == 0) atomic_inc (&kqswnal_packets_launched); @@ -595,7 +592,7 @@ kqswnal_sendmsg (nal_cb_t *nal, return (-1); } - if (!kqswnal_ispeer (nid)) { /* Can't send direct: find gateway? */ + if (kqswnal_nid2elanid (nid) < 0) { /* Can't send direct: find gateway? */ rc = kpr_lookup (&kqswnal_data.kqn_router, nid, &gatewaynid); if (rc != 0) { CERROR("Can't route to "LPX64": router error %d\n", @@ -603,7 +600,7 @@ kqswnal_sendmsg (nal_cb_t *nal, lib_finalize (&kqswnal_lib, private, cookie); return (-1); } - if (!kqswnal_ispeer (gatewaynid)) { + if (kqswnal_nid2elanid (gatewaynid) < 0) { CERROR("Bad gateway "LPX64" for "LPX64"\n", gatewaynid, nid); lib_finalize (&kqswnal_lib, private, cookie); @@ -757,7 +754,7 @@ kqswnal_fwd_packet (void *arg, kpr_fwd_desc_t *fwd) if (nid == kqswnal_lib.ni.nid) /* gateway is me */ nid = fwd->kprfd_target_nid; /* target is final dest */ - if (!kqswnal_ispeer (nid)) { + if (kqswnal_nid2elanid (nid) < 0) { CERROR("Can't forward [%p] to "LPX64": not a peer\n", fwd, nid); rc = -EHOSTUNREACH; goto failed; @@ -844,7 +841,7 @@ kqswnal_rx (kqswnal_rx_t *krx) CERROR ("checksums for forwarded packets not implemented\n"); LBUG (); #endif - if (kqswnal_ispeer (dest_nid)) /* should have gone direct to peer */ + if (kqswnal_nid2elanid (dest_nid) >= 0) /* should have gone direct to peer */ { CERROR("dropping packet from "LPX64" for "LPX64 ": target is peer\n", NTOH__u64(hdr->src_nid), dest_nid); diff --git a/lnet/klnds/socklnd/socklnd.c b/lnet/klnds/socklnd/socklnd.c index d15d8c8..91d971c 100644 --- a/lnet/klnds/socklnd/socklnd.c +++ b/lnet/klnds/socklnd/socklnd.c @@ -95,10 +95,8 @@ nal_t * ksocknal_init(int interface, ptl_pt_index_t ptl_size, ptl_ac_index_t ac_size, ptl_pid_t requested_pid) { - CDEBUG(D_NET, "calling lib_init with nid "LPX64"\n", - ksocknal_data.ksnd_mynid); - lib_init(&ksocknal_lib, ksocknal_data.ksnd_mynid, 0, 10, ptl_size, - ac_size); + CDEBUG(D_NET, "calling lib_init with nid "LPX64"\n", (ptl_nid_t)0); + lib_init(&ksocknal_lib, (ptl_nid_t)0, 0, 10, ptl_size, ac_size); return (&ksocknal_api); } @@ -129,7 +127,6 @@ ksocknal_set_mynid(ptl_nid_t nid) CDEBUG(D_IOCTL, "setting mynid to "LPX64" (old nid="LPX64")\n", nid, ni->nid); - ksocknal_data.ksnd_mynid = nid; ni->nid = nid; return (0); } @@ -190,7 +187,7 @@ ksocknal_add_sock (ptl_nid_t nid, int fd, int bind_irq) if (!conn) GOTO(error, ret); - memset (conn, 0, sizeof (conn)); /* zero for consistency */ + sock->sk->allocation = GFP_NOFS; /* don't call info fs for alloc */ conn->ksnc_file = file; conn->ksnc_sock = sock; @@ -818,7 +815,7 @@ ksocknal_module_init (void) LASSERT (fmb->fmb_npages > 0); for (j = 0; j < fmb->fmb_npages; j++) { - fmb->fmb_pages[j] = alloc_page (GFP_KERNEL); + fmb->fmb_pages[j] = alloc_page(GFP_KERNEL); if (fmb->fmb_pages[j] == NULL) { ksocknal_module_fini (); diff --git a/lnet/klnds/socklnd/socklnd.h b/lnet/klnds/socklnd/socklnd.h index 0bf60ea..86cdeb0 100644 --- a/lnet/klnds/socklnd/socklnd.h +++ b/lnet/klnds/socklnd/socklnd.h @@ -106,7 +106,6 @@ typedef struct { struct list_head ksnd_socklist; /* all my connections */ rwlock_t ksnd_socklist_lock; /* stabilise add/find/remove */ - ptl_nid_t ksnd_mynid; nal_cb_t *ksnd_nal_cb; spinlock_t ksnd_nal_cb_lock; /* lib cli/sti lock */ diff --git a/lnet/klnds/toelnd/toenal.c b/lnet/klnds/toelnd/toenal.c index 178ea41..1f5dc38 100644 --- a/lnet/klnds/toelnd/toenal.c +++ b/lnet/klnds/toelnd/toenal.c @@ -522,7 +522,7 @@ ktoenal_module_init (void) LASSERT (fmb->fmb_npages > 0); for (j = 0; j < fmb->fmb_npages; j++) { - fmb->fmb_pages[j] = alloc_page (GFP_KERNEL); + fmb->fmb_pages[j] = alloc_page(GFP_KERNEL); if (fmb->fmb_pages[j] == NULL) { diff --git a/lnet/klnds/toelnd/toenal_cb.c b/lnet/klnds/toelnd/toenal_cb.c index 8270196..ec37f6f 100644 --- a/lnet/klnds/toelnd/toenal_cb.c +++ b/lnet/klnds/toelnd/toenal_cb.c @@ -423,11 +423,10 @@ ktoenal_send(nal_cb_t *nal, void *private, lib_msg_t *cookie, * packet as a side-effect of another packet, such as when an ACK has * been requested. -phil */ - CDEBUG(D_NET, "sending "LPSZ" bytes from [%d](%p,%d)... to nid: "LPX64" pid %d\n", - payload_len, payload_niov, + CDEBUG(D_NET, "sending %d bytes from [%d](%p,%d)... to nid: " + LPX64" pid %d\n", (int)payload_len, payload_niov, payload_niov > 0 ? payload_iov[0].iov_base : NULL, - payload_niov > 0 ? payload_iov[0].iov_len : 0, - nid, pid); + (int)(payload_niov > 0 ? payload_iov[0].iov_len : 0), nid, pid); if ((conn = ktoenal_get_conn (nid)) == NULL) { diff --git a/lnet/router/router.c b/lnet/router/router.c index 8a1de08..6074c3c 100644 --- a/lnet/router/router.c +++ b/lnet/router/router.c @@ -202,15 +202,15 @@ kpr_forward_packet (void *arg, kpr_fwd_desc_t *fwd) LASSERT (nob == lib_iov_nob (fwd->kprfd_niov, fwd->kprfd_iov)); atomic_inc (&kpr_queue_depth); + atomic_inc (&src_ne->kpne_refcount); /* source nal is busy until fwd completes */ kpr_fwd_packets++; /* (loose) stats accounting */ kpr_fwd_bytes += nob; - if (src_ne->kpne_shutdown) /* caller is shutting down */ + if (src_ne->kpne_shutdown) /* caller is shutting down */ goto out; - fwd->kprfd_router_arg = src_ne; /* stash caller's nal entry */ - atomic_inc (&src_ne->kpne_refcount); /* source nal is busy until fwd completes */ + fwd->kprfd_router_arg = src_ne; /* stash caller's nal entry */ read_lock (&kpr_rwlock); diff --git a/lnet/ulnds/connection.c b/lnet/ulnds/connection.c index 89c9f78..310e899 100644 --- a/lnet/ulnds/connection.c +++ b/lnet/ulnds/connection.c @@ -120,9 +120,10 @@ int read_connection(connection c, return(1); } -static int connection_input(connection c) +static int connection_input(void *d) { - return((*c->m->handler)(c->m->handler_arg,c)); + connection c = d; + return((*c->m->handler)(c->m->handler_arg,c)); } @@ -280,7 +281,7 @@ void shutdown_connections(manager m) * zero if the fixed port could not be bound */ manager init_connections(unsigned short pid, - int (*input)(), + int (*input)(void *, void *), void *a) { manager m=(manager)malloc(sizeof(struct manager)); diff --git a/lnet/ulnds/connection.h b/lnet/ulnds/connection.h index f6b2994..6f57287 100644 --- a/lnet/ulnds/connection.h +++ b/lnet/ulnds/connection.h @@ -25,14 +25,8 @@ typedef struct connection { manager m; } *connection; -connection force_tcp_connection(manager m, - unsigned int ip, - unsigned int short); -manager init_connections(unsigned short, - int (*f)(void *,connection), - void *); +connection force_tcp_connection(manager m, unsigned int ip, unsigned int short); +manager init_connections(unsigned short, int (*f)(void *, void *), void *); void remove_connection(void *arg); void shutdown_connections(manager m); -int read_connection(connection c, - unsigned char *dest, - int len); +int read_connection(connection c, unsigned char *dest, int len); diff --git a/lnet/ulnds/socklnd/connection.c b/lnet/ulnds/socklnd/connection.c index 89c9f78..310e899 100644 --- a/lnet/ulnds/socklnd/connection.c +++ b/lnet/ulnds/socklnd/connection.c @@ -120,9 +120,10 @@ int read_connection(connection c, return(1); } -static int connection_input(connection c) +static int connection_input(void *d) { - return((*c->m->handler)(c->m->handler_arg,c)); + connection c = d; + return((*c->m->handler)(c->m->handler_arg,c)); } @@ -280,7 +281,7 @@ void shutdown_connections(manager m) * zero if the fixed port could not be bound */ manager init_connections(unsigned short pid, - int (*input)(), + int (*input)(void *, void *), void *a) { manager m=(manager)malloc(sizeof(struct manager)); diff --git a/lnet/ulnds/socklnd/connection.h b/lnet/ulnds/socklnd/connection.h index f6b2994..6f57287 100644 --- a/lnet/ulnds/socklnd/connection.h +++ b/lnet/ulnds/socklnd/connection.h @@ -25,14 +25,8 @@ typedef struct connection { manager m; } *connection; -connection force_tcp_connection(manager m, - unsigned int ip, - unsigned int short); -manager init_connections(unsigned short, - int (*f)(void *,connection), - void *); +connection force_tcp_connection(manager m, unsigned int ip, unsigned int short); +manager init_connections(unsigned short, int (*f)(void *, void *), void *); void remove_connection(void *arg); void shutdown_connections(manager m); -int read_connection(connection c, - unsigned char *dest, - int len); +int read_connection(connection c, unsigned char *dest, int len); diff --git a/lnet/ulnds/socklnd/tcplnd.c b/lnet/ulnds/socklnd/tcplnd.c index 8bf55c4..534fc17 100644 --- a/lnet/ulnds/socklnd/tcplnd.c +++ b/lnet/ulnds/socklnd/tcplnd.c @@ -151,15 +151,17 @@ int tcpnal_recv(nal_cb_t *n, * available. It attempts to read the portals header and * pass it to the generic library for processing. */ -static int from_connection(void *a,connection c) +static int from_connection(void *a, void *d) { - bridge b=a; - ptl_hdr_t hdr; - if (read_connection(c, (unsigned char *)&hdr, sizeof(hdr))){ - lib_parse(b->nal_cb, &hdr, c); - return(1); - } - return(0); + connection c = d; + bridge b=a; + ptl_hdr_t hdr; + + if (read_connection(c, (unsigned char *)&hdr, sizeof(hdr))){ + lib_parse(b->nal_cb, &hdr, c); + return(1); + } + return(0); } diff --git a/lnet/ulnds/tcplnd.c b/lnet/ulnds/tcplnd.c index 8bf55c4..534fc17 100644 --- a/lnet/ulnds/tcplnd.c +++ b/lnet/ulnds/tcplnd.c @@ -151,15 +151,17 @@ int tcpnal_recv(nal_cb_t *n, * available. It attempts to read the portals header and * pass it to the generic library for processing. */ -static int from_connection(void *a,connection c) +static int from_connection(void *a, void *d) { - bridge b=a; - ptl_hdr_t hdr; - if (read_connection(c, (unsigned char *)&hdr, sizeof(hdr))){ - lib_parse(b->nal_cb, &hdr, c); - return(1); - } - return(0); + connection c = d; + bridge b=a; + ptl_hdr_t hdr; + + if (read_connection(c, (unsigned char *)&hdr, sizeof(hdr))){ + lib_parse(b->nal_cb, &hdr, c); + return(1); + } + return(0); } diff --git a/lnet/utils/portals.c b/lnet/utils/portals.c index 8235271..90d66f5 100644 --- a/lnet/utils/portals.c +++ b/lnet/utils/portals.c @@ -106,13 +106,6 @@ nal2name (int nal) return ((e == NULL) ? "???" : e->name); } -static int -nid2nal (ptl_nid_t nid) -{ - /* BIG pragmatic assumption */ - return ((((__u32)nid) & 0xffff0000) != 0 ? SOCKNAL : QSWNAL); -} - int ptl_parse_nid (ptl_nid_t *nidp, char *str) { @@ -160,35 +153,15 @@ ptl_parse_nid (ptl_nid_t *nidp, char *str) char * ptl_nid2str (char *buffer, ptl_nid_t nid) { - switch (nid2nal(nid)) - { - case QSWNAL: - sprintf (buffer, LPD64, nid); - return (buffer); + __u32 addr = htonl((__u32)nid); /* back to NETWORK byte order */ + struct hostent *he = gethostbyaddr ((const char *)&addr, sizeof (addr), AF_INET); - case SCIMACNAL: - sprintf (buffer, LPX64, nid); - return (buffer); - - case SOCKNAL: { - __u32 addr = htonl((__u32)nid); /* back to NETWORK byte order */ - struct hostent *he = gethostbyaddr ((const char *)&addr, sizeof (addr), AF_INET); - - if (he != NULL) - strcpy (buffer, he->h_name); - else - { - addr = (__u32)nid; - sprintf (buffer, "%d.%d.%d.%d", - (addr>>24)&0xff, (addr>>16)&0xff, (addr>>8)&0xff, addr&0xff); - } - return (buffer); - } + if (he != NULL) + strcpy (buffer, he->h_name); + else + sprintf (buffer, "0x"LPX64, nid); - default: - sprintf (buffer, "nid2nal broken"); - return (buffer); - } + return (buffer); } int @@ -704,18 +677,43 @@ int jt_ptl_ping(int argc, char **argv) return 0; } +int jt_ptl_shownid(int argc, char **argv) +{ + struct portal_ioctl_data data; + int rc; + + if (argc > 1) { + fprintf(stderr, "usage: %s\n", argv[0]); + return 0; + } + + if (g_nal == 0) { + fprintf(stderr, "Error: you must run the 'network' command first\n"); + return -1; + } + + PORTAL_IOC_INIT (data); + data.ioc_nal = g_nal; + rc = l_ioctl(PORTALS_DEV_ID, IOC_PORTAL_GET_NID, &data); + if (rc < 0) + fprintf(stderr, "getting my NID failed: %s\n", + strerror (errno)); + else + printf(LPX64"\n", data.ioc_nid); + return 0; +} + int jt_ptl_mynid(int argc, char **argv) { int rc; - struct hostent *h; - char buf[1024], *hostname; + char hostname[1024]; + char *nidstr; struct portal_ioctl_data data; ptl_nid_t mynid; if (argc > 2) { - fprintf(stderr, "usage: %s [hostname]\n", argv[0]); - fprintf(stderr, "hostname defaults to the hostname of the " - "machine.\n"); + fprintf(stderr, "usage: %s [NID]\n", argv[0]); + fprintf(stderr, "NID defaults to the primary IP address of the machine.\n"); return 0; } @@ -725,42 +723,21 @@ int jt_ptl_mynid(int argc, char **argv) return -1; } - if (g_nal == QSWNAL) { - fprintf(stderr, "'mynid' doesn't make any sense for elan.\n"); - return -1; - } else if (g_nal == GMNAL) { - fprintf(stderr, "'mynid' doesn't make any sense for GM.\n"); - return -1; - } else if (g_nal == SCIMACNAL) { - fprintf(stderr, "'mynid' doesn't make any sense for SCI.\n"); - return -1; - } - - if (g_nal != SOCKNAL && g_nal != TOENAL) { - fprintf(stderr, "This should never happen. Also it is very " - "bad.\n"); + if (argc >= 2) + nidstr = argv[1]; + else if (gethostname(hostname, sizeof(hostname)) != 0) { + fprintf(stderr, "gethostname failed: %s\n", + strerror(errno)); return -1; } + else + nidstr = hostname; - if (argc == 1) { - if (gethostname(buf, sizeof(buf)) != 0) { - fprintf(stderr, "gethostname failed: %s\n", - strerror(errno)); - return -1; - } - hostname = buf; - } else { - hostname = argv[1]; - } - - h = gethostbyname(hostname); - - if (!h) { - fprintf(stderr, "cannot get address for host '%s': %d\n", - hostname, h_errno); + rc = ptl_parse_nid (&mynid, nidstr); + if (rc != 0) { + fprintf (stderr, "Can't convert '%s' into a NID\n", nidstr); return -1; } - mynid = (ptl_nid_t)ntohl (*(__u32 *)h->h_addr); /* HOST byte order */ PORTAL_IOC_INIT(data); data.ioc_nid = mynid; @@ -769,7 +746,7 @@ int jt_ptl_mynid(int argc, char **argv) rc = l_ioctl(PORTALS_DEV_ID, IOC_PORTAL_NAL_CMD, &data); if (rc < 0) - fprintf(stderr, "IOC_PORTAL_REGISTER_MYNID failed: %s\n", + fprintf(stderr, "setting my NID failed: %s\n", strerror(errno)); else printf("registered my nid "LPX64" (%s)\n", mynid, hostname); @@ -888,7 +865,6 @@ jt_ptl_add_route (int argc, char **argv) ptl_nid_t nid1; ptl_nid_t nid2; ptl_nid_t gateway_nid; - int gateway_nal; int rc; if (argc < 3) @@ -897,14 +873,18 @@ jt_ptl_add_route (int argc, char **argv) return (0); } + if (g_nal == 0) { + fprintf(stderr, "Error: you must run the 'network' command " + "first.\n"); + return (-1); + } + if (ptl_parse_nid (&gateway_nid, argv[1]) != 0) { fprintf (stderr, "Can't parse gateway NID \"%s\"\n", argv[1]); return (-1); } - gateway_nal = nid2nal (gateway_nid); - if (ptl_parse_nid (&nid1, argv[2]) != 0) { fprintf (stderr, "Can't parse first target NID \"%s\"\n", argv[2]); @@ -921,7 +901,7 @@ jt_ptl_add_route (int argc, char **argv) PORTAL_IOC_INIT(data); data.ioc_nid = gateway_nid; - data.ioc_nal = gateway_nal; + data.ioc_nal = g_nal; data.ioc_nid2 = MIN (nid1, nid2); data.ioc_nid3 = MAX (nid1, nid2); diff --git a/lnet/utils/ptlctl.c b/lnet/utils/ptlctl.c index d38bd4a..8c56d93 100644 --- a/lnet/utils/ptlctl.c +++ b/lnet/utils/ptlctl.c @@ -34,6 +34,7 @@ command_t list[] = { {"disconnect", jt_ptl_disconnect, 0, "disconnect from a remote nid (args: [hostname]"}, {"push", jt_ptl_push_connection, 0, "flush connection to a remote nid (args: [hostname]"}, {"ping", jt_ptl_ping, 0, "do a ping test (args: nid [count] [size] [timeout])"}, + {"shownid", jt_ptl_shownid, 0, "print the local NID"}, {"mynid", jt_ptl_mynid, 0, "inform the socknal of the local NID (args: [hostname])"}, {"add_route", jt_ptl_add_route, 0, "add an entry to the routing table (args: gatewayNID targetNID [targetNID])"}, {"del_route", jt_ptl_del_route, 0, "delete an entry from the routing table (args: targetNID"}, diff --git a/lustre/portals/archdep.m4 b/lustre/portals/archdep.m4 index 2c7fa59..a2a407c 100644 --- a/lustre/portals/archdep.m4 +++ b/lustre/portals/archdep.m4 @@ -219,7 +219,7 @@ else AC_MSG_RESULT(no) fi -AC_ARG_ENABLE(zerocopy, [ --enable-zerocopy enable socknal zerocopy],enable_zerocopy="-DSOCKNAL_ZC=1", enable_zercopy=$enable_zerocopy_temp) +AC_ARG_ENABLE(zerocopy, [ --enable-zerocopy enable socknal zerocopy],enable_zerocopy="-DSOCKNAL_ZC=1", enable_zerocopy=$enable_zerocopy_temp) AC_ARG_ENABLE(affinity, [ --enable-affinity enable process/irq affinity],enable_affinity="-DCPU_AFFINITY=1", enable_affinity=$enable_affinity_temp) ##################################### diff --git a/lustre/portals/include/linux/kp30.h b/lustre/portals/include/linux/kp30.h index b21167a..d0f3f09 100644 --- a/lustre/portals/include/linux/kp30.h +++ b/lustre/portals/include/linux/kp30.h @@ -83,8 +83,10 @@ extern unsigned int portal_printk; #ifndef __KERNEL__ #define THREAD_SIZE 8192 #endif -#ifdef __arch_ia64__ -#define CDEBUG_STACK() ((unsigned long)__builtin_dwarf_cfa()&(THREAD_SIZE - 1)) +#ifdef __ia64__ +#define CDEBUG_STACK() (THREAD_SIZE - \ + ((unsigned long)__builtin_dwarf_cfa() & \ + (THREAD_SIZE - 1))) #else #define CDEBUG_STACK() (THREAD_SIZE - \ ((unsigned long)__builtin_frame_address(0) & \ @@ -265,7 +267,7 @@ do { \ if (s > PORTAL_VMALLOC_SIZE) \ (ptr) = vmalloc(s); \ else \ - (ptr) = kmalloc(s, GFP_KERNEL); \ + (ptr) = kmalloc(s, GFP_NOFS); \ if ((ptr) == NULL) \ CERROR("PORTALS: out of memory at %s:%d (tried to alloc" \ " '" #ptr "' = %ld)\n", __FILE__, __LINE__, s); \ @@ -465,8 +467,8 @@ kpr_lookup (kpr_router_t *router, ptl_nid_t nid, ptl_nid_t *gateway_nid) } static inline void -kpr_fwd_init (kpr_fwd_desc_t *fwd, ptl_nid_t nid, - int nob, int niov, struct iovec *iov, +kpr_fwd_init (kpr_fwd_desc_t *fwd, ptl_nid_t nid, + int nob, int niov, struct iovec *iov, kpr_fwd_callback_t callback, void *callback_arg) { fwd->kprfd_target_nid = nid; @@ -610,7 +612,7 @@ extern void kportal_blockallsigs (void); # ifdef PORTAL_DEBUG # undef NDEBUG # include -# define LASSERT(e) assert(e) +# define LASSERT(e) assert(e) # else # define LASSERT(e) # endif @@ -914,7 +916,7 @@ void kportal_put_ni (int nal); #ifndef BITS_PER_LONG #if (~0UL) == 0xffffffffUL #define BITS_PER_LONG 32 -#else +#else #define BITS_PER_LONG 64 #endif #endif diff --git a/lustre/portals/include/portals/lib-p30.h b/lustre/portals/include/portals/lib-p30.h index ca92e5f..b623b93 100644 --- a/lustre/portals/include/portals/lib-p30.h +++ b/lustre/portals/include/portals/lib-p30.h @@ -183,28 +183,28 @@ static inline lib_eq_t * lib_eq_alloc (nal_cb_t *nal) { /* NEVER called with statelock held */ - lib_eq_t *eq = kmem_cache_alloc(ptl_eq_slab, GFP_KERNEL); - + lib_eq_t *eq = kmem_cache_alloc(ptl_eq_slab, GFP_NOFS); + if (eq == NULL) return (NULL); - + atomic_inc (&eq_in_use_count); return (eq); } -static inline void +static inline void lib_eq_free (nal_cb_t *nal, lib_eq_t *eq) { /* ALWAYS called with statelock held */ atomic_dec (&eq_in_use_count); - kmem_cache_free(ptl_eq_slab, eq); + kmem_cache_free(ptl_eq_slab, eq); } static inline lib_md_t * lib_md_alloc (nal_cb_t *nal) { /* NEVER called with statelock held */ - lib_md_t *md = kmem_cache_alloc(ptl_md_slab, GFP_KERNEL); + lib_md_t *md = kmem_cache_alloc(ptl_md_slab, GFP_NOFS); if (md == NULL) return (NULL); @@ -225,11 +225,11 @@ static inline lib_me_t * lib_me_alloc (nal_cb_t *nal) { /* NEVER called with statelock held */ - lib_me_t *me = kmem_cache_alloc(ptl_me_slab, GFP_KERNEL); + lib_me_t *me = kmem_cache_alloc(ptl_me_slab, GFP_NOFS); if (me == NULL) return (NULL); - + atomic_inc (&me_in_use_count); return (me); } diff --git a/lustre/portals/include/portals/ptlctl.h b/lustre/portals/include/portals/ptlctl.h index fdaae69..dc02780 100644 --- a/lustre/portals/include/portals/ptlctl.h +++ b/lustre/portals/include/portals/ptlctl.h @@ -38,6 +38,7 @@ int jt_ptl_connect(int argc, char **argv); int jt_ptl_disconnect(int argc, char **argv); int jt_ptl_push_connection(int argc, char **argv); int jt_ptl_ping(int argc, char **argv); +int jt_ptl_shownid(int argc, char **argv); int jt_ptl_mynid(int argc, char **argv); int jt_ptl_add_uuid(int argc, char **argv); int jt_ptl_add_uuid_old(int argc, char **argv); /* backwards compatibility */ diff --git a/lustre/portals/knals/qswnal/qswnal.c b/lustre/portals/knals/qswnal/qswnal.c index d64b7ad..1a8fb74 100644 --- a/lustre/portals/knals/qswnal/qswnal.c +++ b/lustre/portals/knals/qswnal/qswnal.c @@ -101,16 +101,36 @@ static nal_t * kqswnal_init(int interface, ptl_pt_index_t ptl_size, ptl_ac_index_t ac_size, ptl_pid_t requested_pid) { - ptl_nid_t mynid = ep_nodeid (kqswnal_data.kqn_epdev); - int nnids = ep_numnodes (kqswnal_data.kqn_epdev); + ptl_nid_t mynid = kqswnal_elanid2nid (kqswnal_data.kqn_elanid); + int nnids = kqswnal_data.kqn_nnodes; - CDEBUG(D_NET, "calling lib_init with nid "LPX64" of %d\n", mynid,nnids); + CDEBUG(D_NET, "calling lib_init with nid "LPX64" of %d\n", mynid, nnids); lib_init(&kqswnal_lib, mynid, 0, nnids, ptl_size, ac_size); return (&kqswnal_api); } +int +kqswnal_cmd (struct portal_ioctl_data *data, void *private) +{ + LASSERT (data != NULL); + + switch (data->ioc_nal_cmd) { + case NAL_CMD_REGISTER_MYNID: + CDEBUG (D_IOCTL, "setting NID offset to "LPX64" (was "LPX64")\n", + data->ioc_nid - kqswnal_data.kqn_elanid, + kqswnal_data.kqn_nid_offset); + kqswnal_data.kqn_nid_offset = + data->ioc_nid - kqswnal_data.kqn_elanid; + kqswnal_lib.ni.nid = data->ioc_nid; + return (0); + + default: + return (-EINVAL); + } +} + void __exit kqswnal_finalise (void) { @@ -324,6 +344,10 @@ kqswnal_initialise (void) return (-ENOMEM); } + kqswnal_data.kqn_nid_offset = 0; + kqswnal_data.kqn_nnodes = ep_numnodes (kqswnal_data.kqn_epdev); + kqswnal_data.kqn_elanid = ep_nodeid (kqswnal_data.kqn_epdev); + /**********************************************************************/ /* Get the transmitter */ @@ -477,7 +501,7 @@ kqswnal_initialise (void) LASSERT (krx->krx_npages > 0); for (j = 0; j < krx->krx_npages; j++) { - krx->krx_pages[j] = alloc_page (GFP_KERNEL); + krx->krx_pages[j] = alloc_page(GFP_KERNEL); if (krx->krx_pages[j] == NULL) { kqswnal_finalise (); @@ -554,13 +578,19 @@ kqswnal_initialise (void) rc = kpr_register (&kqswnal_data.kqn_router, &kqswnal_router_interface); CDEBUG(D_NET, "Can't initialise routing interface (rc = %d): not routing\n",rc); + rc = kportal_nal_register (QSWNAL, &kqswnal_cmd, NULL); + if (rc != 0) { + CERROR ("Can't initialise command interface (rc = %d)\n", rc); + kqswnal_finalise (); + return (rc); + } + PORTAL_SYMBOL_REGISTER(kqswnal_ni); kqswnal_data.kqn_init = KQN_INIT_ALL; printk(KERN_INFO "Routing QSW NAL loaded on node %d of %d " "(Routing %s, initial mem %d)\n", - ep_nodeid (kqswnal_data.kqn_epdev), - ep_numnodes (kqswnal_data.kqn_epdev), + kqswnal_data.kqn_elanid, kqswnal_data.kqn_nnodes, kpr_routing (&kqswnal_data.kqn_router) ? "enabled" : "disabled", pkmem); diff --git a/lustre/portals/knals/qswnal/qswnal.h b/lustre/portals/knals/qswnal/qswnal.h index 657b02b..88ab74f 100644 --- a/lustre/portals/knals/qswnal/qswnal.h +++ b/lustre/portals/knals/qswnal/qswnal.h @@ -200,6 +200,10 @@ typedef struct ELAN3_DMA_HANDLE *kqn_eptxdmahandle; /* elan reserved tx vaddrs */ ELAN3_DMA_HANDLE *kqn_eprxdmahandle; /* elan reserved rx vaddrs */ kpr_router_t kqn_router; /* connection to Kernel Portals Router module */ + + ptl_nid_t kqn_nid_offset; /* this cluster's NID offset */ + int kqn_nnodes; /* this cluster's size */ + int kqn_elanid; /* this nodes's elan ID */ } kqswnal_data_t; /* kqn_init state */ @@ -217,6 +221,23 @@ extern void kqswnal_rxhandler(EP_RXD *rxd); extern int kqswnal_scheduler (void *); extern void kqswnal_fwd_packet (void *arg, kpr_fwd_desc_t *fwd); +static inline ptl_nid_t +kqswnal_elanid2nid (int elanid) +{ + return (kqswnal_data.kqn_nid_offset + elanid); +} + +static inline int +kqswnal_nid2elanid (ptl_nid_t nid) +{ + /* not in this cluster? */ + if (nid < kqswnal_data.kqn_nid_offset || + nid >= kqswnal_data.kqn_nid_offset + kqswnal_data.kqn_nnodes) + return (-1); + + return (nid - kqswnal_data.kqn_nid_offset); +} + static inline void kqswnal_requeue_rx (kqswnal_rx_t *krx) { diff --git a/lustre/portals/knals/qswnal/qswnal_cb.c b/lustre/portals/knals/qswnal/qswnal_cb.c index 5979885..3b47a25 100644 --- a/lustre/portals/knals/qswnal/qswnal_cb.c +++ b/lustre/portals/knals/qswnal/qswnal_cb.c @@ -109,21 +109,15 @@ kqswnal_sti(nal_cb_t *nal, unsigned long *flags) static int kqswnal_dist(nal_cb_t *nal, ptl_nid_t nid, unsigned long *dist) { - /* network distance doesn't mean much for this nal */ - *dist = (nid == nal->ni.nid) ? 0 : 1; + if (nid == nal->ni.nid) + *dist = 0; /* it's me */ + else if (kqswnal_nid2elanid (nid) >= 0) + *dist = 1; /* it's my peer */ + else + *dist = 2; /* via router */ return (0); } -int -kqswnal_ispeer (ptl_nid_t nid) -{ - unsigned int elanid = (unsigned int)nid; - - /* didn't lose high bits on conversion and it's in this machine? */ - return ((ptl_nid_t)elanid == nid && - elanid < ep_numnodes (kqswnal_data.kqn_epdev)); -} - void kqswnal_unmap_tx (kqswnal_tx_t *ktx) { @@ -453,11 +447,14 @@ kqswnal_launch (kqswnal_tx_t *ktx) { /* Don't block for transmit descriptor if we're in interrupt context */ int attr = in_interrupt() ? (EP_NO_SLEEP | EP_NO_ALLOC) : 0; - int rc = ep_transmit_large(kqswnal_data.kqn_eptx, ktx->ktx_nid, - ktx->ktx_port, attr, kqswnal_txhandler, - ktx, ktx->ktx_iov, ktx->ktx_niov); + int dest = kqswnal_nid2elanid (ktx->ktx_nid); long flags; - + int rc; + + LASSERT (dest >= 0); /* must be a peer */ + rc = ep_transmit_large(kqswnal_data.kqn_eptx, dest, + ktx->ktx_port, attr, kqswnal_txhandler, + ktx, ktx->ktx_iov, ktx->ktx_niov); if (rc == 0) atomic_inc (&kqswnal_packets_launched); @@ -595,7 +592,7 @@ kqswnal_sendmsg (nal_cb_t *nal, return (-1); } - if (!kqswnal_ispeer (nid)) { /* Can't send direct: find gateway? */ + if (kqswnal_nid2elanid (nid) < 0) { /* Can't send direct: find gateway? */ rc = kpr_lookup (&kqswnal_data.kqn_router, nid, &gatewaynid); if (rc != 0) { CERROR("Can't route to "LPX64": router error %d\n", @@ -603,7 +600,7 @@ kqswnal_sendmsg (nal_cb_t *nal, lib_finalize (&kqswnal_lib, private, cookie); return (-1); } - if (!kqswnal_ispeer (gatewaynid)) { + if (kqswnal_nid2elanid (gatewaynid) < 0) { CERROR("Bad gateway "LPX64" for "LPX64"\n", gatewaynid, nid); lib_finalize (&kqswnal_lib, private, cookie); @@ -757,7 +754,7 @@ kqswnal_fwd_packet (void *arg, kpr_fwd_desc_t *fwd) if (nid == kqswnal_lib.ni.nid) /* gateway is me */ nid = fwd->kprfd_target_nid; /* target is final dest */ - if (!kqswnal_ispeer (nid)) { + if (kqswnal_nid2elanid (nid) < 0) { CERROR("Can't forward [%p] to "LPX64": not a peer\n", fwd, nid); rc = -EHOSTUNREACH; goto failed; @@ -844,7 +841,7 @@ kqswnal_rx (kqswnal_rx_t *krx) CERROR ("checksums for forwarded packets not implemented\n"); LBUG (); #endif - if (kqswnal_ispeer (dest_nid)) /* should have gone direct to peer */ + if (kqswnal_nid2elanid (dest_nid) >= 0) /* should have gone direct to peer */ { CERROR("dropping packet from "LPX64" for "LPX64 ": target is peer\n", NTOH__u64(hdr->src_nid), dest_nid); diff --git a/lustre/portals/knals/socknal/socknal.c b/lustre/portals/knals/socknal/socknal.c index d15d8c8..91d971c 100644 --- a/lustre/portals/knals/socknal/socknal.c +++ b/lustre/portals/knals/socknal/socknal.c @@ -95,10 +95,8 @@ nal_t * ksocknal_init(int interface, ptl_pt_index_t ptl_size, ptl_ac_index_t ac_size, ptl_pid_t requested_pid) { - CDEBUG(D_NET, "calling lib_init with nid "LPX64"\n", - ksocknal_data.ksnd_mynid); - lib_init(&ksocknal_lib, ksocknal_data.ksnd_mynid, 0, 10, ptl_size, - ac_size); + CDEBUG(D_NET, "calling lib_init with nid "LPX64"\n", (ptl_nid_t)0); + lib_init(&ksocknal_lib, (ptl_nid_t)0, 0, 10, ptl_size, ac_size); return (&ksocknal_api); } @@ -129,7 +127,6 @@ ksocknal_set_mynid(ptl_nid_t nid) CDEBUG(D_IOCTL, "setting mynid to "LPX64" (old nid="LPX64")\n", nid, ni->nid); - ksocknal_data.ksnd_mynid = nid; ni->nid = nid; return (0); } @@ -190,7 +187,7 @@ ksocknal_add_sock (ptl_nid_t nid, int fd, int bind_irq) if (!conn) GOTO(error, ret); - memset (conn, 0, sizeof (conn)); /* zero for consistency */ + sock->sk->allocation = GFP_NOFS; /* don't call info fs for alloc */ conn->ksnc_file = file; conn->ksnc_sock = sock; @@ -818,7 +815,7 @@ ksocknal_module_init (void) LASSERT (fmb->fmb_npages > 0); for (j = 0; j < fmb->fmb_npages; j++) { - fmb->fmb_pages[j] = alloc_page (GFP_KERNEL); + fmb->fmb_pages[j] = alloc_page(GFP_KERNEL); if (fmb->fmb_pages[j] == NULL) { ksocknal_module_fini (); diff --git a/lustre/portals/knals/socknal/socknal.h b/lustre/portals/knals/socknal/socknal.h index 0bf60ea..86cdeb0 100644 --- a/lustre/portals/knals/socknal/socknal.h +++ b/lustre/portals/knals/socknal/socknal.h @@ -106,7 +106,6 @@ typedef struct { struct list_head ksnd_socklist; /* all my connections */ rwlock_t ksnd_socklist_lock; /* stabilise add/find/remove */ - ptl_nid_t ksnd_mynid; nal_cb_t *ksnd_nal_cb; spinlock_t ksnd_nal_cb_lock; /* lib cli/sti lock */ diff --git a/lustre/portals/knals/toenal/toenal.c b/lustre/portals/knals/toenal/toenal.c index 178ea41..1f5dc38 100644 --- a/lustre/portals/knals/toenal/toenal.c +++ b/lustre/portals/knals/toenal/toenal.c @@ -522,7 +522,7 @@ ktoenal_module_init (void) LASSERT (fmb->fmb_npages > 0); for (j = 0; j < fmb->fmb_npages; j++) { - fmb->fmb_pages[j] = alloc_page (GFP_KERNEL); + fmb->fmb_pages[j] = alloc_page(GFP_KERNEL); if (fmb->fmb_pages[j] == NULL) { diff --git a/lustre/portals/knals/toenal/toenal_cb.c b/lustre/portals/knals/toenal/toenal_cb.c index 8270196..ec37f6f 100644 --- a/lustre/portals/knals/toenal/toenal_cb.c +++ b/lustre/portals/knals/toenal/toenal_cb.c @@ -423,11 +423,10 @@ ktoenal_send(nal_cb_t *nal, void *private, lib_msg_t *cookie, * packet as a side-effect of another packet, such as when an ACK has * been requested. -phil */ - CDEBUG(D_NET, "sending "LPSZ" bytes from [%d](%p,%d)... to nid: "LPX64" pid %d\n", - payload_len, payload_niov, + CDEBUG(D_NET, "sending %d bytes from [%d](%p,%d)... to nid: " + LPX64" pid %d\n", (int)payload_len, payload_niov, payload_niov > 0 ? payload_iov[0].iov_base : NULL, - payload_niov > 0 ? payload_iov[0].iov_len : 0, - nid, pid); + (int)(payload_niov > 0 ? payload_iov[0].iov_len : 0), nid, pid); if ((conn = ktoenal_get_conn (nid)) == NULL) { diff --git a/lustre/portals/router/router.c b/lustre/portals/router/router.c index 8a1de08..6074c3c 100644 --- a/lustre/portals/router/router.c +++ b/lustre/portals/router/router.c @@ -202,15 +202,15 @@ kpr_forward_packet (void *arg, kpr_fwd_desc_t *fwd) LASSERT (nob == lib_iov_nob (fwd->kprfd_niov, fwd->kprfd_iov)); atomic_inc (&kpr_queue_depth); + atomic_inc (&src_ne->kpne_refcount); /* source nal is busy until fwd completes */ kpr_fwd_packets++; /* (loose) stats accounting */ kpr_fwd_bytes += nob; - if (src_ne->kpne_shutdown) /* caller is shutting down */ + if (src_ne->kpne_shutdown) /* caller is shutting down */ goto out; - fwd->kprfd_router_arg = src_ne; /* stash caller's nal entry */ - atomic_inc (&src_ne->kpne_refcount); /* source nal is busy until fwd completes */ + fwd->kprfd_router_arg = src_ne; /* stash caller's nal entry */ read_lock (&kpr_rwlock); diff --git a/lustre/portals/unals/connection.c b/lustre/portals/unals/connection.c index 89c9f78..310e899 100644 --- a/lustre/portals/unals/connection.c +++ b/lustre/portals/unals/connection.c @@ -120,9 +120,10 @@ int read_connection(connection c, return(1); } -static int connection_input(connection c) +static int connection_input(void *d) { - return((*c->m->handler)(c->m->handler_arg,c)); + connection c = d; + return((*c->m->handler)(c->m->handler_arg,c)); } @@ -280,7 +281,7 @@ void shutdown_connections(manager m) * zero if the fixed port could not be bound */ manager init_connections(unsigned short pid, - int (*input)(), + int (*input)(void *, void *), void *a) { manager m=(manager)malloc(sizeof(struct manager)); diff --git a/lustre/portals/unals/connection.h b/lustre/portals/unals/connection.h index f6b2994..6f57287 100644 --- a/lustre/portals/unals/connection.h +++ b/lustre/portals/unals/connection.h @@ -25,14 +25,8 @@ typedef struct connection { manager m; } *connection; -connection force_tcp_connection(manager m, - unsigned int ip, - unsigned int short); -manager init_connections(unsigned short, - int (*f)(void *,connection), - void *); +connection force_tcp_connection(manager m, unsigned int ip, unsigned int short); +manager init_connections(unsigned short, int (*f)(void *, void *), void *); void remove_connection(void *arg); void shutdown_connections(manager m); -int read_connection(connection c, - unsigned char *dest, - int len); +int read_connection(connection c, unsigned char *dest, int len); diff --git a/lustre/portals/unals/tcpnal.c b/lustre/portals/unals/tcpnal.c index 8bf55c4..534fc17 100644 --- a/lustre/portals/unals/tcpnal.c +++ b/lustre/portals/unals/tcpnal.c @@ -151,15 +151,17 @@ int tcpnal_recv(nal_cb_t *n, * available. It attempts to read the portals header and * pass it to the generic library for processing. */ -static int from_connection(void *a,connection c) +static int from_connection(void *a, void *d) { - bridge b=a; - ptl_hdr_t hdr; - if (read_connection(c, (unsigned char *)&hdr, sizeof(hdr))){ - lib_parse(b->nal_cb, &hdr, c); - return(1); - } - return(0); + connection c = d; + bridge b=a; + ptl_hdr_t hdr; + + if (read_connection(c, (unsigned char *)&hdr, sizeof(hdr))){ + lib_parse(b->nal_cb, &hdr, c); + return(1); + } + return(0); } diff --git a/lustre/portals/utils/portals.c b/lustre/portals/utils/portals.c index 8235271..90d66f5 100644 --- a/lustre/portals/utils/portals.c +++ b/lustre/portals/utils/portals.c @@ -106,13 +106,6 @@ nal2name (int nal) return ((e == NULL) ? "???" : e->name); } -static int -nid2nal (ptl_nid_t nid) -{ - /* BIG pragmatic assumption */ - return ((((__u32)nid) & 0xffff0000) != 0 ? SOCKNAL : QSWNAL); -} - int ptl_parse_nid (ptl_nid_t *nidp, char *str) { @@ -160,35 +153,15 @@ ptl_parse_nid (ptl_nid_t *nidp, char *str) char * ptl_nid2str (char *buffer, ptl_nid_t nid) { - switch (nid2nal(nid)) - { - case QSWNAL: - sprintf (buffer, LPD64, nid); - return (buffer); + __u32 addr = htonl((__u32)nid); /* back to NETWORK byte order */ + struct hostent *he = gethostbyaddr ((const char *)&addr, sizeof (addr), AF_INET); - case SCIMACNAL: - sprintf (buffer, LPX64, nid); - return (buffer); - - case SOCKNAL: { - __u32 addr = htonl((__u32)nid); /* back to NETWORK byte order */ - struct hostent *he = gethostbyaddr ((const char *)&addr, sizeof (addr), AF_INET); - - if (he != NULL) - strcpy (buffer, he->h_name); - else - { - addr = (__u32)nid; - sprintf (buffer, "%d.%d.%d.%d", - (addr>>24)&0xff, (addr>>16)&0xff, (addr>>8)&0xff, addr&0xff); - } - return (buffer); - } + if (he != NULL) + strcpy (buffer, he->h_name); + else + sprintf (buffer, "0x"LPX64, nid); - default: - sprintf (buffer, "nid2nal broken"); - return (buffer); - } + return (buffer); } int @@ -704,18 +677,43 @@ int jt_ptl_ping(int argc, char **argv) return 0; } +int jt_ptl_shownid(int argc, char **argv) +{ + struct portal_ioctl_data data; + int rc; + + if (argc > 1) { + fprintf(stderr, "usage: %s\n", argv[0]); + return 0; + } + + if (g_nal == 0) { + fprintf(stderr, "Error: you must run the 'network' command first\n"); + return -1; + } + + PORTAL_IOC_INIT (data); + data.ioc_nal = g_nal; + rc = l_ioctl(PORTALS_DEV_ID, IOC_PORTAL_GET_NID, &data); + if (rc < 0) + fprintf(stderr, "getting my NID failed: %s\n", + strerror (errno)); + else + printf(LPX64"\n", data.ioc_nid); + return 0; +} + int jt_ptl_mynid(int argc, char **argv) { int rc; - struct hostent *h; - char buf[1024], *hostname; + char hostname[1024]; + char *nidstr; struct portal_ioctl_data data; ptl_nid_t mynid; if (argc > 2) { - fprintf(stderr, "usage: %s [hostname]\n", argv[0]); - fprintf(stderr, "hostname defaults to the hostname of the " - "machine.\n"); + fprintf(stderr, "usage: %s [NID]\n", argv[0]); + fprintf(stderr, "NID defaults to the primary IP address of the machine.\n"); return 0; } @@ -725,42 +723,21 @@ int jt_ptl_mynid(int argc, char **argv) return -1; } - if (g_nal == QSWNAL) { - fprintf(stderr, "'mynid' doesn't make any sense for elan.\n"); - return -1; - } else if (g_nal == GMNAL) { - fprintf(stderr, "'mynid' doesn't make any sense for GM.\n"); - return -1; - } else if (g_nal == SCIMACNAL) { - fprintf(stderr, "'mynid' doesn't make any sense for SCI.\n"); - return -1; - } - - if (g_nal != SOCKNAL && g_nal != TOENAL) { - fprintf(stderr, "This should never happen. Also it is very " - "bad.\n"); + if (argc >= 2) + nidstr = argv[1]; + else if (gethostname(hostname, sizeof(hostname)) != 0) { + fprintf(stderr, "gethostname failed: %s\n", + strerror(errno)); return -1; } + else + nidstr = hostname; - if (argc == 1) { - if (gethostname(buf, sizeof(buf)) != 0) { - fprintf(stderr, "gethostname failed: %s\n", - strerror(errno)); - return -1; - } - hostname = buf; - } else { - hostname = argv[1]; - } - - h = gethostbyname(hostname); - - if (!h) { - fprintf(stderr, "cannot get address for host '%s': %d\n", - hostname, h_errno); + rc = ptl_parse_nid (&mynid, nidstr); + if (rc != 0) { + fprintf (stderr, "Can't convert '%s' into a NID\n", nidstr); return -1; } - mynid = (ptl_nid_t)ntohl (*(__u32 *)h->h_addr); /* HOST byte order */ PORTAL_IOC_INIT(data); data.ioc_nid = mynid; @@ -769,7 +746,7 @@ int jt_ptl_mynid(int argc, char **argv) rc = l_ioctl(PORTALS_DEV_ID, IOC_PORTAL_NAL_CMD, &data); if (rc < 0) - fprintf(stderr, "IOC_PORTAL_REGISTER_MYNID failed: %s\n", + fprintf(stderr, "setting my NID failed: %s\n", strerror(errno)); else printf("registered my nid "LPX64" (%s)\n", mynid, hostname); @@ -888,7 +865,6 @@ jt_ptl_add_route (int argc, char **argv) ptl_nid_t nid1; ptl_nid_t nid2; ptl_nid_t gateway_nid; - int gateway_nal; int rc; if (argc < 3) @@ -897,14 +873,18 @@ jt_ptl_add_route (int argc, char **argv) return (0); } + if (g_nal == 0) { + fprintf(stderr, "Error: you must run the 'network' command " + "first.\n"); + return (-1); + } + if (ptl_parse_nid (&gateway_nid, argv[1]) != 0) { fprintf (stderr, "Can't parse gateway NID \"%s\"\n", argv[1]); return (-1); } - gateway_nal = nid2nal (gateway_nid); - if (ptl_parse_nid (&nid1, argv[2]) != 0) { fprintf (stderr, "Can't parse first target NID \"%s\"\n", argv[2]); @@ -921,7 +901,7 @@ jt_ptl_add_route (int argc, char **argv) PORTAL_IOC_INIT(data); data.ioc_nid = gateway_nid; - data.ioc_nal = gateway_nal; + data.ioc_nal = g_nal; data.ioc_nid2 = MIN (nid1, nid2); data.ioc_nid3 = MAX (nid1, nid2); diff --git a/lustre/portals/utils/ptlctl.c b/lustre/portals/utils/ptlctl.c index d38bd4a..8c56d93 100644 --- a/lustre/portals/utils/ptlctl.c +++ b/lustre/portals/utils/ptlctl.c @@ -34,6 +34,7 @@ command_t list[] = { {"disconnect", jt_ptl_disconnect, 0, "disconnect from a remote nid (args: [hostname]"}, {"push", jt_ptl_push_connection, 0, "flush connection to a remote nid (args: [hostname]"}, {"ping", jt_ptl_ping, 0, "do a ping test (args: nid [count] [size] [timeout])"}, + {"shownid", jt_ptl_shownid, 0, "print the local NID"}, {"mynid", jt_ptl_mynid, 0, "inform the socknal of the local NID (args: [hostname])"}, {"add_route", jt_ptl_add_route, 0, "add an entry to the routing table (args: gatewayNID targetNID [targetNID])"}, {"del_route", jt_ptl_del_route, 0, "delete an entry from the routing table (args: targetNID"}, -- 1.8.3.1