From c1c483919973594b209d37d5f30bcdc1165e0b9e Mon Sep 17 00:00:00 2001 From: eeb Date: Wed, 4 May 2005 09:49:28 +0000 Subject: [PATCH] * placeholder --- lnet/include/libcfs/kp30.h | 3 + lnet/include/lnet/lib-lnet.h | 5 +- lnet/include/lnet/lib-p30.h | 5 +- lnet/include/lnet/lib-types.h | 35 ++- lnet/klnds/gmlnd/gmlnd_api.c | 7 +- lnet/klnds/iiblnd/iiblnd.c | 7 +- lnet/klnds/iiblnd/iiblnd.h | 2 +- lnet/klnds/lolnd/lolnd.c | 2 +- lnet/klnds/lolnd/lolnd.h | 2 +- lnet/klnds/openiblnd/openiblnd.c | 7 +- lnet/klnds/openiblnd/openiblnd.h | 2 +- lnet/klnds/qswlnd/qswlnd.c | 11 +- lnet/klnds/qswlnd/qswlnd.h | 2 +- lnet/klnds/ralnd/ralnd.c | 54 ++++- lnet/klnds/ralnd/ralnd.h | 2 +- lnet/klnds/socklnd/socklnd.c | 7 +- lnet/klnds/socklnd/socklnd.h | 2 +- lnet/klnds/viblnd/viblnd.c | 7 +- lnet/klnds/viblnd/viblnd.h | 2 +- lnet/klnds/viblnd/viblnd_cb.c | 4 +- lnet/libcfs/nidstrings.c | 63 ++++- lnet/lnet/api-ni.c | 492 +++++++++++++++++++++++++++++++-------- lnet/lnet/lib-eq.c | 6 +- lnet/lnet/lib-md.c | 8 +- lnet/lnet/lib-me.c | 6 +- lnet/lnet/lib-move.c | 6 +- lnet/lnet/module.c | 2 - lnet/ulnds/procapi.c | 2 +- lnet/ulnds/socklnd/procapi.c | 2 +- 29 files changed, 590 insertions(+), 165 deletions(-) diff --git a/lnet/include/libcfs/kp30.h b/lnet/include/libcfs/kp30.h index a74756c..50ab197 100644 --- a/lnet/include/libcfs/kp30.h +++ b/lnet/include/libcfs/kp30.h @@ -209,6 +209,7 @@ void portals_debug_dumplog(void); /* support decl needed both by kernel and liblustre */ int libcfs_isknown_nal(int type); +char *libcfs_nal2modname(int type); char *libcfs_nal2str(int type); int libcfs_str2nal(char *str); char *libcfs_net2str(__u32 net); @@ -438,6 +439,8 @@ extern int portal_ioctl_getdata(char *buf, char *end, void *arg); enum { + /* Only add to these values (i.e. don't ever change or redefine them): + * network addresses depend on them... */ LONAL = 1, SOCKNAL = 2, QSWNAL = 3, diff --git a/lnet/include/lnet/lib-lnet.h b/lnet/include/lnet/lib-lnet.h index 2b02a1c..2a83b6c 100644 --- a/lnet/include/lnet/lib-lnet.h +++ b/lnet/include/lnet/lib-lnet.h @@ -24,7 +24,6 @@ #include #include -extern int ptl_init; /* PtlInit()/PtlFini() counter */ extern ptl_apini_t ptl_apini; /* THE network interface (at the API) */ static inline int ptl_is_wire_handle_none (ptl_handle_wire_t *wh) @@ -45,11 +44,15 @@ static inline int ptl_md_exhausted (ptl_libmd_t *md) spin_lock_irqsave(&ptl_apini.apini_lock, flags) #define PTL_UNLOCK(flags) \ spin_unlock_irqrestore(&ptl_apini.apini_lock, flags) +#define PTL_MUTEX_DOWN(m) mutex_down(m) +#define PTL_MUTEX_UP(m) mutex_up(m) #else #define PTL_LOCK(flags) \ (pthread_mutex_lock(&ptl_apini.apini_mutex), (flags) = 0) #define PTL_UNLOCK(flags) \ pthread_mutex_unlock(&ptl_apini.apini_mutex) +#define PTL_MUTEX_DOWN(m) pthread_mutex_lock(m) +#define PTL_MUTEX_UP(m) pthread_mutex_up(m) #endif #ifdef PTL_USE_LIB_FREELIST diff --git a/lnet/include/lnet/lib-p30.h b/lnet/include/lnet/lib-p30.h index 2b02a1c..2a83b6c 100644 --- a/lnet/include/lnet/lib-p30.h +++ b/lnet/include/lnet/lib-p30.h @@ -24,7 +24,6 @@ #include #include -extern int ptl_init; /* PtlInit()/PtlFini() counter */ extern ptl_apini_t ptl_apini; /* THE network interface (at the API) */ static inline int ptl_is_wire_handle_none (ptl_handle_wire_t *wh) @@ -45,11 +44,15 @@ static inline int ptl_md_exhausted (ptl_libmd_t *md) spin_lock_irqsave(&ptl_apini.apini_lock, flags) #define PTL_UNLOCK(flags) \ spin_unlock_irqrestore(&ptl_apini.apini_lock, flags) +#define PTL_MUTEX_DOWN(m) mutex_down(m) +#define PTL_MUTEX_UP(m) mutex_up(m) #else #define PTL_LOCK(flags) \ (pthread_mutex_lock(&ptl_apini.apini_mutex), (flags) = 0) #define PTL_UNLOCK(flags) \ pthread_mutex_unlock(&ptl_apini.apini_mutex) +#define PTL_MUTEX_DOWN(m) pthread_mutex_lock(m) +#define PTL_MUTEX_UP(m) pthread_mutex_up(m) #endif #ifdef PTL_USE_LIB_FREELIST diff --git a/lnet/include/lnet/lib-types.h b/lnet/include/lnet/lib-types.h index ae84a5e..3638141 100644 --- a/lnet/include/lnet/lib-types.h +++ b/lnet/include/lnet/lib-types.h @@ -258,7 +258,7 @@ typedef struct ptl_nal /* fields initialised by the NAL */ unsigned int nal_type; - ptl_err_t (*nal_startup) (struct ptl_ni *ni, char **interfaces); + ptl_err_t (*nal_startup) (struct ptl_ni *ni); void (*nal_shutdown) (struct ptl_ni *ni); int (*nal_ctl)(struct ptl_ni *ni, unsigned int cmd, void *arg); @@ -313,6 +313,8 @@ typedef struct ptl_nal } ptl_nal_t; +#define PTL_MAX_INTERFACES 8 + typedef struct ptl_ni { struct list_head ni_list; /* chain on apini_nis */ ptl_nid_t ni_nid; /* interface's NID */ @@ -320,6 +322,7 @@ typedef struct ptl_ni { ptl_nal_t *ni_nal; /* procedural interface */ int ni_shutdown; /* shutting down? */ atomic_t ni_refcount; /* reference count */ + char *ni_interfaces[PTL_MAX_INTERFACES]; /* equivalent interfaces to use */ } ptl_ni_t; typedef struct /* loopback descriptor */ @@ -339,7 +342,25 @@ typedef struct /* loopback descriptor */ typedef struct { + /* Stuff initialised at PtlInit() */ + int apini_init; /* PtlInit() called? */ int apini_refcount; /* PtlNIInit/PtlNIFini counter */ + + struct list_head apini_nals; /* registered NALs */ + +#ifdef __KERNEL__ + spinlock_t apini_lock; + cfs_waitq_t apini_waitq; + struct semaphore apini_api_mutex; + struct semaphore apini_nal_mutex; +#else + pthread_mutex_t apini_mutex; + pthread_cond_t apini_cond; + pthread_mutex_t apini_api_mutex; + pthread_mutex_t apini_nal_mutex; +#endif + + /* Stuff initialised at PtlNIInit() */ int apini_nportals; /* # portals */ struct list_head *apini_portals; /* the vector of portals */ @@ -347,6 +368,9 @@ typedef struct ptl_pid_t apini_pid; /* requested pid */ ptl_ni_limits_t apini_actual_limits; + char *apini_net_tokens; /* tokenized 'networks' */ + int apini_net_tokens_nob; + struct list_head apini_nis; /* NAL instances */ struct list_head apini_zombie_nis; /* dying NAL instances */ int apini_nzombie_nis; /* # of NIS to wait for */ @@ -364,19 +388,10 @@ typedef struct ptl_freelist_t apini_free_mds; ptl_freelist_t apini_free_eqs; #endif - struct list_head apini_active_msgs; struct list_head apini_active_mds; struct list_head apini_active_eqs; -#ifdef __KERNEL__ - spinlock_t apini_lock; - cfs_waitq_t apini_waitq; -#else - pthread_mutex_t apini_mutex; - pthread_cond_t apini_cond; -#endif - struct { long recv_count; long recv_length; diff --git a/lnet/klnds/gmlnd/gmlnd_api.c b/lnet/klnds/gmlnd/gmlnd_api.c index 88d8671..a8d179f 100644 --- a/lnet/klnds/gmlnd/gmlnd_api.c +++ b/lnet/klnds/gmlnd/gmlnd_api.c @@ -93,7 +93,7 @@ gmnal_shutdown(ptl_ni_t *ni) int -gmnal_startup(ptl_ni_t *ni, char **interfaces) +gmnal_startup(ptl_ni_t *ni) { gmnal_data_t *nal_data = NULL; gmnal_srxd_t *srxd = NULL; @@ -110,6 +110,11 @@ gmnal_startup(ptl_ni_t *ni, char **interfaces) return PTL_FAIL; } + if (ni->ni_interfaces[0] != NULL) { + CERROR("Explicit interface config not supported\n"); + return PTL_FAIL; + } + PORTAL_ALLOC(nal_data, sizeof(gmnal_data_t)); if (!nal_data) { CDEBUG(D_ERROR, "can't get memory\n"); diff --git a/lnet/klnds/iiblnd/iiblnd.c b/lnet/klnds/iiblnd/iiblnd.c index 6a59e59..37a8051 100644 --- a/lnet/klnds/iiblnd/iiblnd.c +++ b/lnet/klnds/iiblnd/iiblnd.c @@ -1299,7 +1299,7 @@ static __u64 max_phys_mem(IB_CA_ATTRIBUTES *ca_attr) #undef roundup_power ptl_err_t -kibnal_startup (ptl_ni_t *ni, char **interfaces) +kibnal_startup (ptl_ni_t *ni) { int pkmem = atomic_read(&portal_kmemory); IB_PORT_ATTRIBUTES *pattr; @@ -1315,6 +1315,11 @@ kibnal_startup (ptl_ni_t *ni, char **interfaces) CERROR ("Only 1 instance supported\n"); return PTL_FAIL; } + + if (ni->ni_interfaces[0] != NULL) { + CERROR("Explicit interface config not supported\n"); + return PTL_FAIL; + } ni->ni_data = &kibnal_data; kibnal_data.kib_ni = ni; diff --git a/lnet/klnds/iiblnd/iiblnd.h b/lnet/klnds/iiblnd/iiblnd.h index 4eadbc9..3909e49 100644 --- a/lnet/klnds/iiblnd/iiblnd.h +++ b/lnet/klnds/iiblnd/iiblnd.h @@ -856,7 +856,7 @@ kibnal_whole_mem(void) return kibnal_data.kib_md.md_handle != NULL; } -extern ptl_err_t kibnal_startup (ptl_ni_t *ni, char **interfaces); +extern ptl_err_t kibnal_startup (ptl_ni_t *ni); extern void kibnal_shutdown (ptl_ni_t *ni); extern int kibnal_ctl(ptl_ni_t *ni, unsigned int cmd, void *arg); ptl_err_t kibnal_send (ptl_ni_t *ni, void *private, diff --git a/lnet/klnds/lolnd/lolnd.c b/lnet/klnds/lolnd/lolnd.c index 0469b23..d53c43b 100644 --- a/lnet/klnds/lolnd/lolnd.c +++ b/lnet/klnds/lolnd/lolnd.c @@ -45,7 +45,7 @@ klonal_shutdown(ptl_ni_t *ni) } ptl_err_t -klonal_startup (ptl_ni_t *ni, char **interfaces) +klonal_startup (ptl_ni_t *ni) { LASSERT (ni->ni_nal == &klonal_nal); diff --git a/lnet/klnds/lolnd/lolnd.h b/lnet/klnds/lolnd/lolnd.h index 092af64..4bf7cf5 100644 --- a/lnet/klnds/lolnd/lolnd.h +++ b/lnet/klnds/lolnd/lolnd.h @@ -54,7 +54,7 @@ typedef struct } klod_iov; } klo_desc_t; -ptl_err_t klonal_startup (ptl_ni_t *ni, char **interfaces); +ptl_err_t klonal_startup (ptl_ni_t *ni); void klonal_shutdown (ptl_ni_t *ni); ptl_err_t klonal_send (ptl_ni_t *ni, void *private, ptl_msg_t *ptlmsg, ptl_hdr_t *hdr, diff --git a/lnet/klnds/openiblnd/openiblnd.c b/lnet/klnds/openiblnd/openiblnd.c index a2ee0ed..432f937 100644 --- a/lnet/klnds/openiblnd/openiblnd.c +++ b/lnet/klnds/openiblnd/openiblnd.c @@ -1882,7 +1882,7 @@ kibnal_shutdown (ptl_ni_t *ni) } ptl_err_t -kibnal_startup (ptl_ni_t *ni, char **interfaces) +kibnal_startup (ptl_ni_t *ni) { struct timeval tv; int pkmem = atomic_read(&portal_kmemory); @@ -1897,6 +1897,11 @@ kibnal_startup (ptl_ni_t *ni, char **interfaces) return PTL_FAIL; } + if (ni->ni_interfaces[0] != NULL) { + CERROR("Explicit interface config not supported\n"); + return PTL_FAIL; + } + PORTAL_MODULE_USE; memset (&kibnal_data, 0, sizeof (kibnal_data)); /* zero pointers, flags etc */ diff --git a/lnet/klnds/openiblnd/openiblnd.h b/lnet/klnds/openiblnd/openiblnd.h index fdfc043..d922572 100644 --- a/lnet/klnds/openiblnd/openiblnd.h +++ b/lnet/klnds/openiblnd/openiblnd.h @@ -504,7 +504,7 @@ kibnal_wreqid_is_rx (__u64 wreqid) # define sk_sleep sleep #endif -ptl_err_t kibnal_startup (ptl_ni_t *ni, char **interfaces); +ptl_err_t kibnal_startup (ptl_ni_t *ni); void kibnal_shutdown (ptl_ni_t *ni); int kibnal_ctl(ptl_ni_t *ni, unsigned int cmd, void *arg); ptl_err_t kibnal_send (ptl_ni_t *ni, void *private, diff --git a/lnet/klnds/qswlnd/qswlnd.c b/lnet/klnds/qswlnd/qswlnd.c index db7184f..4cec274 100644 --- a/lnet/klnds/qswlnd/qswlnd.c +++ b/lnet/klnds/qswlnd/qswlnd.c @@ -307,7 +307,7 @@ kqswnal_shutdown(ptl_ni_t *ni) } ptl_err_t -kqswnal_startup (ptl_ni_t *ni, char **interfaces) +kqswnal_startup (ptl_ni_t *ni) { #if MULTIRAIL_EKC EP_RAILMASK all_rails = EP_RAILMASK_ALL; @@ -321,12 +321,19 @@ kqswnal_startup (ptl_ni_t *ni, char **interfaces) int elan_page_idx; int pkmem = atomic_read(&portal_kmemory); + LASSERT (ni->ni_nal == &kqswnal_nal); + /* Only 1 instance supported */ if (kqswnal_data.kqn_init != KQN_INIT_NOTHING) { CERROR ("Only 1 instance supported\n"); return PTL_FAIL; } + if (ni->ni_interfaces[0] != NULL) { + CERROR("Explicit interface config not supported\n"); + return PTL_FAIL; + } + CDEBUG (D_MALLOC, "start kmem %d\n", atomic_read(&portal_kmemory)); /* ensure all pointers NULL etc */ @@ -454,7 +461,7 @@ kqswnal_startup (ptl_ni_t *ni, char **interfaces) &dmareq, &kqswnal_data.kqn_eptxdmahandle); if (rc != DDI_SUCCESS) { - CERROR ("Can't reserve rx dma space\n"); + CERROR ("Can't reserve tx dma space\n"); kqswnal_shutdown (ni); return (PTL_NO_SPACE); } diff --git a/lnet/klnds/qswlnd/qswlnd.h b/lnet/klnds/qswlnd/qswlnd.h index 6caa8c3..767fce8 100644 --- a/lnet/klnds/qswlnd/qswlnd.h +++ b/lnet/klnds/qswlnd/qswlnd.h @@ -372,7 +372,7 @@ ep_free_rcvr(EP_RCVR *r) } #endif -ptl_err_t kqswnal_startup (ptl_ni_t *ni, char **interfaces); +ptl_err_t kqswnal_startup (ptl_ni_t *ni); void kqswnal_shutdown (ptl_ni_t *ni); int kqswnal_ctl (ptl_ni_t *ni, unsigned int cmd, void *arg); ptl_err_t kqswnal_send (ptl_ni_t *ni, void *private, diff --git a/lnet/klnds/ralnd/ralnd.c b/lnet/klnds/ralnd/ralnd.c index 5cab1d0..e48d5e2 100644 --- a/lnet/klnds/ralnd/ralnd.c +++ b/lnet/klnds/ralnd/ralnd.c @@ -1907,7 +1907,7 @@ kranal_shutdown (ptl_ni_t *ni) } ptl_err_t -kranal_startup (ptl_ni_t *ni, char **interfaces) +kranal_startup (ptl_ni_t *ni) { struct timeval tv; int pkmem = atomic_read(&portal_kmemory); @@ -2014,19 +2014,51 @@ kranal_startup (ptl_ni_t *ni, char **interfaces) LASSERT (kranal_data.kra_ndevs == 0); - for (i = 0; i < sizeof(kranal_devids)/sizeof(kranal_devids[0]); i++) { - LASSERT (i < RANAL_MAXDEVS); + if (ni->ni_interfaces[0] == NULL) { + /* Use all available RapidArray devices */ + for (i = 0; i < sizeof(kranal_devids)/sizeof(kranal_devids[0]); i++) { + LASSERT (i < RANAL_MAXDEVS); - dev = &kranal_data.kra_devices[kranal_data.kra_ndevs]; + dev = &kranal_data.kra_devices[kranal_data.kra_ndevs]; - rc = kranal_device_init(kranal_devids[i], dev); - if (rc == 0) + rc = kranal_device_init(kranal_devids[i], dev); + if (rc == 0) + kranal_data.kra_ndevs++; + } + + if (kranal_data.kra_ndevs == 0) { + CERROR("Can't initialise any RapidArray devices\n"); + goto failed; + } + } else { + /* Use specified RapidArray devices */ + for (i = 0; i < PTL_MAX_INTERFACES; i++) { + int devid; + int len; + + if (kranal_data.kra_ndevs == RANAL_MAXDEVS) { + CERROR("Too many interfaces\n"); + goto failed; + } + + dev = &kranal_data.kra_devices[kranal_data.kra_ndevs]; + + if (sscanf(ni->ni_interfaces[i], "%d%n", &devid, &len) < 1 || + len != strlen(ni->ni_interfaces[i])) { + CERROR("Can't parse interface '%s'\n", + ni->ni_interfaces[i]); + goto failed; + } + + rc = kranal_device_init(devid, dev); + if (rc != 0) { + CERROR("Can't open interface '%s': %d\n", + ni->ni_interfaces[i], rc); + goto failed; + } + kranal_data.kra_ndevs++; - } - - if (kranal_data.kra_ndevs == 0) { - CERROR("Can't initialise any RapidArray devices\n"); - goto failed; + } } for (i = 0; i < kranal_data.kra_ndevs; i++) { diff --git a/lnet/klnds/ralnd/ralnd.h b/lnet/klnds/ralnd/ralnd.h index 942e5bb..71cd6660 100644 --- a/lnet/klnds/ralnd/ralnd.h +++ b/lnet/klnds/ralnd/ralnd.h @@ -445,7 +445,7 @@ kranal_page2phys (struct page *p) return page_to_phys(p); } -ptl_err_t kranal_startup (ptl_ni_t *ni, char **interfaces); +ptl_err_t kranal_startup (ptl_ni_t *ni); void kranal_shutdown (ptl_ni_t *ni); int kranal_ctl(ptl_ni_t *ni, unsigned int cmd, void *arg); ptl_err_t kranal_send (ptl_ni_t *ni, void *private, diff --git a/lnet/klnds/socklnd/socklnd.c b/lnet/klnds/socklnd/socklnd.c index af36f04..0afe554 100644 --- a/lnet/klnds/socklnd/socklnd.c +++ b/lnet/klnds/socklnd/socklnd.c @@ -2052,7 +2052,7 @@ ksocknal_init_incarnation (void) } ptl_err_t -ksocknal_startup (ptl_ni_t *ni, char **interfaces) +ksocknal_startup (ptl_ni_t *ni) { int pkmem = atomic_read(&portal_kmemory); int rc; @@ -2066,6 +2066,11 @@ ksocknal_startup (ptl_ni_t *ni, char **interfaces) return PTL_FAIL; } + if (ni->ni_interfaces[0] != NULL) { + CERROR("Explicit interface config not supported\n"); + return PTL_FAIL; + } + memset (&ksocknal_data, 0, sizeof (ksocknal_data)); /* zero pointers */ ksocknal_data.ksnd_ni = ni; /* temp hack */ diff --git a/lnet/klnds/socklnd/socklnd.h b/lnet/klnds/socklnd/socklnd.h index 916ba9c..ee7bba4 100644 --- a/lnet/klnds/socklnd/socklnd.h +++ b/lnet/klnds/socklnd/socklnd.h @@ -483,7 +483,7 @@ ksocknal_peer_decref (ksock_peer_t *peer) } -ptl_err_t ksocknal_startup (ptl_ni_t *ni, char **interfaces); +ptl_err_t ksocknal_startup (ptl_ni_t *ni); void ksocknal_shutdown (ptl_ni_t *ni); int ksocknal_ctl(ptl_ni_t *ni, unsigned int cmd, void *arg); ptl_err_t ksocknal_send (ptl_ni_t *ni, void *private, diff --git a/lnet/klnds/viblnd/viblnd.c b/lnet/klnds/viblnd/viblnd.c index 7e28052..32e1ff1 100644 --- a/lnet/klnds/viblnd/viblnd.c +++ b/lnet/klnds/viblnd/viblnd.c @@ -1589,7 +1589,7 @@ kibnal_shutdown (ptl_ni_t *ni) } ptl_err_t -kibnal_startup (ptl_ni_t *ni, char **interfaces) +kibnal_startup (ptl_ni_t *ni) { struct timeval tv; int pkmem = atomic_read(&portal_kmemory); @@ -1606,6 +1606,11 @@ kibnal_startup (ptl_ni_t *ni, char **interfaces) return PTL_FAIL; } + if (ni->ni_interfaces[0] != NULL) { + CERROR("Explicit interface config not supported\n"); + return PTL_FAIL; + } + PORTAL_MODULE_USE; memset (&kibnal_data, 0, sizeof (kibnal_data)); /* zero pointers, flags etc */ diff --git a/lnet/klnds/viblnd/viblnd.h b/lnet/klnds/viblnd/viblnd.h index be45e8f..f3fbacd 100644 --- a/lnet/klnds/viblnd/viblnd.h +++ b/lnet/klnds/viblnd/viblnd.h @@ -409,7 +409,7 @@ typedef struct kib_peer extern kib_data_t kibnal_data; extern kib_tunables_t kibnal_tunables; -ptl_err_t kibnal_startup (ptl_ni_t *ni, char **interfaces); +ptl_err_t kibnal_startup (ptl_ni_t *ni); void kibnal_shutdown (ptl_ni_t *ni); int kibnal_ctl(ptl_ni_t *ni, unsigned int cmd, void *arg); ptl_err_t kibnal_send (ptl_ni_t *ni, void *private, diff --git a/lnet/klnds/viblnd/viblnd_cb.c b/lnet/klnds/viblnd/viblnd_cb.c index 60ccfc7..6cea29c 100644 --- a/lnet/klnds/viblnd/viblnd_cb.c +++ b/lnet/klnds/viblnd/viblnd_cb.c @@ -1179,7 +1179,7 @@ kibnal_init_rdma (kib_tx_t *tx, int type, int nob, resid -= wrknob; if (wrknob < srcfrag->rf_nob) { kibnal_rf_set(srcfrag, - kibnal_rf_addr(srcfrag) + resid, + kibnal_rf_addr(srcfrag) + wrknob, srcfrag->rf_nob - wrknob); } else { srcfrag++; @@ -1188,7 +1188,7 @@ kibnal_init_rdma (kib_tx_t *tx, int type, int nob, if (wrknob < dstfrag->rf_nob) { kibnal_rf_set(dstfrag, - kibnal_rf_addr(dstfrag) + resid, + kibnal_rf_addr(dstfrag) + wrknob, dstfrag->rf_nob - wrknob); } else { dstfrag++; diff --git a/lnet/libcfs/nidstrings.c b/lnet/libcfs/nidstrings.c index fc64c99..c3e4b81 100644 --- a/lnet/libcfs/nidstrings.c +++ b/lnet/libcfs/nidstrings.c @@ -90,19 +90,52 @@ static int libcfs_num_str2addr(char *str, int nob, __u32 *addr); struct nalstrfns { int nf_nal; char *nf_name; + char *nf_modname; void (*nf_addr2str)(__u32 addr, char *str); int (*nf_str2addr)(char *str, int nob, __u32 *addr); }; static struct nalstrfns libcfs_nalstrfns[] = { - {LONAL, "lo", libcfs_lo_addr2str, libcfs_lo_str2addr}, - {SOCKNAL, "tcp", libcfs_ip_addr2str, libcfs_ip_str2addr}, - {OPENIBNAL, "openib", libcfs_ip_addr2str, libcfs_ip_str2addr}, - {IIBNAL, "iib", libcfs_ip_addr2str, libcfs_ip_str2addr}, - {VIBNAL, "vib", libcfs_ip_addr2str, libcfs_ip_str2addr}, - {RANAL, "ra", libcfs_ip_addr2str, libcfs_ip_str2addr}, - {QSWNAL, "elan", libcfs_num_addr2str, libcfs_num_str2addr}, - {GMNAL, "gm", libcfs_num_addr2str, libcfs_num_str2addr}, + {.nf_nal = LONAL, + .nf_name = "lo", + .nf_modname = "klonal", + .nf_addr2str = libcfs_lo_addr2str, + .nf_str2addr = libcfs_lo_str2addr}, + {.nf_nal = SOCKNAL, + .nf_name = "tcp", + .nf_modname = "ksocknal", + .nf_addr2str = libcfs_ip_addr2str, + .nf_str2addr = libcfs_ip_str2addr}, + {.nf_nal = OPENIBNAL, + .nf_name = "openib", + .nf_modname = "kopenibnal", + .nf_addr2str = libcfs_ip_addr2str, + .nf_str2addr = libcfs_ip_str2addr}, + {.nf_nal = IIBNAL, + .nf_name = "iib", + .nf_modname = "kiibnal", + .nf_addr2str = libcfs_ip_addr2str, + .nf_str2addr = libcfs_ip_str2addr}, + {.nf_nal = VIBNAL, + .nf_name = "vib", + .nf_modname = "kvibnal", + .nf_addr2str = libcfs_ip_addr2str, + .nf_str2addr = libcfs_ip_str2addr}, + {.nf_nal = RANAL, + .nf_name = "ra", + .nf_modname = "kranal", + .nf_addr2str = libcfs_ip_addr2str, + .nf_str2addr = libcfs_ip_str2addr}, + {.nf_nal = QSWNAL, + .nf_name = "elan", + .nf_modname = "kqswnal", + .nf_addr2str = libcfs_num_addr2str, + .nf_str2addr = libcfs_num_str2addr}, + {.nf_nal = GMNAL, + .nf_name = "gm", + .nf_modname = "kgmnal", + .nf_addr2str = libcfs_num_addr2str, + .nf_str2addr = libcfs_num_str2addr}, }; const int libcfs_nnalstrfns = sizeof(libcfs_nalstrfns)/sizeof(libcfs_nalstrfns[0]); @@ -239,6 +272,14 @@ libcfs_isknown_nal(int nal) } char * +libcfs_nal2modname(int nal) +{ + struct nalstrfns *nf = libcfs_nal2nalstrfns(nal); + + return (nf == NULL) ? NULL : nf->nf_modname; +} + +char * libcfs_nal2str(int nal) { char *str; @@ -389,6 +430,12 @@ libcfs_isknown_nal(int nal) } char * +libcfs_nal2modname(int nal) +{ + return "cray"; +} + +char * libcfs_nal2str(int nal) { return "cray"; diff --git a/lnet/lnet/api-ni.c b/lnet/lnet/api-ni.c index 20c96b3..5b53f54 100644 --- a/lnet/lnet/api-ni.c +++ b/lnet/lnet/api-ni.c @@ -22,10 +22,13 @@ #define DEBUG_SUBSYSTEM S_PORTALS #include -int ptl_init; /* PtlInit() flag */ -struct list_head ptl_nal_table; /* registered NALs */ ptl_apini_t ptl_apini; /* THE network interface (at the API) */ +#define DEFAULT_NETWORKS "tcp" +static char *networks = DEFAULT_NETWORKS; +CFS_MODULE_PARM(networks, "s", charp, 0444, + "local networks (default='"DEFAULT_NETWORKS"')"); + void ptl_assert_wire_constants (void) { /* Wire protocol assertions generated by 'wirecheck' @@ -119,37 +122,14 @@ void ptl_assert_wire_constants (void) CLASSERT ((int)sizeof(((ptl_hdr_t *)0)->msg.hello.type) == 4); } -#ifdef __KERNEL__ -struct semaphore ptl_mutex ; - -static void ptl_mutex_enter (void) -{ - mutex_down (&ptl_mutex); -} - -static void ptl_mutex_exit (void) -{ - mutex_up (&ptl_mutex); -} -#else -static void ptl_mutex_enter (void) -{ -} - -static void ptl_mutex_exit (void) -{ -} -#endif - - ptl_nal_t * ptl_find_nal_by_type (int type) { ptl_nal_t *nal; struct list_head *tmp; - /* holding mutex */ - list_for_each (tmp, &ptl_nal_table) { + /* holding nal mutex */ + list_for_each (tmp, &ptl_apini.apini_nals) { nal = list_entry(tmp, ptl_nal_t, nal_list); if (nal->nal_type == type) @@ -163,30 +143,30 @@ ptl_find_nal_by_type (int type) void ptl_register_nal (ptl_nal_t *nal) { - ptl_mutex_enter(); + PTL_MUTEX_DOWN(&ptl_apini.apini_nal_mutex); - LASSERT (ptl_init); + LASSERT (ptl_apini.apini_init); LASSERT (libcfs_isknown_nal(nal->nal_type)); LASSERT (ptl_find_nal_by_type(nal->nal_type) == NULL); - list_add (&nal->nal_list, &ptl_nal_table); + list_add (&nal->nal_list, &ptl_apini.apini_nals); atomic_set(&nal->nal_refcount, 0); - ptl_mutex_exit(); + PTL_MUTEX_UP(&ptl_apini.apini_nal_mutex); } void ptl_unregister_nal (ptl_nal_t *nal) { - ptl_mutex_enter(); + PTL_MUTEX_DOWN(&ptl_apini.apini_nal_mutex); - LASSERT (ptl_init); + LASSERT (ptl_apini.apini_init); LASSERT (ptl_find_nal_by_type(nal->nal_type) == nal); LASSERT (atomic_read(&nal->nal_refcount) == 0); list_del (&nal->nal_list); - ptl_mutex_exit(); + PTL_MUTEX_UP(&ptl_apini.apini_nal_mutex); } #ifndef PTL_USE_LIB_FREELIST @@ -420,6 +400,14 @@ ptl_apini_init(ptl_pid_t requested_pid, LASSERT (ptl_apini.apini_refcount == 0); + ptl_apini.apini_net_tokens_nob = strlen(networks) + 1; + PORTAL_ALLOC(ptl_apini.apini_net_tokens, + ptl_apini.apini_net_tokens_nob); + if (ptl_apini.apini_net_tokens == NULL) { + CERROR("Can't allocate net tokens\n"); + goto out; + } + ptl_apini.apini_pid = requested_pid; rc = ptl_descriptor_setup (requested_limits, @@ -437,14 +425,6 @@ ptl_apini_init(ptl_pid_t requested_pid, CFS_INIT_LIST_HEAD (&ptl_apini.apini_nis); CFS_INIT_LIST_HEAD (&ptl_apini.apini_zombie_nis); -#ifdef __KERNEL__ - spin_lock_init (&ptl_apini.apini_lock); - cfs_waitq_init (&ptl_apini.apini_waitq); -#else - pthread_mutex_init(&ptl_apini.apini_mutex, NULL); - pthread_cond_init(&ptl_apini.apini_cond, NULL); -#endif - ptl_apini.apini_interface_cookie = ptl_create_interface_cookie(); rc = ptl_setup_handle_hash (); @@ -485,6 +465,9 @@ ptl_apini_init(ptl_pid_t requested_pid, if (rc != PTL_OK) { ptl_cleanup_handle_hash (); ptl_descriptor_cleanup (); + if (ptl_apini.apini_net_tokens != NULL) + PORTAL_FREE(ptl_apini.apini_net_tokens, + ptl_apini.apini_net_tokens_nob); } RETURN (rc); @@ -550,6 +533,7 @@ ptl_apini_fini (void) ptl_cleanup_handle_hash (); ptl_descriptor_cleanup (); + PORTAL_FREE(ptl_apini.apini_net_tokens, ptl_apini.apini_net_tokens_nob); #ifndef __KERNEL__ pthread_mutex_destroy(&ptl_apini.apini_mutex); @@ -587,7 +571,7 @@ ptl_queue_zombie_ni (ptl_ni_t *ni) unsigned long flags; LASSERT (atomic_read(&ni->ni_refcount) == 0); - LASSERT (ptl_init); + LASSERT (ptl_apini.apini_init); PTL_LOCK(flags); list_add_tail(&ni->ni_list, &ptl_apini.apini_zombie_nis); @@ -662,83 +646,393 @@ ptl_shutdown_nalnis (void) PTL_UNLOCK(flags); } +void ptl_syntax(char *name, char *str, int offset, int width) +{ + const char *dots = "................................" + "................................" + "................................" + "................................" + "................................" + "................................" + "................................" + "................................"; + const char *dashes = "--------------------------------" + "--------------------------------" + "--------------------------------" + "--------------------------------" + "--------------------------------" + "--------------------------------" + "--------------------------------" + "--------------------------------"; + + LCONSOLE_ERROR("Error parsing '%s=\"%s\"'\n", name, str); + LCONSOLE_ERROR("here...........%.*s..%.*s|%.*s|\n", + strlen(name), dots, offset, dots, + (width < 1) ? 0 : width - 1, dashes); +} + +int +ptl_nis_conflict(ptl_ni_t *ni1, ptl_ni_t *ni2) +{ + int i; + int j; + + if (PTL_NETNAL(PTL_NIDNET(ni1->ni_nid)) != /* different NALs */ + PTL_NETNAL(PTL_NIDNET(ni2->ni_nid))) + return 0; + + if (ni1 != ni2 && + PTL_NIDNET(ni1->ni_nid) == PTL_NIDNET(ni2->ni_nid)) { + CERROR("Duplicate network: %s\n", + libcfs_net2str(PTL_NIDNET(ni1->ni_nid))); + return 1; + } + + if (ni1->ni_interfaces[0] == NULL || + ni2->ni_interfaces[0] == NULL) { + /* one (or both) using all available interfaces */ + if (ni1 != ni2) { + CERROR("Interface conflict: %s, %s\n", + libcfs_net2str(PTL_NIDNET(ni1->ni_nid)), + libcfs_net2str(PTL_NIDNET(ni2->ni_nid))); + return 1; + } + return 0; + } + + for (i = 0; i < PTL_MAX_INTERFACES; i++) { + if (ni1->ni_interfaces[i] == NULL) + break; + + for (j = 0; j < PTL_MAX_INTERFACES; j++) { + if (ni2->ni_interfaces[j] == NULL) + break; + + if (ni1 == ni2 && i == j) + continue; + + if (strcmp(ni1->ni_interfaces[i], + ni2->ni_interfaces[j])) + continue; + + CERROR("Duplicate interface: %s(%s), %s(%s)\n", + libcfs_net2str(PTL_NIDNET(ni1->ni_nid)), + ni1->ni_interfaces[i], + libcfs_net2str(PTL_NIDNET(ni2->ni_nid)), + ni2->ni_interfaces[i]); + return 1; + } + } + + return 0; +} + +ptl_err_t +ptl_check_ni_conflicts(ptl_ni_t *ni, struct list_head *nilist) +{ + struct list_head *tmp; + ptl_ni_t *ni2; + + /* Yes! ni just added to this list. + * Check its network is unique and its interfaces don't conflict */ + LASSERT (ni == list_entry(nilist->prev, ptl_ni_t, ni_list)); + + list_for_each (tmp, nilist) { + ni2 = list_entry(tmp, ptl_ni_t, ni_list); + + if (ptl_nis_conflict(ni, ni2)) + return PTL_FAIL; + } + + return PTL_OK; +} + +int +ptl_iswhite (char c) +{ + switch (c) { + case ' ': + case '\t': + case '\n': + case '\r': + return 1; + default: + return 0; + } +} + +char * +ptl_trimwhite(char *str) +{ + char *end; + + while (ptl_iswhite(*str)) + str++; + + end = str + strlen(str); + while (end > str) { + if (!ptl_iswhite(end[-1])) + break; + end--; + } + + *end = 0; + return str; +} + +ptl_err_t +ptl_parse_networks(struct list_head *nilist) +{ + char *tokens = ptl_apini.apini_net_tokens; + char *str = tokens; + ptl_ni_t *ni = NULL; + __u32 net; + int rc; + + LASSERT (ptl_apini.apini_net_tokens_nob == strlen(networks) + 1); + memcpy (tokens, networks, ptl_apini.apini_net_tokens_nob); + + while (str != NULL && *str != 0) { + char *comma = strchr(str, ','); + char *bracket = strchr(str, '('); + int niface; + char *iface; + + PORTAL_ALLOC(ni, sizeof(*ni)); + if (ni == NULL) { + CERROR ("ENOMEM parsing 'networks=\"%s\"'\n", networks); + goto failed; + } + /* zero counters/flags, NULL pointers... */ + memset(ni, 0, sizeof(*ni)); + list_add_tail(&ni->ni_list, nilist); + + if (bracket == NULL || + (comma != NULL && comma < bracket)) { + if (comma != NULL) + *comma++ = 0; + net = libcfs_str2net(ptl_trimwhite(str)); + + if (net == PTL_NIDNET(PTL_NID_ANY)) { + ptl_syntax("networks", networks, + str - tokens, strlen(str)); + goto failed; + } + + ni->ni_nid = PTL_MKNID(net, 0); + if (ptl_check_ni_conflicts(ni, nilist) != PTL_OK) + goto failed; + + str = comma; + continue; + } + + *bracket = 0; + net = libcfs_str2net(ptl_trimwhite(str)); + if (net == PTL_NIDNET(PTL_NID_ANY)) { + ptl_syntax("networks", networks, + str - tokens, strlen(str)); + goto failed; + } + + ni->ni_nid = PTL_MKNID(net, 0); + + niface = 0; + iface = bracket + 1; + + bracket = strchr(iface, ')'); + if (bracket == NULL) { + ptl_syntax ("networks", networks, + iface - tokens, strlen(iface)); + goto failed; + } + + *bracket = 0; + do { + comma = strchr(iface, ','); + if (comma != NULL) + *comma++ = 0; + + iface = ptl_trimwhite(iface); + if (*iface == 0) { + ptl_syntax("networks", networks, + iface - tokens, strlen(iface)); + goto failed; + } + + if (niface == PTL_MAX_INTERFACES) { + LCONSOLE_ERROR("Too many interfaces for %s\n", + libcfs_net2str(PTL_NIDNET(ni->ni_nid))); + goto failed; + } + + ni->ni_interfaces[niface++] = iface; + iface = comma; + } while (iface != NULL); + + if (ptl_check_ni_conflicts(ni, nilist) != PTL_OK) + goto failed; + + str = bracket + 1; + comma = strchr(bracket + 1, ','); + if (comma != NULL) { + *comma = 0; + str = ptl_trimwhite(str); + if (*str != 0) { + ptl_syntax ("networks", networks, + str - tokens, strlen(str)); + goto failed; + } + str = comma + 1; + continue; + } + + str = ptl_trimwhite(str); + if (*str != 0) { + ptl_syntax ("networks", networks, + str - tokens, strlen(str)); + goto failed; + } + } + + if (list_empty(nilist)) { + LCONSOLE_ERROR("No networks specified\n"); + goto failed; + } + return PTL_OK; + + failed: + while (!list_empty(nilist)) { + ni = list_entry(nilist->next, ptl_ni_t, ni_list); + + list_del(&ni->ni_list); + PORTAL_FREE(ni, sizeof(*ni)); + } + return PTL_FAIL; +} + +ptl_err_t +ptl_load_nal (int type) +{ + CERROR("Automatic NAL loading not implemented (%s)\n", + libcfs_nal2str(type)); + return PTL_FAIL; +} + ptl_err_t ptl_startup_nalnis (void) { ptl_nal_t *nal; ptl_ni_t *ni; - ptl_ni_t *ni2; - struct list_head *tmp; - struct list_head *tmp2; + struct list_head nilist; ptl_err_t rc = PTL_OK; char *interface = NULL; unsigned long flags; + int nal_type; + int retry; + + INIT_LIST_HEAD(&nilist); + rc = ptl_parse_networks(&nilist); + if (rc != PTL_OK) + return rc; - list_for_each (tmp, &ptl_nal_table) { - nal = list_entry(tmp, ptl_nal_t, nal_list); + while (!list_empty(&nilist)) { + ni = list_entry(nilist.next, ptl_ni_t, ni_list); + nal_type = PTL_NETNAL(PTL_NIDNET(ni->ni_nid)); - PORTAL_ALLOC(ni, sizeof(*ni)); - if (ni == NULL) { - CERROR("Can't allocate NI for %s NAL\n", - libcfs_nal2str(nal->nal_type)); - rc = PTL_FAIL; - break; + LASSERT (libcfs_isknown_nal(nal_type)); + + PTL_MUTEX_DOWN(&ptl_apini.apini_nal_mutex); + + for (retry = 0;; retry = 1) { + nal = ptl_find_nal_by_type(nal_type); + if (nal != NULL) + break; + + PTL_MUTEX_UP(&ptl_apini.apini_nal_mutex); + + if (retry) { + CERROR("Can't load NAL %s\n", + libcfs_nal2str(nal_type)); + goto failed; + } + + rc = ptl_load_nal(nal_type); + if (rc != PTL_OK) + goto failed; + + PTL_MUTEX_DOWN(&ptl_apini.apini_nal_mutex); } atomic_set(&ni->ni_refcount, 1); atomic_inc(&nal->nal_refcount); ni->ni_nal = nal; - ni->ni_nid = PTL_MKNID(PTL_MKNET(nal->nal_type, 0), 0); - /* for now */ - rc = (nal->nal_startup)(ni, &interface); + rc = (nal->nal_startup)(ni); + + PTL_MUTEX_UP(&ptl_apini.apini_nal_mutex); + if (rc != PTL_OK) { CERROR("Error %d staring up NI %s\n", rc, libcfs_nal2str(nal->nal_type)); - PORTAL_FREE(ni, sizeof(*ni)); atomic_dec(&nal->nal_refcount); - break; + goto failed; } + list_del(&ni->ni_list); + PTL_LOCK(flags); list_add_tail(&ni->ni_list, &ptl_apini.apini_nis); PTL_UNLOCK(flags); } - - if (rc != PTL_OK) - ptl_shutdown_nalnis(); - return rc; + return PTL_OK; + + failed: + ptl_shutdown_nalnis(); + + while (!list_empty(&nilist)) { + ni = list_entry(nilist.next, ptl_ni_t, ni_list); + list_del(&ni->ni_list); + PORTAL_FREE(ni, sizeof(*ni)); + } + + return PTL_FAIL; } +#ifndef __KERNEL__ +extern ptl_nal_t tcpnal_nal; +#endif + ptl_err_t PtlInit(int *max_interfaces) { LASSERT(!strcmp(ptl_err_str[PTL_MAX_ERRNO], "PTL_MAX_ERRNO")); ptl_assert_wire_constants (); - ptl_mutex_enter(); - - LASSERT (!ptl_init); + LASSERT (!ptl_apini.apini_init); - CFS_INIT_LIST_HEAD(&ptl_nal_table); ptl_apini.apini_refcount = 0; + CFS_INIT_LIST_HEAD(&ptl_apini.apini_nals); + +#ifdef __KERNEL__ + spin_lock_init (&ptl_apini.apini_lock); + cfs_waitq_init (&ptl_apini.apini_waitq); + init_mutex(&ptl_apini.apini_nal_mutex); + init_mutex(&ptl_apini.apini_api_mutex); +#else + pthread_mutex_init(&ptl_apini.apini_mutex, NULL); + pthread_cond_init(&ptl_apini.apini_cond, NULL); + pthread_mutex_init(&ptl_apini.apini_nal_mutex); + pthread_mutex_init(&ptl_apini.apini_api_mutex); -#ifndef __KERNEL__ - /* process */ /* Kernel NALs register themselves when their module loads, and * unregister themselves when their module is unloaded. Userspace NALs * are plugged in explicitly here... */ - { - extern ptl_nal_t tcpnal_nal; - - ptl_register_nal (&tcpnal_nal); - } + ptl_register_nal (&tcpnal_nal); #endif - ptl_init = 1; + ptl_apini.apini_init = 1; - ptl_mutex_exit(); - if (max_interfaces != NULL) *max_interfaces = 1; @@ -748,14 +1042,16 @@ PtlInit(int *max_interfaces) void PtlFini(void) { - ptl_mutex_enter(); - - LASSERT (ptl_init); - LASSERT (list_empty(&ptl_nal_table)); + LASSERT (ptl_apini.apini_init); LASSERT (ptl_apini.apini_refcount == 0); - ptl_init = 0; - ptl_mutex_exit(); +#ifndef __KERNEL__ + /* See comment where tcpnal_nal registers itself */ + ptl_unregister_nal(&tcpnal_nal); +#endif + LASSERT (list_empty(&ptl_apini.apini_nals)); + + ptl_apini.apini_init = 0; } ptl_err_t @@ -765,9 +1061,9 @@ PtlNIInit(ptl_interface_t interface, ptl_pid_t requested_pid, { int rc; - ptl_mutex_enter (); + PTL_MUTEX_DOWN(&ptl_apini.apini_api_mutex); - LASSERT (ptl_init); + LASSERT (ptl_apini.apini_init); CDEBUG(D_OTHER, "refs %d\n", ptl_apini.apini_refcount); if (ptl_apini.apini_refcount > 0) { @@ -795,17 +1091,16 @@ PtlNIInit(ptl_interface_t interface, ptl_pid_t requested_pid, /* Handle can be anything; PTL_INVALID_HANDLE isn't wise though :) */ out: - ptl_mutex_exit (); - + PTL_MUTEX_UP(&ptl_apini.apini_api_mutex); return rc; } ptl_err_t PtlNIFini(ptl_handle_ni_t ni) { - ptl_mutex_enter (); + PTL_MUTEX_DOWN(&ptl_apini.apini_api_mutex); - LASSERT (ptl_init); + LASSERT (ptl_apini.apini_init); LASSERT (ptl_apini.apini_refcount > 0); ptl_apini.apini_refcount--; @@ -815,7 +1110,7 @@ PtlNIFini(ptl_handle_ni_t ni) ptl_apini_fini(); } - ptl_mutex_exit (); + PTL_MUTEX_UP(&ptl_apini.apini_api_mutex); return PTL_OK; } @@ -829,9 +1124,7 @@ PtlNICtl(ptl_handle_ni_t nih, unsigned int cmd, void *arg) unsigned long flags; int count; - ptl_mutex_enter (); - - LASSERT (ptl_init); + LASSERT (ptl_apini.apini_init); LASSERT (ptl_apini.apini_refcount > 0); switch (cmd) { @@ -879,7 +1172,6 @@ PtlNICtl(ptl_handle_ni_t nih, unsigned int cmd, void *arg) break; } - ptl_mutex_exit(); return rc; } @@ -889,7 +1181,7 @@ PtlGetId(ptl_handle_ni_t ni_handle, ptl_process_id_t *id) ptl_ni_t *ni; unsigned long flags; - LASSERT (ptl_init); + LASSERT (ptl_apini.apini_init); LASSERT (ptl_apini.apini_refcount > 0); /* pretty useless; just return the NID of the first local interface */ @@ -915,7 +1207,7 @@ PtlGetId(ptl_handle_ni_t ni_handle, ptl_process_id_t *id) ptl_err_t PtlNIHandle(ptl_handle_any_t handle_in, ptl_handle_ni_t *ni_out) { - LASSERT (ptl_init); + LASSERT (ptl_apini.apini_init); LASSERT (ptl_apini.apini_refcount > 0); *ni_out = handle_in; @@ -931,7 +1223,7 @@ PtlSnprintHandle(char *str, int len, ptl_handle_any_t h) ptl_err_t PtlGetUid(ptl_handle_ni_t ni_handle, ptl_uid_t *uid) { - LASSERT (ptl_init); + LASSERT (ptl_apini.apini_init); LASSERT (ptl_apini.apini_refcount > 0); *uid = 0; /* fake it */ @@ -942,7 +1234,7 @@ ptl_err_t PtlNIDist(ptl_handle_ni_t interface_in, ptl_process_id_t process_in, unsigned long *distance_out) { - LASSERT (ptl_init); + LASSERT (ptl_apini.apini_init); LASSERT (ptl_apini.apini_refcount > 0); return 1; /* fake it */ @@ -952,7 +1244,7 @@ ptl_err_t PtlNIStatus(ptl_handle_ni_t interface_in, ptl_sr_index_t register_in, ptl_sr_value_t *status_out) { - LASSERT (ptl_init); + LASSERT (ptl_apini.apini_init); LASSERT (ptl_apini.apini_refcount > 0); return PTL_FAIL; /* not supported */ @@ -962,7 +1254,7 @@ ptl_err_t PtlACEntry(ptl_handle_ni_t ni_in, ptl_ac_index_t index_in, ptl_process_id_t match_id_in, ptl_pt_index_t portal_in) { - LASSERT (ptl_init); + LASSERT (ptl_apini.apini_init); LASSERT (ptl_apini.apini_refcount > 0); return PTL_FAIL; /* not supported */ diff --git a/lnet/lnet/lib-eq.c b/lnet/lnet/lib-eq.c index a8db651..d89dcdb 100644 --- a/lnet/lnet/lib-eq.c +++ b/lnet/lnet/lib-eq.c @@ -32,7 +32,7 @@ PtlEQAlloc(ptl_handle_ni_t interface, ptl_size_t count, ptl_eq_t *eq; unsigned long flags; - LASSERT (ptl_init); + LASSERT (ptl_apini.apini_init); LASSERT (ptl_apini.apini_refcount > 0); /* We need count to be a power of 2 so that when eq_{enq,deq}_seq @@ -90,7 +90,7 @@ PtlEQFree(ptl_handle_eq_t eqh) ptl_event_t *events; unsigned long flags; - LASSERT (ptl_init); + LASSERT (ptl_apini.apini_init); LASSERT (ptl_apini.apini_refcount > 0); PTL_LOCK(flags); @@ -188,7 +188,7 @@ PtlEQPoll (ptl_handle_eq_t *eventqs, int neq, int timeout_ms, #endif ENTRY; - LASSERT (ptl_init); + LASSERT (ptl_apini.apini_init); LASSERT (ptl_apini.apini_refcount > 0); if (neq < 1) diff --git a/lnet/lnet/lib-md.c b/lnet/lnet/lib-md.c index f47b033..ac3c05e 100644 --- a/lnet/lnet/lib-md.c +++ b/lnet/lnet/lib-md.c @@ -200,7 +200,7 @@ PtlMDAttach(ptl_handle_me_t meh, ptl_md_t umd, unsigned long flags; int rc; - LASSERT (ptl_init); + LASSERT (ptl_apini.apini_init); LASSERT (ptl_apini.apini_refcount > 0); if ((umd.options & (PTL_MD_KIOV | PTL_MD_IOVEC)) != 0 && @@ -245,7 +245,7 @@ PtlMDBind(ptl_handle_ni_t nih, ptl_md_t umd, unsigned long flags; int rc; - LASSERT (ptl_init); + LASSERT (ptl_apini.apini_init); LASSERT (ptl_apini.apini_refcount > 0); if ((umd.options & (PTL_MD_KIOV | PTL_MD_IOVEC)) != 0 && @@ -280,7 +280,7 @@ PtlMDUnlink (ptl_handle_md_t mdh) ptl_libmd_t *md; unsigned long flags; - LASSERT (ptl_init); + LASSERT (ptl_apini.apini_init); LASSERT (ptl_apini.apini_refcount > 0); PTL_LOCK(flags); @@ -324,7 +324,7 @@ PtlMDUpdate(ptl_handle_md_t mdh, unsigned long flags; int rc; - LASSERT (ptl_init); + LASSERT (ptl_apini.apini_init); LASSERT (ptl_apini.apini_refcount > 0); PTL_LOCK(flags); diff --git a/lnet/lnet/lib-me.c b/lnet/lnet/lib-me.c index 2891222..d10af61 100644 --- a/lnet/lnet/lib-me.c +++ b/lnet/lnet/lib-me.c @@ -38,7 +38,7 @@ PtlMEAttach(ptl_handle_ni_t interface, ptl_me_t *me; unsigned long flags; - LASSERT (ptl_init); + LASSERT (ptl_apini.apini_init); LASSERT (ptl_apini.apini_refcount > 0); if (portal >= ptl_apini.apini_nportals) @@ -82,7 +82,7 @@ PtlMEInsert(ptl_handle_me_t current_meh, ptl_me_t *new_me; unsigned long flags; - LASSERT (ptl_init); + LASSERT (ptl_apini.apini_init); LASSERT (ptl_apini.apini_refcount > 0); new_me = ptl_me_alloc(); @@ -126,7 +126,7 @@ PtlMEUnlink(ptl_handle_me_t meh) ptl_me_t *me; int rc; - LASSERT (ptl_init); + LASSERT (ptl_apini.apini_init); LASSERT (ptl_apini.apini_refcount > 0); PTL_LOCK(flags); diff --git a/lnet/lnet/lib-move.c b/lnet/lnet/lib-move.c index 5658166..24b26ff 100644 --- a/lnet/lnet/lib-move.c +++ b/lnet/lnet/lib-move.c @@ -167,7 +167,7 @@ ptl_fail_nid (ptl_nid_t nid, unsigned int threshold) struct list_head *next; struct list_head cull; - LASSERT (ptl_init); + LASSERT (ptl_apini.apini_init); if (threshold != 0) { /* Adding a new entry */ @@ -1463,7 +1463,7 @@ PtlPut(ptl_handle_md_t mdh, ptl_ack_req_t ack, unsigned long flags; int rc; - LASSERT (ptl_init); + LASSERT (ptl_apini.apini_init); LASSERT (ptl_apini.apini_refcount > 0); if (!list_empty (&ptl_apini.apini_test_peers) && /* normally we don't */ @@ -1619,7 +1619,7 @@ PtlGet(ptl_handle_md_t mdh, ptl_process_id_t target, unsigned long flags; int rc; - LASSERT (ptl_init); + LASSERT (ptl_apini.apini_init); LASSERT (ptl_apini.apini_refcount > 0); if (!list_empty (&ptl_apini.apini_test_peers) && /* normally we don't */ diff --git a/lnet/lnet/module.c b/lnet/lnet/module.c index 526183f..6ec216e 100644 --- a/lnet/lnet/module.c +++ b/lnet/lnet/module.c @@ -42,14 +42,12 @@ static int kportal_ioctl(unsigned int cmd, struct portal_ioctl_data *data) } DECLARE_IOCTL_HANDLER(kportal_ioctl_handler, kportal_ioctl); -extern struct semaphore ptl_mutex; static int init_kportals_module(void) { int rc; ENTRY; - init_mutex(&ptl_mutex); rc = PtlInit(NULL); if (rc) { CERROR("PtlInit: error %d\n", rc); diff --git a/lnet/ulnds/procapi.c b/lnet/ulnds/procapi.c index 3aec8b8..e404fe4 100644 --- a/lnet/ulnds/procapi.c +++ b/lnet/ulnds/procapi.c @@ -125,7 +125,7 @@ procbridge __global_procbridge = NULL; * error wrapper to cut down clutter. */ ptl_err_t -procbridge_startup (ptl_ni_t *ni, char **interfaces) +procbridge_startup (ptl_ni_t *ni) { procbridge p; bridge b; diff --git a/lnet/ulnds/socklnd/procapi.c b/lnet/ulnds/socklnd/procapi.c index 3aec8b8..e404fe4 100644 --- a/lnet/ulnds/socklnd/procapi.c +++ b/lnet/ulnds/socklnd/procapi.c @@ -125,7 +125,7 @@ procbridge __global_procbridge = NULL; * error wrapper to cut down clutter. */ ptl_err_t -procbridge_startup (ptl_ni_t *ni, char **interfaces) +procbridge_startup (ptl_ni_t *ni) { procbridge p; bridge b; -- 1.8.3.1