and with different hopcounts; it just uses the shortest hopcount.
NB this still needs a change to LNet{Put,Get} to allow the caller to
specify the source NI and for ptlrpc to use it.
* forwarding="enabled" explicitly enables the node as a router
forwarding="disabled" explicitly disables the node as a router
otherwise if any of the node's NIDs are mentioned as routers in 'routes='
the node is implicitly enabled as a router.
* Added LNET_SINGLE_THREADED to tell LNET if the userspace runtime supports
posix threads.
* Added lnd_wait() to tell LNET that to call into the LND if the APP wants to
block for an event.
}
#ifdef __KERNEL__
-#define LNET_LOCK() \
- spin_lock(&the_lnet.ln_lock)
-#define LNET_UNLOCK() \
- spin_unlock(&the_lnet.ln_lock)
+#define LNET_LOCK() spin_lock(&the_lnet.ln_lock)
+#define LNET_UNLOCK() spin_unlock(&the_lnet.ln_lock)
#define LNET_MUTEX_DOWN(m) mutex_down(m)
#define LNET_MUTEX_UP(m) mutex_up(m)
-#else
-#define LNET_LOCK() \
- pthread_mutex_lock(&the_lnet.ln_mutex)
-#define LNET_UNLOCK() \
- pthread_mutex_unlock(&the_lnet.ln_mutex)
+#else
+# if LNET_SINGLE_THREADED
+#define LNET_SINGLE_THREADED_LOCK(l) \
+do { \
+ LASSERT ((l) == 0); \
+ (l) = 1; \
+} while (0)
+
+#define LNET_SINGLE_THREADED_UNLOCK(l) \
+do { \
+ LASSERT ((l) == 1); \
+ (l) = 0; \
+} while (0)
+
+#define LNET_LOCK() LNET_SINGLE_THREADED_LOCK(the_lnet.ln_lock)
+#define LNET_UNLOCK() LNET_SINGLE_THREADED_UNLOCK(the_lnet.ln_lock)
+#define LNET_MUTEX_DOWN(m) LNET_SINGLE_THREADED_LOCK(*(m))
+#define LNET_MUTEX_UP(m) LNET_SINGLE_THREADED_UNLOCK(*(m))
+# else
+#define LNET_LOCK() pthread_mutex_lock(&the_lnet.ln_lock)
+#define LNET_UNLOCK() pthread_mutex_unlock(&the_lnet.ln_lock)
#define LNET_MUTEX_DOWN(m) pthread_mutex_lock(m)
#define LNET_MUTEX_UP(m) pthread_mutex_unlock(m)
+# endif
#endif
#define MAX_PORTALS 64
return &the_lnet.ln_peer_hash[idx];
}
-extern lnd_t the_lolnd;
-extern lnet_ni_t *lnet_loni;
+extern lnd_t the_lolnd;
#ifndef __KERNEL__
#define LNET_REGISTER_LND_IF_PRESENT(lnd) \
lnet_nid_t *gateway, __u32 *alive);
void lnet_proc_init(void);
void lnet_proc_fini(void);
-int lnet_alloc_rtrpools(void);
+void lnet_init_rtrpools(void);
+int lnet_alloc_rtrpools(int im_a_router);
void lnet_free_rtrpools(void);
lnet_remotenet_t *lnet_find_net_locked (__u32 net);
int lnet_acceptor_start(void);
void lnet_acceptor_stop(void);
-int lnet_parse_routes (char *route_str);
+int lnet_parse_routes (char *route_str, int *im_a_router);
int lnet_parse_networks (struct list_head *nilist, char *networks);
int lnet_nid2peer_locked(lnet_peer_t **lpp, lnet_nid_t nid);
* code in the header's dest_nid, the peer's NID in the src_nid, and
* LNET_MSG_HELLO in the type field. All other common fields are zero
* (including payload_size; i.e. no payload).
- * This is for use by byte-stream NALs (e.g. TCP/IP) to check the peer is
- * running the same protocol and to find out its NID. These NALs should
+ * This is for use by byte-stream LNDs (e.g. TCP/IP) to check the peer is
+ * running the same protocol and to find out its NID. These LNDs should
* exchange HELLO messages when a connection is first established. Individual
- * NALs can put whatever else they fancy in lnet_hdr_t::msg.
+ * LNDs can put whatever else they fancy in lnet_hdr_t::msg.
*/
typedef struct {
__u32 magic; /* LNET_PROTO_TCP_MAGIC */
__u16 version_minor; /* increment on compatible change */
} WIRE_ATTR lnet_magicversion_t;
-/* PROTO MAGIC for NALs that once used their own private acceptor */
+/* PROTO MAGIC for LNDs that once used their own private acceptor */
#define LNET_PROTO_OPENIB_MAGIC 0x0be91b91
#define LNET_PROTO_RA_MAGIC 0x0be91b92
#define LNET_PROTO_TCP_MAGIC 0xeebc0ded
typedef struct lnet_lnd
{
/* fields managed by portals */
- struct list_head lnd_list; /* stash in the NAL table */
+ struct list_head lnd_list; /* stash in the LND table */
int lnd_refcount; /* # active instances */
- /* fields initialised by the NAL */
+ /* fields initialised by the LND */
unsigned int lnd_type;
int (*lnd_startup) (struct lnet_ni *ni);
* in virtual memory (struct iovec *iov != NULL)
* OR
* in pages (kernel only: plt_kiov_t *kiov != NULL).
- * The NAL may NOT overwrite these fragment descriptors.
+ * The LND may NOT overwrite these fragment descriptors.
* An 'offset' and may specify a byte offset within the set of
* fragments to start from
*/
#ifdef __KERNEL__
/* accept a new connection */
int (*lnd_accept)(struct lnet_ni *ni, struct socket *sock);
+#else
+ /* wait for something to happen */
+ void (*lnd_wait)(struct lnet_ni *ni, int milliseconds);
#endif
} lnd_t;
int ln_ptlcompat; /* do I support talking to portals? */
- struct list_head ln_lnds; /* registered NALs */
+ struct list_head ln_lnds; /* registered LNDs */
#ifdef __KERNEL__
spinlock_t ln_lock;
struct semaphore ln_api_mutex;
struct semaphore ln_lnd_mutex;
#else
- pthread_mutex_t ln_mutex;
+# if LNET_SINGLE_THREADED
+ int ln_lock;
+ int ln_api_mutex;
+ int ln_lnd_mutex;
+# else
pthread_cond_t ln_cond;
+ pthread_mutex_t ln_lock;
pthread_mutex_t ln_api_mutex;
pthread_mutex_t ln_lnd_mutex;
+# endif
#endif
/* Stuff initialised at LNetNIInit() */
lnet_pid_t ln_pid; /* requested pid */
- struct list_head ln_nis; /* NAL instances */
- struct list_head ln_zombie_nis; /* dying NAL instances */
- int ln_nzombie_nis; /* # of NIS to wait for */
+ struct list_head ln_nis; /* LND instances */
+ lnet_ni_t *ln_loni; /* the loopback NI */
+ lnet_ni_t *ln_eqwaitni; /* NI to wait for events in */
+ struct list_head ln_zombie_nis; /* dying LND instances */
+ int ln_nzombie_nis; /* # of NIs to wait for */
struct list_head ln_remote_nets; /* remote networks with routes to them */
__u64 ln_remote_nets_version; /* validity stamp */
return networks;
}
+int
+lnet_get_portals_compatibility(void)
+{
+ if (!strcmp(portals_compatibility, "none")) {
+ return 0;
+ }
+
+ if (!strcmp(portals_compatibility, "weak")) {
+ return 1;
+ LCONSOLE_WARN("Starting in weak portals-compatible mode\n");
+ }
+
+ if (!strcmp(portals_compatibility, "strong")) {
+ return 2;
+ LCONSOLE_WARN("Starting in strong portals-compatible mode\n");
+ }
+
+ LCONSOLE_ERROR("portals_compatibility=\"%s\" not supported\n",
+ portals_compatibility);
+ return -EINVAL;
+}
+
+void
+lnet_init_locks(void)
+{
+ spin_lock_init (&the_lnet.ln_lock);
+ cfs_waitq_init (&the_lnet.ln_waitq);
+ init_mutex(&the_lnet.ln_lnd_mutex);
+ init_mutex(&the_lnet.ln_api_mutex);
+}
+
+void
+lnet_fini_locks(void)
+{
+}
+
#else
char *
return default_networks;
}
-void lnet_proc_init(void)
+int
+lnet_get_portals_compatibility(void)
{
+ return 0;
+}
+
+# if LNET_SINGLE_THREADED
+
+void lnet_init_locks(void)
+{
+ the_lnet.ln_lock = 0;
+ the_lnet.ln_lnd_mutex = 0;
+ the_lnet.ln_api_mutex = 0;
+}
+
+void lnet_fini_locks(void)
+{
+ LASSERT (the_lnet.ln_api_mutex == 0);
+ LASSERT (the_lnet.ln_lnd_mutex == 0);
+ LASSERT (the_lnet.ln_lock == 0);
+}
+
+# else
+
+void lnet_init_locks(void)
+{
+ pthread_cond_init(&the_lnet.ln_cond, NULL);
+ pthread_mutex_init(&the_lnet.ln_lock, NULL);
+ pthread_mutex_init(&the_lnet.ln_lnd_mutex, NULL);
+ pthread_mutex_init(&the_lnet.ln_api_mutex, NULL);
}
-void lnet_proc_fini(void)
+void lnet_fini_locks(void)
{
+ pthread_mutex_destroy(&the_lnet.ln_api_mutex);
+ pthread_mutex_destroy(&the_lnet.ln_lnd_mutex);
+ pthread_mutex_destroy(&the_lnet.ln_lock);
+ pthread_cond_destroy(&the_lnet.ln_cond);
}
+# endif
#endif
void lnet_assert_wire_constants (void)
}
int
-lnet_init(lnet_pid_t requested_pid)
+lnet_prepare(lnet_pid_t requested_pid)
{
+ /* Prepare to bring up the network */
int rc = 0;
int i;
the_lnet.ln_interface_cookie = lnet_create_interface_cookie();
+ lnet_init_rtrpools();
+
rc = lnet_setup_handle_hash ();
if (rc != 0)
goto failed0;
if (rc != 0)
goto failed1;
- rc = lnet_alloc_rtrpools();
- if (rc != 0)
- goto failed2;
-
the_lnet.ln_nportals = MAX_PORTALS;
PORTAL_ALLOC(the_lnet.ln_portals,
the_lnet.ln_nportals *
sizeof(*the_lnet.ln_portals));
if (the_lnet.ln_portals == NULL) {
rc = -ENOMEM;
- goto failed3;
+ goto failed2;
}
for (i = 0; i < the_lnet.ln_nportals; i++)
return 0;
- failed3:
- lnet_free_rtrpools();
failed2:
lnet_destroy_peer_table();
failed1:
}
int
-lnet_fini (void)
+lnet_unprepare (void)
{
int idx;
lnet_ni_decref_locked(ni); /* drop apini's ref */
}
+ /* Drop the cached eqwait NI. */
+ if (the_lnet.ln_eqwaitni != NULL) {
+ lnet_ni_decref_locked(the_lnet.ln_eqwaitni);
+ the_lnet.ln_eqwaitni = NULL;
+ }
+
/* Drop the cached loopback NI. */
- if (lnet_loni != NULL) {
- lnet_ni_decref_locked(lnet_loni);
- lnet_loni = NULL;
+ if (the_lnet.ln_loni != NULL) {
+ lnet_ni_decref_locked(the_lnet.ln_loni);
+ the_lnet.ln_loni = NULL;
}
+
LNET_UNLOCK();
/* Clear the peer table and wait for all peers to go (they hold refs on
* their NIs) */
struct list_head nilist;
int rc = 0;
int lnd_type;
- int retry;
+ int nicount = 0;
INIT_LIST_HEAD(&nilist);
rc = lnet_parse_networks(&nilist, lnet_get_networks());
LASSERT (libcfs_isknown_lnd(lnd_type));
LNET_MUTEX_DOWN(&the_lnet.ln_lnd_mutex);
+ lnd = lnet_find_lnd_by_type(lnd_type);
- for (retry = 0;; retry = 1) {
- lnd = lnet_find_lnd_by_type(lnd_type);
- if (lnd != NULL)
- break;
-
- LNET_MUTEX_UP(&the_lnet.ln_lnd_mutex);
#ifdef __KERNEL__
- if (retry) {
+ if (lnd == NULL) {
+ LNET_MUTEX_UP(&the_lnet.ln_lnd_mutex);
+ request_module(libcfs_lnd2modname(lnd_type));
+ LNET_MUTEX_DOWN(&the_lnet.ln_lnd_mutex);
+
+ lnd = lnet_find_lnd_by_type(lnd_type);
+ if (lnd == NULL) {
+ LNET_MUTEX_UP(&the_lnet.ln_lnd_mutex);
CERROR("Can't load LND %s, module %s\n",
libcfs_lnd2str(lnd_type),
libcfs_lnd2modname(lnd_type));
goto failed;
}
-
- request_module(libcfs_lnd2modname(lnd_type));
+ }
#else
+ if (lnd == NULL) {
+ LNET_MUTEX_UP(&the_lnet.ln_lnd_mutex);
CERROR("LND %s not supported\n",
libcfs_lnd2str(lnd_type));
goto failed;
-#endif
- LNET_MUTEX_DOWN(&the_lnet.ln_lnd_mutex);
}
+#endif
ni->ni_refcount = 1;
LNET_MUTEX_UP(&the_lnet.ln_lnd_mutex);
if (rc != 0) {
- CERROR("Error %d starting up NI %s\n",
- rc, libcfs_lnd2str(lnd->lnd_type));
+ LCONSOLE_ERROR("Error %d starting up NI %s\n",
+ rc, libcfs_lnd2str(lnd->lnd_type));
LNET_LOCK();
lnd->lnd_refcount--;
LNET_UNLOCK();
goto failed;
}
- if (lnd->lnd_type != LOLND) {
- if (ni->ni_peertxcredits == 0 ||
- ni->ni_maxtxcredits == 0) {
- LCONSOLE_ERROR("NI %s has no %scredits\n",
- libcfs_lnd2str(lnd->lnd_type),
- ni->ni_peertxcredits == 0 ?
- "" : "per-peer ");
- goto failed;
- }
-
- ni->ni_txcredits =
- ni->ni_mintxcredits = ni->ni_maxtxcredits;
-
- LCONSOLE(0, "Added NI %s [%d/%d]\n",
- libcfs_nid2str(ni->ni_nid),
- ni->ni_peertxcredits, ni->ni_txcredits);
-
- /* Handle nidstrings for network 0 just like this one */
- if (the_lnet.ln_ptlcompat > 0)
- libcfs_setnet0alias(lnd->lnd_type);
- }
-
list_del(&ni->ni_list);
LNET_LOCK();
list_add_tail(&ni->ni_list, &the_lnet.ln_nis);
LNET_UNLOCK();
+
+ if (lnd->lnd_type == LOLND) {
+ lnet_ni_addref(ni);
+ LASSERT (the_lnet.ln_loni == NULL);
+ the_lnet.ln_loni = ni;
+ continue;
+ }
+
+#ifndef __KERNEL__
+ if (lnd->lnd_wait != NULL) {
+ if (the_lnet.ln_eqwaitni == NULL) {
+ lnet_ni_addref(ni);
+ the_lnet.ln_eqwaitni = ni;
+ }
+ } else {
+# if LNET_SINGLE_THREADED
+ LCONSOLE_ERROR("LND %s not supported in a "
+ "single-threaded runtime\n",
+ libcfs_lnd2str(lnd_type));
+ goto failed;
+# endif
+ }
+#endif
+ if (ni->ni_peertxcredits == 0 ||
+ ni->ni_maxtxcredits == 0) {
+ LCONSOLE_ERROR("NI %s has no %scredits\n",
+ libcfs_lnd2str(lnd->lnd_type),
+ ni->ni_peertxcredits == 0 ?
+ "" : "per-peer ");
+ goto failed;
+ }
+
+ ni->ni_txcredits = ni->ni_mintxcredits = ni->ni_maxtxcredits;
+
+ LCONSOLE(0, "Added NI %s [%d/%d]\n",
+ libcfs_nid2str(ni->ni_nid),
+ ni->ni_peertxcredits, ni->ni_txcredits);
+
+ /* Handle nidstrings for network 0 just like this one */
+ if (the_lnet.ln_ptlcompat > 0)
+ libcfs_setnet0alias(lnd->lnd_type);
+
+ nicount++;
}
- lnet_loni = lnet_net2ni(PTL_MKNET(LOLND, 0));
- LASSERT (lnet_loni != NULL);
+ if (nicount > 1 && the_lnet.ln_eqwaitni != NULL) {
+ lnd_type = the_lnet.ln_eqwaitni->ni_lnd->lnd_type;
+ LCONSOLE_ERROR("LND %s can only run single-network\n",
+ libcfs_lnd2str(lnd_type));
+ goto failed;
+ }
return 0;
int
LNetInit(void)
{
- lnet_assert_wire_constants ();
+ int rc;
+ lnet_assert_wire_constants ();
LASSERT (!the_lnet.ln_init);
+
+ rc = lnet_get_portals_compatibility();
+ if (rc < 0)
+ return rc;
- the_lnet.ln_refcount = 0;
+ lnet_init_locks();
CFS_INIT_LIST_HEAD(&the_lnet.ln_lnds);
-
-#ifdef __KERNEL__
- spin_lock_init (&the_lnet.ln_lock);
- cfs_waitq_init (&the_lnet.ln_waitq);
- init_mutex(&the_lnet.ln_lnd_mutex);
- init_mutex(&the_lnet.ln_api_mutex);
-
- if (!strcmp(portals_compatibility, "none")) {
- the_lnet.ln_ptlcompat = 0;
- } else if (!strcmp(portals_compatibility, "weak")) {
- the_lnet.ln_ptlcompat = 1;
- LCONSOLE_WARN("Starting in weak portals-compatible mode\n");
- } else if (!strcmp(portals_compatibility, "strong")) {
- the_lnet.ln_ptlcompat = 2;
- LCONSOLE_WARN("Starting in strong portals-compatible mode\n");
- } else {
- LCONSOLE_ERROR("portals_compatibility=\"%s\" not supported\n",
- portals_compatibility);
- return -1;
- }
-#else
- pthread_mutex_init(&the_lnet.ln_mutex, NULL);
- pthread_cond_init(&the_lnet.ln_cond, NULL);
- pthread_mutex_init(&the_lnet.ln_lnd_mutex, NULL);
- pthread_mutex_init(&the_lnet.ln_api_mutex, NULL);
-#endif
-
+ the_lnet.ln_ptlcompat = rc;
+ the_lnet.ln_refcount = 0;
the_lnet.ln_init = 1;
#ifdef __KERNEL__
#else
/* Register all LNDs that have been loaded
* NB the order here determines default 'networks=' order */
+# if LNET_SINGLE_THREADED
LNET_REGISTER_LND_IF_PRESENT(the_ptllnd);
+# else
LNET_REGISTER_LND_IF_PRESENT(the_tcplnd);
+# endif
#endif
lnet_register_lnd(&the_lolnd);
-
return 0;
}
LASSERT (the_lnet.ln_init);
LASSERT (the_lnet.ln_refcount == 0);
-#ifdef __KERNEL__
- /* LNDs unregister themselves when their module unloads */
- lnet_unregister_lnd(&the_lolnd);
- LASSERT (list_empty(&the_lnet.ln_lnds));
-#else
while (!list_empty(&the_lnet.ln_lnds))
lnet_unregister_lnd(list_entry(the_lnet.ln_lnds.next,
lnd_t, lnd_list));
-
- pthread_mutex_destroy(&the_lnet.ln_api_mutex);
- pthread_mutex_destroy(&the_lnet.ln_lnd_mutex);
- pthread_cond_destroy(&the_lnet.ln_cond);
- pthread_mutex_destroy(&the_lnet.ln_mutex);
-#endif
+ lnet_fini_locks();
the_lnet.ln_init = 0;
}
int
LNetNIInit(lnet_pid_t requested_pid)
{
+ int im_a_router = 0;
int rc;
LNET_MUTEX_DOWN(&the_lnet.ln_api_mutex);
goto out;
}
- rc = lnet_init(requested_pid);
+ rc = lnet_prepare(requested_pid);
if (rc != 0)
- goto out;
+ goto failed0;
rc = lnet_startup_lndnis();
- if (rc != 0) {
- lnet_fini();
- goto out;
- }
+ if (rc != 0)
+ goto failed1;
- rc = lnet_parse_routes(lnet_get_routes());
- if (rc != 0) {
- lnet_shutdown_lndnis();
- lnet_fini();
- goto out;
- }
+ rc = lnet_parse_routes(lnet_get_routes(), &im_a_router);
+ if (rc != 0)
+ goto failed2;
+
+ rc = lnet_alloc_rtrpools(im_a_router);
+ if (rc != 0)
+ goto failed3;
rc = lnet_acceptor_start();
- if (rc != 0) {
- lnet_destroy_routes();
- lnet_shutdown_lndnis();
- lnet_fini();
- goto out;
- }
+ if (rc != 0)
+ goto failed3;
lnet_proc_init();
-
the_lnet.ln_refcount = 1;
+ goto out;
+ failed3:
+ lnet_destroy_routes();
+ failed2:
+ lnet_shutdown_lndnis();
+ failed1:
+ lnet_unprepare();
+ failed0:
+ LASSERT (rc < 0);
out:
LNET_MUTEX_UP(&the_lnet.ln_api_mutex);
return rc;
lnet_acceptor_stop();
lnet_destroy_routes();
lnet_shutdown_lndnis();
- lnet_fini();
+ lnet_unprepare();
}
LNET_MUTEX_UP(&the_lnet.ln_api_mutex);
my_sources = api-errno.c api-ni.c config.c \
lib-me.c lib-msg.c lib-eq.c \
lib-md.c lib-move.c lo.c \
- router.c acceptor.c peer.c
+ router.c router_proc.c \
+ acceptor.c peer.c
if LIBLUSTRE
int
-lnet_parse_route (char *str)
+lnet_parse_route (char *str, int *im_a_router)
{
/* static scratch buffer OK (single threaded) */
static char cmd[LNET_SINGLE_TEXTBUF_NOB];
nid = libcfs_str2nid(ptb->ptb_text);
LASSERT (nid != LNET_NID_ANY);
+ if (lnet_islocalnid(nid)) {
+ *im_a_router = 1;
+ continue;
+ }
+
rc = lnet_add_route (net, hops, nid);
if (rc != 0) {
CERROR("Can't create route "
}
int
-lnet_parse_route_tbs(struct list_head *tbs)
+lnet_parse_route_tbs(struct list_head *tbs, int *im_a_router)
{
lnet_text_buf_t *ptb;
while (!list_empty(tbs)) {
ptb = list_entry(tbs->next, lnet_text_buf_t, ptb_list);
- if (lnet_parse_route(ptb->ptb_text) < 0) {
+ if (lnet_parse_route(ptb->ptb_text, im_a_router) < 0) {
lnet_free_text_bufs(tbs);
return -EINVAL;
}
}
int
-lnet_parse_routes (char *routes)
+lnet_parse_routes (char *routes, int *im_a_router)
{
struct list_head tbs;
int rc = 0;
+ *im_a_router = 0;
+
if (the_lnet.ln_ptlcompat > 0 &&
routes[0] != 0) {
/* Can't route when running in compatibility mode */
CERROR("Error parsing routes\n");
rc = -EINVAL;
} else {
- rc = lnet_parse_route_tbs(&tbs);
+ rc = lnet_parse_route_tbs(&tbs, im_a_router);
}
LASSERT (lnet_tbnob == 0);
struct timeval then;
struct timeval now;
struct timespec ts;
+ lnet_ni_t *ni;
#endif
ENTRY;
RETURN (0);
}
- /* Some architectures force us to do spin locking/unlocking
- * in the same stack frame, means we can abstract the
- * locking here */
#ifdef __KERNEL__
cfs_waitlink_init(&wl);
set_current_state(TASK_INTERRUPTIBLE);
LNET_LOCK();
cfs_waitq_del(&the_lnet.ln_waitq, &wl);
#else
+ ni = the_lnet.ln_eqwaitni;
+ if (ni != NULL) {
+ lnet_ni_addref_locked(ni);
+ LNET_UNLOCK();
+ (ni->ni_lnd->lnd_wait)(ni, timeout_ms);
+ LNET_LOCK();
+ lnet_ni_decref_locked(ni);
+ continue;
+ }
+
+# if LNET_SINGLE_THREADED
+ LNET_UNLOCK();
+ return -ENOENT;
+# else
if (timeout_ms < 0) {
pthread_cond_wait(&the_lnet.ln_cond,
- &the_lnet.ln_mutex);
+ &the_lnet.ln_lock);
} else {
gettimeofday(&then, NULL);
}
pthread_cond_timedwait(&the_lnet.ln_cond,
- &the_lnet.ln_mutex, &ts);
+ &the_lnet.ln_lock, &ts);
gettimeofday(&now, NULL);
timeout_ms -= (now.tv_sec - then.tv_sec) * 1000 +
if (timeout_ms < 0)
timeout_ms = 0;
}
+# endif
#endif
}
}
/* send via lo0? */
if (implicit_loopback &&
+ the_lnet.ln_loni != NULL &&
lnet_ptlcompat_matchnid(src_ni->ni_nid, dst_nid)) {
lnet_ni_decref_locked(src_ni);
- src_ni = lnet_loni;
+ src_ni = the_lnet.ln_loni;
lnet_ni_addref_locked(src_ni);
}
- if (src_ni == lnet_loni) {
+ if (src_ni == the_lnet.ln_loni) {
/* No send credit hassles with LOLND */
LNET_UNLOCK();
CERROR("No route to %s\n", libcfs_id2str(msg->msg_target));
return -EHOSTUNREACH;
}
- src_ni = rnet->lrn_ni;
-
- if (!msg->msg_routing) {
- /* I'm the source and now I know which NI to send on */
- src_nid = lnet_ptlcompat_srcnid(src_ni->ni_nid, dst_nid);
- msg->msg_hdr.src_nid = cpu_to_le64(src_nid);
- msg->msg_hdr.src_pid = cpu_to_le32(the_lnet.ln_pid);
- }
/* Find the best gateway I can use */
lp = NULL;
route = list_entry(tmp, lnet_route_t, lr_list);
lp2 = route->lr_gateway;
- LASSERT (lp2->lp_ni == src_ni);
-
- if (!lp2->lp_alive)
- continue;
-
- if (lp == NULL ||
- lnet_compare_routers(lp2, lp))
+ if (lp2->lp_alive &&
+ (lp == NULL || lnet_compare_routers(lp2, lp)))
lp = lp2;
}
}
lnet_peer_addref_locked(lp);
+ src_ni = rnet->lrn_ni;
+
+ if (!msg->msg_routing) {
+ /* I'm the source and now I know which NI to send on */
+ src_nid = lnet_ptlcompat_srcnid(src_ni->ni_nid, dst_nid);
+ msg->msg_hdr.src_nid = cpu_to_le64(src_nid);
+ msg->msg_hdr.src_pid = cpu_to_le32(the_lnet.ln_pid);
+ }
msg->msg_target_is_router = 1;
msg->msg_target.nid = lp->lp_nid;
if (eq->eq_callback != NULL)
eq->eq_callback (eq_slot);
- /* Wake anyone sleeping for an event (see lib-eq.c) */
#ifdef __KERNEL__
+ /* Wake anyone waiting in LNetEQPoll() */
if (cfs_waitq_active(&the_lnet.ln_waitq))
cfs_waitq_broadcast(&the_lnet.ln_waitq);
#else
+# if LNET_SINGLE_THREADED
+ /* LNetEQPoll() calls into _the_ LND to wait for action */
+# else
+ /* Wake anyone waiting in LNetEQPoll() */
pthread_cond_broadcast(&the_lnet.ln_cond);
+# endif
#endif
}
.lnd_recv = lolnd_recv,
};
-lnet_ni_t *lnet_loni;
#ifdef __KERNEL__
-static int forwarding = 0;
-CFS_MODULE_PARM(forwarding, "i", int, 0444,
- "Boolean: set non-zero to forward between networks");
+static char *forwarding = "";
+CFS_MODULE_PARM(forwarding, "s", charp, 0444,
+ "Explicitly enable/disable forwarding between networks");
static int tiny_router_buffers = 512;
CFS_MODULE_PARM(tiny_router_buffers, "i", int, 0444,
int
lnet_add_route (__u32 net, unsigned int hops, lnet_nid_t gateway)
{
+ struct list_head zombies;
struct list_head *e;
lnet_remotenet_t *rnet;
lnet_remotenet_t *rnet2;
lnet_route_t *route;
lnet_route_t *route2;
lnet_peer_t *lp;
- int dup;
- int hops2;
- __u32 net2;
int rc;
CDEBUG(D_NET, "Add route: net %s hops %u gw %s\n",
libcfs_net2str(net), hops, libcfs_nid2str(gateway));
if (gateway == LNET_NID_ANY ||
+ net == PTL_NIDNET(LNET_NID_ANY) ||
+ PTL_NIDNET(gateway) == net ||
hops < 1 || hops > 255)
return (-EINVAL);
return 0;
}
- hops2 = rnet2->lrn_hops;
- net2 = PTL_NIDNET(rnet2->lrn_ni->ni_nid);
-
- if (rnet2->lrn_ni == lp->lp_ni && hops2 == hops) {
- /* New route consistent with existing routes; search for
- * duplicate route (NOOP if this is) */
- dup = 0;
- list_for_each (e, &rnet2->lrn_routes) {
- route2 = list_entry(e, lnet_route_t, lr_list);
-
- if (route2->lr_gateway->lp_nid == gateway) {
- dup = 1;
- break;
- }
- }
-
- if (!dup) {
- /* New route */
- list_add_tail(&route->lr_list, &rnet2->lrn_routes);
- the_lnet.ln_remote_nets_version++;
- } else {
- lnet_peer_decref_locked(lp);
- }
-
+ if (rnet2->lrn_hops < hops) {
+ /* new route is longer than ones we have already:
+ * Ignore it silently */
+ LNET_UNLOCK();
+ PORTAL_FREE(route, sizeof(*route));
+ goto out;
+ }
+
+ if (rnet2->lrn_hops > hops) {
+ /* new route supercedes all currently known routes to this
+ * net */
+ list_add(&zombies, &rnet2->lrn_routes);
+ list_del_init(&rnet2->lrn_routes);
+ list_add(&route->lr_list, &rnet2->lrn_routes);
+ the_lnet.ln_remote_nets_version++;
LNET_UNLOCK();
- PORTAL_FREE(rnet, sizeof(*rnet));
- if (dup)
+ while (!list_empty(&zombies)) {
+ route = list_entry(zombies.next, lnet_route_t, lr_list);
+ list_del(&route->lr_list);
+
+ LNET_LOCK();
+ lnet_peer_decref_locked(route->lr_gateway);
+ LNET_UNLOCK();
PORTAL_FREE(route, sizeof(*route));
-
- return 0;
+ }
+ goto out;
}
+
+ /* New route has the same hopcount as existing routes; search for
+ * a duplicate route (it's a NOOP if it is) */
+ list_for_each (e, &rnet2->lrn_routes) {
+ route2 = list_entry(e, lnet_route_t, lr_list);
- lnet_peer_decref_locked(lp);
+ if (route2->lr_gateway->lp_nid == gateway) {
+ LNET_UNLOCK();
+ PORTAL_FREE(route, sizeof(*route));
+ goto out;
+ }
+ }
+
+ /* Add the new route */
+ list_add_tail(&route->lr_list, &rnet2->lrn_routes);
+ the_lnet.ln_remote_nets_version++;
LNET_UNLOCK();
+
+ out:
PORTAL_FREE(rnet, sizeof(*rnet));
- PORTAL_FREE(route, sizeof(*route));
-
- if (hops != hops2)
- CERROR("Hopcount not consistent on route: %s %d(%d) %s\n",
- libcfs_net2str(net), hops, hops2,
- libcfs_nid2str(gateway));
- else
- CERROR("Router network not consistent on route: %s %d %s(%s)\n",
- libcfs_net2str(net), hops,
- libcfs_nid2str(gateway), libcfs_net2str(net2));
- return -EINVAL;
+ return 0;
}
int
lnet_rtrbuf_t *rb;
int i;
+ if (rbp->rbp_nbuffers != 0) {
+ LASSERT (rbp->rbp_nbuffers == nbufs);
+ return 0;
+ }
+
for (i = 0; i < nbufs; i++) {
rb = lnet_new_rtrbuf(rbp);
rbp->rbp_credits++;
list_add(&rb->rb_list, &rbp->rbp_bufs);
- /* NB if this is live there need to be code to schedule blocked
- * msgs */
+ /* No allocation "under fire" */
+ /* Otherwise we'd need code to schedule blocked msgs etc */
+ LASSERT (!the_lnet.ln_routing);
}
LASSERT (rbp->rbp_credits == nbufs);
lnet_rtrpool_free_bufs(&the_lnet.ln_rtrpools[2]);
}
-int
-lnet_alloc_rtrpools(void)
+void
+lnet_init_rtrpools(void)
{
int small_pages = 1;
int large_pages = (PTL_MTU + PAGE_SIZE - 1) / PAGE_SIZE;
lnet_rtrpool_init(&the_lnet.ln_rtrpools[0], 0);
lnet_rtrpool_init(&the_lnet.ln_rtrpools[1], small_pages);
lnet_rtrpool_init(&the_lnet.ln_rtrpools[2], large_pages);
+}
- for (rc = 0; rc < LNET_NRBPOOLS; rc++)
- CDEBUG(D_NET, "Pages[%d]: %d\n", rc,
- the_lnet.ln_rtrpools[rc].rbp_npages);
- the_lnet.ln_routing = forwarding;
- if (!forwarding)
+int
+lnet_alloc_rtrpools(int im_a_router)
+{
+ int rc;
+
+ if (!strcmp(forwarding, "")) {
+ /* not set either way */
+ if (!im_a_router)
+ return 0;
+ } else if (!strcmp(forwarding, "disabled")) {
+ /* explicitly disabled */
return 0;
-
+ } else if (!strcmp(forwarding, "enabled")) {
+ /* explicitly enabled */
+ } else {
+ LCONSOLE_ERROR("'forwarding' not set to either "
+ "'enabled' or 'disabled'\n");
+ return -EINVAL;
+ }
+
if (tiny_router_buffers <= 0) {
LCONSOLE_ERROR("tiny_router_buffers=%d invalid when "
"routing enabled\n", tiny_router_buffers);
if (rc != 0)
goto failed;
+ LNET_LOCK();
+ the_lnet.ln_routing = 1;
+ LNET_UNLOCK();
+
return 0;
failed:
{
}
+void
+lnet_init_rtrpools (void)
+{
+}
+
int
-lnet_alloc_rtrpools (void)
+lnet_alloc_rtrpools (int im_a_arouter)
{
- /* No userspace routing */
- the_lnet.ln_routing = 0;
- return 0;
+ return -EOPNOTSUPP;
}
#endif
*/
#include <lnet/lib-lnet.h>
+
+#ifdef __KERNEL__
+
#include <linux/seq_file.h>
#include <linux/lustre_compat25.h>
remove_proc_entry(LNET_PROC_BUFFERS, 0);
remove_proc_entry(LNET_PROC_NIS, 0);
}
+
+#else
+
+void
+lnet_proc_init(void)
+{
+}
+
+void
+lnet_proc_fini(void)
+{
+}
+
+#endif
#include <portals/p30.h>
+typedef struct
+{
+ ptl_handle_ni_t pls_ni;
+ ptl_handle_eq_t pls_eq;
+
+} ptllnd_state;
+
extern int ptllnd_startup(struct lnet_ni *ni);
extern void ptllnd_shutdown(struct lnet_ni *ni);
extern int ptllnd_send(struct lnet_ni *ni, void *private, lnet_msg_t *msg);
# include <sys/utsname.h>
#endif
+#if LNET_SINGLE_THREADED
+# error "This LND requires a multi-threaded runtime"
+#endif
+
/* XXX CFS workaround, to give a chance to let nal thread wake up
* from waiting in select
*/
+++ /dev/null
-/* -*- mode: c; c-basic-offset: 8; indent-tabs-mode: nil; -*-
- * vim:expandtab:shiftwidth=8:tabstop=8:
- *
- * Copyright (c) 2002 Cray Inc.
- *
- * This file is part of Portals, http://www.sf.net/projects/sandiaportals/
- */
-
-typedef unsigned short uint16;
-typedef unsigned long uint32;
-typedef unsigned long long uint64;
-typedef unsigned char uint8;
git://git.whamcloud.com / fs / lustre-release.git / commitdiff