#define LNET_NRB_LARGE_PAGES ((LNET_MTU + PAGE_SIZE - 1) >> \
PAGE_SHIFT)
+extern unsigned int lnet_current_net_count;
+
static char *forwarding = "";
module_param(forwarding, charp, 0444);
MODULE_PARM_DESC(forwarding, "Explicitly enable/disable forwarding between networks");
module_param(avoid_asym_router_failure, int, 0644);
MODULE_PARM_DESC(avoid_asym_router_failure, "Avoid asymmetrical router failures (0 to disable)");
+int dead_router_check_interval = INT_MIN;
+module_param(dead_router_check_interval, int, 0444);
+MODULE_PARM_DESC(dead_router_check_interval, "(DEPRECATED - Use alive_router_check_interval)");
+
+int live_router_check_interval = INT_MIN;
+module_param(live_router_check_interval, int, 0444);
+MODULE_PARM_DESC(live_router_check_interval, "(DEPRECATED - Use alive_router_check_interval)");
+
int alive_router_check_interval = 60;
module_param(alive_router_check_interval, int, 0644);
MODULE_PARM_DESC(alive_router_check_interval, "Seconds between live router health checks (<= 0 to disable)");
static void
lnet_add_route_to_rnet(struct lnet_remotenet *rnet, struct lnet_route *route)
{
- unsigned int len = 0;
+ struct lnet_peer_net *lpn;
unsigned int offset = 0;
+ unsigned int len = 0;
struct list_head *e;
lnet_shuffle_seed();
* different nodes are using the same list of routers, they end up
* preferring different routers.
*/
- offset = cfs_rand() % (len + 1);
+ offset = prandom_u32_max(len + 1);
list_for_each(e, &rnet->lrn_routes) {
if (offset == 0)
break;
* force a router check on the gateway to make sure the route is
* alive
*/
- route->lr_gateway->lp_rtrcheck_timestamp = 0;
+ list_for_each_entry(lpn, &route->lr_gateway->lp_peer_nets,
+ lpn_peer_nets) {
+ lpn->lpn_rtrcheck_timestamp = 0;
+ }
the_lnet.ln_remote_nets_version++;
int
lnet_add_route(__u32 net, __u32 hops, lnet_nid_t gateway,
- unsigned int priority)
+ __u32 priority, __u32 sensitivity)
{
struct list_head *route_entry;
struct lnet_remotenet *rnet;
if (lnet_islocalnet(net))
return -EEXIST;
+ if (!lnet_islocalnet(LNET_NIDNET(gateway))) {
+ CERROR("Cannot add route with gateway %s. There is no local interface configured on LNet %s\n",
+ libcfs_nid2str(gateway),
+ libcfs_net2str(LNET_NIDNET(gateway)));
+ return -EINVAL;
+ }
+
/* Assume net, route, all new */
LIBCFS_ALLOC(route, sizeof(*route));
LIBCFS_ALLOC(rnet, sizeof(*rnet));
* to move the routes from the peer that's being deleted to the
* consolidated peer lp_routes list
*/
- if (add_route)
+ if (add_route) {
+ gw->lp_health_sensitivity = sensitivity;
lnet_add_route_to_rnet(rnet2, route);
+ if (lnet_peer_discovery_disabled)
+ CWARN("Consider turning discovery on to enable full "
+ "Multi-Rail routing functionality\n");
+ }
/*
* get rid of the reference on the lpni.
}
delete_zombies:
+ /*
+ * check if there are any routes remaining on the gateway
+ * If there are no more routes make sure to set the peer's
+ * lp_disc_net_id to 0 (invalid), in case we add more routes in
+ * the future on that gateway, then we start our discovery process
+ * from scratch
+ */
+ if (lpni) {
+ if (list_empty(&lp->lp_routes))
+ lp->lp_disc_net_id = 0;
+ }
+
lnet_net_unlock(LNET_LOCK_EX);
while (!list_empty(&zombies)) {
int
lnet_get_route(int idx, __u32 *net, __u32 *hops,
- lnet_nid_t *gateway, __u32 *alive, __u32 *priority)
+ lnet_nid_t *gateway, __u32 *alive, __u32 *priority, __u32 *sensitivity)
{
- struct list_head *e1;
- struct list_head *e2;
struct lnet_remotenet *rnet;
- struct lnet_route *route;
- int cpt;
- int i;
struct list_head *rn_list;
+ struct lnet_route *route;
+ struct list_head *e1;
+ struct list_head *e2;
+ int cpt;
+ int i;
cpt = lnet_net_lock_current();
*gateway = route->lr_nid;
*hops = route->lr_hops;
*priority = route->lr_priority;
+ *sensitivity = route->lr_gateway->
+ lp_health_sensitivity;
*alive = lnet_is_route_alive(route);
lnet_net_unlock(cpt);
return 0;
}
}
-static void
+static inline bool
+lnet_net_set_status_locked(struct lnet_net *net, __u32 status)
+{
+ struct lnet_ni *ni;
+ bool update = false;
+
+ list_for_each_entry(ni, &net->net_ni_list, ni_netlist) {
+ lnet_ni_lock(ni);
+ if (ni->ni_status &&
+ ni->ni_status->ns_status != status) {
+ ni->ni_status->ns_status = status;
+ update = true;
+ }
+ lnet_ni_unlock(ni);
+ }
+
+ return update;
+}
+
+static bool
lnet_update_ni_status_locked(void)
{
- struct lnet_ni *ni = NULL;
+ struct lnet_net *net;
+ bool push = false;
time64_t now;
time64_t timeout;
timeout = router_ping_timeout + alive_router_check_interval;
now = ktime_get_real_seconds();
- while ((ni = lnet_get_next_ni_locked(NULL, ni))) {
- if (ni->ni_net->net_lnd->lnd_type == LOLND)
+ list_for_each_entry(net, &the_lnet.ln_nets, net_list) {
+ if (net->net_lnd->lnd_type == LOLND)
continue;
- if (now < ni->ni_last_alive + timeout)
+ if (now < net->net_last_alive + timeout)
continue;
- lnet_ni_lock(ni);
+ spin_lock(&net->net_lock);
/* re-check with lock */
- if (now < ni->ni_last_alive + timeout) {
- lnet_ni_unlock(ni);
+ if (now < net->net_last_alive + timeout) {
+ spin_unlock(&net->net_lock);
continue;
}
+ spin_unlock(&net->net_lock);
- LASSERT(ni->ni_status != NULL);
-
- if (ni->ni_status->ns_status != LNET_NI_STATUS_DOWN) {
- CDEBUG(D_NET, "NI(%s:%lld) status changed to down\n",
- libcfs_nid2str(ni->ni_nid), timeout);
- /* NB: so far, this is the only place to set
- * NI status to "down" */
- ni->ni_status->ns_status = LNET_NI_STATUS_DOWN;
- }
- lnet_ni_unlock(ni);
+ /*
+ * if the net didn't receive any traffic for past the
+ * timeout on any of its constituent NIs, then mark all
+ * the NIs down.
+ */
+ push = lnet_net_set_status_locked(net, LNET_NI_STATUS_DOWN);
}
+
+ return push;
}
void lnet_wait_router_start(void)
void
lnet_check_routers(void)
{
+ struct lnet_peer_net *first_lpn = NULL;
+ struct lnet_peer_net *lpn;
struct lnet_peer_ni *lpni;
struct list_head *entry;
struct lnet_peer *rtr;
+ bool push = false;
+ bool found_lpn;
__u64 version;
+ __u32 net_id;
time64_t now;
int cpt;
int rc;
* interfaces could be down and in that case they would be
* undergoing recovery separately from this discovery.
*/
- if (now - rtr->lp_rtrcheck_timestamp <
- alive_router_check_interval)
+ /* find next peer net which is also local */
+ net_id = rtr->lp_disc_net_id;
+ do {
+ lpn = lnet_get_next_peer_net_locked(rtr, net_id);
+ if (!lpn) {
+ CERROR("gateway %s has no networks\n",
+ libcfs_nid2str(rtr->lp_primary_nid));
+ break;
+ }
+ if (first_lpn == lpn)
+ break;
+ if (!first_lpn)
+ first_lpn = lpn;
+ found_lpn = lnet_islocalnet_locked(lpn->lpn_net_id);
+ net_id = lpn->lpn_net_id;
+ } while (!found_lpn);
+
+ if (!found_lpn || !lpn) {
+ CERROR("no local network found for gateway %s\n",
+ libcfs_nid2str(rtr->lp_primary_nid));
+ continue;
+ }
+
+ if (now - lpn->lpn_rtrcheck_timestamp <
+ alive_router_check_interval / lnet_current_net_count)
continue;
/*
}
lnet_peer_ni_addref_locked(lpni);
+ /* specify the net to use */
+ rtr->lp_disc_net_id = lpn->lpn_net_id;
+
/* discover the router */
CDEBUG(D_NET, "discover %s, cpt = %d\n",
libcfs_nid2str(lpni->lpni_nid), cpt);
lnet_peer_ni_decref_locked(lpni);
if (!rc)
- rtr->lp_rtrcheck_timestamp = now;
+ lpn->lpn_rtrcheck_timestamp = now;
else
CERROR("Failed to discover router %s\n",
libcfs_nid2str(rtr->lp_primary_nid));
}
if (the_lnet.ln_routing)
- lnet_update_ni_status_locked();
+ push = lnet_update_ni_status_locked();
lnet_net_unlock(cpt);
+
+ /* if the status of the ni changed update the peers */
+ if (push)
+ lnet_push_update_to_peers(1);
}
void
~LNET_PING_FEAT_RTE_DISABLED;
lnet_net_unlock(LNET_LOCK_EX);
+ if (lnet_peer_discovery_disabled)
+ CWARN("Consider turning discovery on to enable full "
+ "Multi-Rail routing functionality\n");
+
return rc;
}
git://git.whamcloud.com / fs / lustre-release.git / blobdiff