#define LNET_NRB_LARGE_PAGES ((LNET_MTU + PAGE_SIZE - 1) >> \
PAGE_SHIFT)
-extern unsigned int lnet_current_net_count;
-
static char *forwarding = "";
module_param(forwarding, charp, 0444);
MODULE_PARM_DESC(forwarding, "Explicitly enable/disable forwarding between networks");
module_param(avoid_asym_router_failure, int, 0644);
MODULE_PARM_DESC(avoid_asym_router_failure, "Avoid asymmetrical router failures (0 to disable)");
+int dead_router_check_interval = INT_MIN;
+module_param(dead_router_check_interval, int, 0444);
+MODULE_PARM_DESC(dead_router_check_interval, "(DEPRECATED - Use alive_router_check_interval)");
+
+int live_router_check_interval = INT_MIN;
+module_param(live_router_check_interval, int, 0444);
+MODULE_PARM_DESC(live_router_check_interval, "(DEPRECATED - Use alive_router_check_interval)");
+
int alive_router_check_interval = 60;
module_param(alive_router_check_interval, int, 0644);
MODULE_PARM_DESC(alive_router_check_interval, "Seconds between live router health checks (<= 0 to disable)");
bool route_alive;
/*
+ * if discovery is disabled then rely on the cached aliveness
+ * information. This is handicapped information which we log when
+ * we receive the discovery ping response. The most uptodate
+ * aliveness information can only be obtained when discovery is
+ * enabled.
+ */
+ if (lnet_is_discovery_disabled(gw))
+ return route->lr_alive;
+
+ /*
* check the gateway's interfaces on the route rnet to make sure
* that the gateway is viable.
*/
}
+static inline void
+lnet_set_route_aliveness(struct lnet_route *route, bool alive)
+{
+ /* Log when there's a state change */
+ if (route->lr_alive != alive) {
+ CERROR("route to %s through %s has gone from %s to %s\n",
+ libcfs_net2str(route->lr_net),
+ libcfs_nid2str(route->lr_gateway->lp_primary_nid),
+ (route->lr_alive) ? "up" : "down",
+ alive ? "up" : "down");
+ route->lr_alive = alive;
+ }
+}
+
+void
+lnet_router_discovery_ping_reply(struct lnet_peer *lp)
+{
+ struct lnet_ping_buffer *pbuf = lp->lp_data;
+ struct lnet_remotenet *rnet;
+ struct lnet_peer_net *llpn;
+ struct lnet_route *route;
+ bool net_up = false;
+ unsigned lp_state;
+ __u32 net, net2;
+ int i, j;
+
+
+ spin_lock(&lp->lp_lock);
+ lp_state = lp->lp_state;
+
+ /* only handle replies if discovery is disabled. */
+ if (!lnet_is_discovery_disabled_locked(lp)) {
+ spin_unlock(&lp->lp_lock);
+ return;
+ }
+
+ spin_unlock(&lp->lp_lock);
+
+ if (lp_state & LNET_PEER_PING_FAILED ||
+ pbuf->pb_info.pi_features & LNET_PING_FEAT_RTE_DISABLED) {
+ CDEBUG(D_NET, "Set routes down for gw %s because %s %d\n",
+ libcfs_nid2str(lp->lp_primary_nid),
+ lp_state & LNET_PEER_PING_FAILED ? "ping failed" :
+ "route feature is disabled", lp->lp_ping_error);
+ /* If the ping failed or the peer has routing disabled then
+ * mark the routes served by this peer down
+ */
+ list_for_each_entry(route, &lp->lp_routes, lr_gwlist)
+ lnet_set_route_aliveness(route, false);
+ return;
+ }
+
+ CDEBUG(D_NET, "Discovery is disabled. Processing reply for gw: %s\n",
+ libcfs_nid2str(lp->lp_primary_nid));
+
+ /*
+ * examine the ping response:
+ * For each NID in the ping response, extract the net
+ * if the net exists on our remote net list then
+ * iterate over the routes on the rnet and if:
+ * The route's local net is healthy and
+ * The remote net status is UP, then mark the route up
+ * otherwise mark the route down
+ */
+ for (i = 1; i < pbuf->pb_info.pi_nnis; i++) {
+ net = LNET_NIDNET(pbuf->pb_info.pi_ni[i].ns_nid);
+ rnet = lnet_find_rnet_locked(net);
+ if (!rnet)
+ continue;
+ list_for_each_entry(route, &rnet->lrn_routes, lr_list) {
+ /* check if this is the route's gateway */
+ if (lp->lp_primary_nid !=
+ route->lr_gateway->lp_primary_nid)
+ continue;
+
+ llpn = lnet_peer_get_net_locked(lp, route->lr_lnet);
+ if (!llpn) {
+ lnet_set_route_aliveness(route, false);
+ continue;
+ }
+
+ if (!lnet_is_gateway_net_alive(llpn)) {
+ lnet_set_route_aliveness(route, false);
+ continue;
+ }
+
+ if (avoid_asym_router_failure &&
+ pbuf->pb_info.pi_ni[i].ns_status !=
+ LNET_NI_STATUS_UP) {
+ net_up = false;
+
+ /*
+ * revisit all previous NIDs and check if
+ * any on the network we're examining is
+ * up. If at least one is up then we consider
+ * the route to be alive.
+ */
+ for (j = 1; j < i; j++) {
+ net2 = LNET_NIDNET(pbuf->pb_info.
+ pi_ni[j].ns_nid);
+ if (net2 == net &&
+ pbuf->pb_info.pi_ni[j].ns_status ==
+ LNET_NI_STATUS_UP)
+ net_up = true;
+ }
+ if (!net_up) {
+ lnet_set_route_aliveness(route, false);
+ continue;
+ }
+ }
+
+ lnet_set_route_aliveness(route, true);
+ }
+ }
+}
+
void
lnet_router_discovery_complete(struct lnet_peer *lp)
{
struct lnet_peer_ni *lpni = NULL;
+ struct lnet_route *route;
spin_lock(&lp->lp_lock);
lp->lp_state &= ~LNET_PEER_RTR_DISCOVERY;
+ lp->lp_state |= LNET_PEER_RTR_DISCOVERED;
spin_unlock(&lp->lp_lock);
/*
libcfs_nid2str(lp->lp_primary_nid), lp->lp_dc_error);
while ((lpni = lnet_get_next_peer_ni_locked(lp, NULL, lpni)) != NULL)
lpni->lpni_ns_status = LNET_NI_STATUS_DOWN;
+
+ list_for_each_entry(route, &lp->lp_routes, lr_gwlist)
+ lnet_set_route_aliveness(route, false);
}
static void
add_device_randomness(&ni->ni_nid, sizeof(ni->ni_nid));
seeded = 1;
- return;
}
/* NB expects LNET_LOCK held */
* different nodes are using the same list of routers, they end up
* preferring different routers.
*/
- offset = cfs_rand() % (len + 1);
+ offset = prandom_u32_max(len + 1);
list_for_each(e, &rnet->lrn_routes) {
if (offset == 0)
break;
CERROR("Cannot add route with gateway %s. There is no local interface configured on LNet %s\n",
libcfs_nid2str(gateway),
libcfs_net2str(LNET_NIDNET(gateway)));
- return -EINVAL;
+ return -EHOSTUNREACH;
}
/* Assume net, route, all new */
if (add_route) {
gw->lp_health_sensitivity = sensitivity;
lnet_add_route_to_rnet(rnet2, route);
+ if (lnet_peer_discovery_disabled)
+ CWARN("Consider turning discovery on to enable full "
+ "Multi-Rail routing functionality\n");
}
/*
LIBCFS_FREE(rnet, sizeof(*rnet));
/* kick start the monitor thread to handle the added route */
- wake_up(&the_lnet.ln_mt_waitq);
+ complete(&the_lnet.ln_mt_wait_complete);
return rc;
}
struct lnet_peer_ni *lpni;
struct lnet_route *route;
struct list_head zombies;
- struct lnet_peer *lp;
+ struct lnet_peer *lp = NULL;
int i = 0;
INIT_LIST_HEAD(&rnet_zombies);
spin_lock(&rtr->lp_lock);
- if ((rtr->lp_state & LNET_PEER_DISCOVERED) == 0) {
+ if ((rtr->lp_state & LNET_PEER_RTR_DISCOVERED) == 0) {
all_known = 0;
spin_unlock(&rtr->lp_lock);
break;
lnet_net_lock(LNET_LOCK_EX);
the_lnet.ln_routing = 1;
lnet_net_unlock(LNET_LOCK_EX);
- wake_up(&the_lnet.ln_mt_waitq);
+ complete(&the_lnet.ln_mt_wait_complete);
return 0;
failed:
~LNET_PING_FEAT_RTE_DISABLED;
lnet_net_unlock(LNET_LOCK_EX);
+ if (lnet_peer_discovery_disabled)
+ CWARN("Consider turning discovery on to enable full "
+ "Multi-Rail routing functionality\n");
+
return rc;
}
time64_t when)
{
struct lnet_peer_ni *lpni = NULL;
+ struct lnet_route *route;
+ struct lnet_peer *lp;
time64_t now = ktime_get_seconds();
int cpt;
cpt = lpni->lpni_cpt;
lnet_net_lock(cpt);
lnet_peer_ni_decref_locked(lpni);
+ if (lpni && lpni->lpni_peer_net && lpni->lpni_peer_net->lpn_peer) {
+ lp = lpni->lpni_peer_net->lpn_peer;
+ list_for_each_entry(route, &lp->lp_routes, lr_gwlist)
+ lnet_set_route_aliveness(route, alive);
+ }
lnet_net_unlock(cpt);
return 0;