Wakeup every gateway aliveness interval / number of local networks.
Discover each local gateway network in round robin.
This is done to make sure the gateway keeps its networks up.
Test-Parameters: forbuildonly
Signed-off-by: Amir Shehat <ashehata@whamcloud.com>
Change-Id: I4035e39c286cb599d4eb8f9df7ed5d278e6d744a
Reviewed-on: https://review.whamcloud.com/34511
Tested-by: Jenkins
Reviewed-by: Olaf Weber <olaf.weber@hpe.com>
int lnet_islocalnid(lnet_nid_t nid);
int lnet_islocalnet(__u32 net);
+int lnet_islocalnet_locked(__u32 net);
void lnet_msg_attach_md(struct lnet_msg *msg, struct lnet_libmd *md,
unsigned int offset, unsigned int mlen);
bool lnet_ni_unique_net(struct list_head *nilist, char *iface);
void lnet_incr_dlc_seq(void);
__u32 lnet_get_dlc_seq_locked(void);
+int lnet_get_net_count(void);
+struct lnet_peer_net *lnet_get_next_peer_net_locked(struct lnet_peer *lp,
+ __u32 prev_lpn_id);
struct lnet_peer_ni *lnet_get_next_peer_ni_locked(struct lnet_peer *peer,
struct lnet_peer_net *peer_net,
struct lnet_peer_ni *prev);
/* primary NID of the peer */
lnet_nid_t lp_primary_nid;
+ /* net to perform discovery on */
+ __u32 lp_disc_net_id;
+
/* CPT of peer_table */
int lp_cpt;
/* routes on this peer */
struct list_head lp_routes;
- /* time of last router check attempt */
- time64_t lp_rtrcheck_timestamp;
-
/* reference count */
atomic_t lp_refcount;
/* Net ID */
__u32 lpn_net_id;
+ /* time of last router net check attempt */
+ time64_t lpn_rtrcheck_timestamp;
+
+
/* reference count */
atomic_t lpn_refcount;
};
unsigned lnet_lnd_timeout = LNET_LND_DEFAULT_TIMEOUT;
+unsigned int lnet_current_net_count;
/*
* This sequence number keeps track of how many times DLC was used to
EXPORT_SYMBOL(lnet_cpt_of_nid);
int
-lnet_islocalnet(__u32 net_id)
+lnet_islocalnet_locked(__u32 net_id)
{
struct lnet_net *net;
- int cpt;
- bool local;
-
- cpt = lnet_net_lock_current();
+ bool local;
net = lnet_get_net_locked(net_id);
local = net != NULL;
+ return local;
+}
+
+int
+lnet_islocalnet(__u32 net_id)
+{
+ int cpt;
+ bool local;
+
+ cpt = lnet_net_lock_current();
+
+ local = lnet_islocalnet_locked(net_id);
+
lnet_net_unlock(cpt);
return local;
return count;
}
+int
+lnet_get_net_count(void)
+{
+ struct lnet_net *net;
+ int count = 0;
+
+ lnet_net_lock(0);
+
+ list_for_each_entry(net, &the_lnet.ln_nets, net_list) {
+ count++;
+ }
+
+ lnet_net_unlock(0);
+
+ return count;
+}
+
void
lnet_swap_pinginfo(struct lnet_ping_buffer *pbuf)
{
lnet_net_unlock(LNET_LOCK_EX);
}
+ /* update net count */
+ lnet_current_net_count = lnet_get_net_count();
+
return ni_count;
failed1:
#include <linux/nsproxy.h>
#include <net/net_namespace.h>
+extern unsigned int lnet_current_net_count;
+
static int local_nid_dist_zero = 1;
module_param(local_nid_dist_zero, int, 0444);
MODULE_PARM_DESC(local_nid_dist_zero, "Reserved");
}
struct lnet_ni *
-lnet_find_best_ni_on_local_net(struct lnet_peer *peer, int md_cpt)
+lnet_find_best_ni_on_local_net(struct lnet_peer *peer, int md_cpt,
+ bool discovery)
{
struct lnet_peer_net *peer_net = NULL;
struct lnet_ni *best_ni = NULL;
continue;
best_ni = lnet_find_best_ni_on_spec_net(best_ni, peer,
peer_net, md_cpt, false);
+
+ /*
+ * if this is a discovery message and lp_disc_net_id is
+ * specified then use that net to send the discovery on.
+ */
+ if (peer->lp_disc_net_id == peer_net->lpn_net_id &&
+ discovery)
+ break;
}
if (best_ni)
* networks.
*/
sd->sd_best_ni = lnet_find_best_ni_on_local_net(sd->sd_peer,
- sd->sd_md_cpt);
+ sd->sd_md_cpt,
+ lnet_msg_discovery(sd->sd_msg));
if (sd->sd_best_ni) {
sd->sd_best_lpni =
lnet_find_best_lpni_on_net(sd, sd->sd_peer,
* if we wake up every 1 second? Although, we've seen
* cases where we get a complaint that an idle thread
* is waking up unnecessarily.
+ *
+ * Take into account the current net_count when you wake
+ * up for alive router checking, since we need to check
+ * possibly as many networks as we have configured.
*/
interval = min(lnet_recovery_interval,
- min((unsigned int) alive_router_check_interval,
+ min((unsigned int) alive_router_check_interval /
+ lnet_current_net_count,
lnet_transaction_timeout / 2));
wait_event_interruptible_timeout(the_lnet.ln_mt_waitq,
false,
return lp;
}
+struct lnet_peer_net *
+lnet_get_next_peer_net_locked(struct lnet_peer *lp, __u32 prev_lpn_id)
+{
+ struct lnet_peer_net *net;
+
+ if (!prev_lpn_id) {
+ /* no net id provided return the first net */
+ net = list_first_entry_or_null(&lp->lp_peer_nets,
+ struct lnet_peer_net,
+ lpn_peer_nets);
+
+ return net;
+ }
+
+ /* find the net after the one provided */
+ list_for_each_entry(net, &lp->lp_peer_nets, lpn_peer_nets) {
+ if (net->lpn_net_id == prev_lpn_id) {
+ /*
+ * if we reached the end of the list loop to the
+ * beginning.
+ */
+ if (net->lpn_peer_nets.next == &lp->lp_peer_nets)
+ return list_first_entry_or_null(&lp->lp_peer_nets,
+ struct lnet_peer_net,
+ lpn_peer_nets);
+ else
+ return list_next_entry(net, lpn_peer_nets);
+ }
+ }
+
+ return NULL;
+}
+
struct lnet_peer_ni *
lnet_get_next_peer_ni_locked(struct lnet_peer *peer,
struct lnet_peer_net *peer_net,
#define LNET_NRB_LARGE_PAGES ((LNET_MTU + PAGE_SIZE - 1) >> \
PAGE_SHIFT)
+extern unsigned int lnet_current_net_count;
+
static char *forwarding = "";
module_param(forwarding, charp, 0444);
MODULE_PARM_DESC(forwarding, "Explicitly enable/disable forwarding between networks");
static void
lnet_add_route_to_rnet(struct lnet_remotenet *rnet, struct lnet_route *route)
{
- unsigned int len = 0;
+ struct lnet_peer_net *lpn;
unsigned int offset = 0;
+ unsigned int len = 0;
struct list_head *e;
lnet_shuffle_seed();
* force a router check on the gateway to make sure the route is
* alive
*/
- route->lr_gateway->lp_rtrcheck_timestamp = 0;
+ list_for_each_entry(lpn, &route->lr_gateway->lp_peer_nets,
+ lpn_peer_nets) {
+ lpn->lpn_rtrcheck_timestamp = 0;
+ }
the_lnet.ln_remote_nets_version++;
}
delete_zombies:
+ /*
+ * check if there are any routes remaining on the gateway
+ * If there are no more routes make sure to set the peer's
+ * lp_disc_net_id to 0 (invalid), in case we add more routes in
+ * the future on that gateway, then we start our discovery process
+ * from scratch
+ */
+ if (lpni) {
+ if (list_empty(&lp->lp_routes))
+ lp->lp_disc_net_id = 0;
+ }
+
lnet_net_unlock(LNET_LOCK_EX);
while (!list_empty(&zombies)) {
void
lnet_check_routers(void)
{
+ struct lnet_peer_net *first_lpn = NULL;
+ struct lnet_peer_net *lpn;
struct lnet_peer_ni *lpni;
struct list_head *entry;
struct lnet_peer *rtr;
bool push = false;
+ bool found_lpn;
__u64 version;
+ __u32 net_id;
time64_t now;
int cpt;
int rc;
* interfaces could be down and in that case they would be
* undergoing recovery separately from this discovery.
*/
- if (now - rtr->lp_rtrcheck_timestamp <
- alive_router_check_interval)
+ /* find next peer net which is also local */
+ net_id = rtr->lp_disc_net_id;
+ do {
+ lpn = lnet_get_next_peer_net_locked(rtr, net_id);
+ if (!lpn) {
+ CERROR("gateway %s has no networks\n",
+ libcfs_nid2str(rtr->lp_primary_nid));
+ break;
+ }
+ if (first_lpn == lpn)
+ break;
+ if (!first_lpn)
+ first_lpn = lpn;
+ found_lpn = lnet_islocalnet_locked(lpn->lpn_net_id);
+ net_id = lpn->lpn_net_id;
+ } while (!found_lpn);
+
+ if (!found_lpn || !lpn) {
+ CERROR("no local network found for gateway %s\n",
+ libcfs_nid2str(rtr->lp_primary_nid));
+ continue;
+ }
+
+ if (now - lpn->lpn_rtrcheck_timestamp <
+ alive_router_check_interval / lnet_current_net_count)
continue;
/*
}
lnet_peer_ni_addref_locked(lpni);
+ /* specify the net to use */
+ rtr->lp_disc_net_id = lpn->lpn_net_id;
+
/* discover the router */
CDEBUG(D_NET, "discover %s, cpt = %d\n",
libcfs_nid2str(lpni->lpni_nid), cpt);
lnet_peer_ni_decref_locked(lpni);
if (!rc)
- rtr->lp_rtrcheck_timestamp = now;
+ lpn->lpn_rtrcheck_timestamp = now;
else
CERROR("Failed to discover router %s\n",
libcfs_nid2str(rtr->lp_primary_nid));