LU-12537 lnet: Sync the start of discovery and monitor threads

[fs/lustre-release.git] / lnet / lnet / api-ni.c

diff --git a/lnet/lnet/api-ni.c b/lnet/lnet/api-ni.c
index 1c6a931..d94f9f3 100644 (file)
--- a/lnet/lnet/api-ni.c
+++ b/lnet/lnet/api-ni.c
@@ -221,6 +221,7 @@ MODULE_PARM_DESC(lnet_retry_count,
 
 
 unsigned lnet_lnd_timeout = LNET_LND_DEFAULT_TIMEOUT;
+unsigned int lnet_current_net_count;
 
 /*
  * This sequence number keeps track of how many times DLC was used to
@@ -1133,6 +1134,7 @@ lnet_prepare(lnet_pid_t requested_pid)
        INIT_LIST_HEAD(&the_lnet.ln_mt_peerNIRecovq);
        init_waitqueue_head(&the_lnet.ln_dc_waitq);
        LNetInvalidateEQHandle(&the_lnet.ln_mt_eqh);
+       init_completion(&the_lnet.ln_started);
 
        rc = lnet_descriptor_setup();
        if (rc != 0)
@@ -1368,18 +1370,28 @@ lnet_cpt_of_nid(lnet_nid_t nid, struct lnet_ni *ni)
 EXPORT_SYMBOL(lnet_cpt_of_nid);
 
 int
-lnet_islocalnet(__u32 net_id)
+lnet_islocalnet_locked(__u32 net_id)
 {
        struct lnet_net *net;
-       int             cpt;
-       bool            local;
-
-       cpt = lnet_net_lock_current();
+       bool local;
 
        net = lnet_get_net_locked(net_id);
 
        local = net != NULL;
 
+       return local;
+}
+
+int
+lnet_islocalnet(__u32 net_id)
+{
+       int cpt;
+       bool local;
+
+       cpt = lnet_net_lock_current();
+
+       local = lnet_islocalnet_locked(net_id);
+
        lnet_net_unlock(cpt);
 
        return local;
@@ -1536,6 +1548,23 @@ lnet_get_ni_count(void)
        return count;
 }
 
+int
+lnet_get_net_count(void)
+{
+       struct lnet_net *net;
+       int count = 0;
+
+       lnet_net_lock(0);
+
+       list_for_each_entry(net, &the_lnet.ln_nets, net_list) {
+               count++;
+       }
+
+       lnet_net_unlock(0);
+
+       return count;
+}
+
 void
 lnet_swap_pinginfo(struct lnet_ping_buffer *pbuf)
 {
@@ -2382,6 +2411,9 @@ lnet_startup_lndnet(struct lnet_net *net, struct lnet_lnd_tunables *tun)
                lnet_net_unlock(LNET_LOCK_EX);
        }
 
+       /* update net count */
+       lnet_current_net_count = lnet_get_net_count();
+
        return ni_count;
 
 failed1:
@@ -2618,29 +2650,34 @@ LNetNIInit(lnet_pid_t requested_pid)
                goto err_stop_ping;
        }
 
-       rc = lnet_monitor_thr_start();
-       if (rc != 0)
-               goto err_stop_ping;
-
        rc = lnet_push_target_init();
        if (rc != 0)
-               goto err_stop_monitor_thr;
+               goto err_stop_ping;
 
        rc = lnet_peer_discovery_start();
        if (rc != 0)
                goto err_destroy_push_target;
 
+       rc = lnet_monitor_thr_start();
+       if (rc != 0)
+               goto err_stop_discovery_thr;
+
        lnet_fault_init();
        lnet_router_debugfs_init();
 
        mutex_unlock(&the_lnet.ln_api_mutex);
 
+       complete_all(&the_lnet.ln_started);
+
+       /* wait for all routers to start */
+       lnet_wait_router_start();
+
        return 0;
 
+err_stop_discovery_thr:
+       lnet_peer_discovery_stop();
 err_destroy_push_target:
        lnet_push_target_fini();
-err_stop_monitor_thr:
-       lnet_monitor_thr_stop();
 err_stop_ping:
        lnet_ping_target_fini();
 err_acceptor_stop:
@@ -2690,9 +2727,9 @@ LNetNIFini()
                lnet_fault_fini();
 
                lnet_router_debugfs_fini();
+               lnet_monitor_thr_stop();
                lnet_peer_discovery_stop();
                lnet_push_target_fini();
-               lnet_monitor_thr_stop();
                lnet_ping_target_fini();
 
                /* Teardown fns that use my own API functions BEFORE here */
@@ -3530,20 +3567,28 @@ LNetCtl(unsigned int cmd, void *arg)
        case IOC_LIBCFS_FAIL_NID:
                return lnet_fail_nid(data->ioc_nid, data->ioc_count);
 
-       case IOC_LIBCFS_ADD_ROUTE:
+       case IOC_LIBCFS_ADD_ROUTE: {
+               /* default router sensitivity to 1 */
+               unsigned int sensitivity = 1;
                config = arg;
 
                if (config->cfg_hdr.ioc_len < sizeof(*config))
                        return -EINVAL;
 
+               if (config->cfg_config_u.cfg_route.rtr_sensitivity) {
+                       sensitivity =
+                         config->cfg_config_u.cfg_route.rtr_sensitivity;
+               }
+
                mutex_lock(&the_lnet.ln_api_mutex);
                rc = lnet_add_route(config->cfg_net,
                                    config->cfg_config_u.cfg_route.rtr_hop,
                                    config->cfg_nid,
                                    config->cfg_config_u.cfg_route.
-                                       rtr_priority);
+                                       rtr_priority, sensitivity);
                mutex_unlock(&the_lnet.ln_api_mutex);
                return rc;
+       }
 
        case IOC_LIBCFS_DEL_ROUTE:
                config = arg;
@@ -3569,7 +3614,9 @@ LNetCtl(unsigned int cmd, void *arg)
                                    &config->cfg_nid,
                                    &config->cfg_config_u.cfg_route.rtr_flags,
                                    &config->cfg_config_u.cfg_route.
-                                       rtr_priority);
+                                       rtr_priority,
+                                   &config->cfg_config_u.cfg_route.
+                                       rtr_sensitivity);
                mutex_unlock(&the_lnet.ln_api_mutex);
                return rc;
 
@@ -3843,7 +3890,7 @@ LNetCtl(unsigned int cmd, void *arg)
                 * that deadline to the wall clock.
                 */
                deadline += ktime_get_seconds();
-               return lnet_notify(NULL, data->ioc_nid, data->ioc_flags,
+               return lnet_notify(NULL, data->ioc_nid, data->ioc_flags, false,
                                   deadline);
        }
 
@@ -3873,9 +3920,9 @@ LNetCtl(unsigned int cmd, void *arg)
                /* If timeout is negative then set default of 3 minutes */
                if (((s32)data->ioc_u32[1] <= 0) ||
                    data->ioc_u32[1] > (DEFAULT_PEER_TIMEOUT * MSEC_PER_SEC))
-                       timeout = msecs_to_jiffies(DEFAULT_PEER_TIMEOUT * MSEC_PER_SEC);
+                       timeout = cfs_time_seconds(DEFAULT_PEER_TIMEOUT);
                else
-                       timeout = msecs_to_jiffies(data->ioc_u32[1]);
+                       timeout = nsecs_to_jiffies(data->ioc_u32[1] * NSEC_PER_MSEC);
 
                rc = lnet_ping(id, timeout, data->ioc_pbuf1,
                               data->ioc_plen1 / sizeof(struct lnet_process_id));
@@ -3895,9 +3942,9 @@ LNetCtl(unsigned int cmd, void *arg)
                /* If timeout is negative then set default of 3 minutes */
                if (((s32)ping->op_param) <= 0 ||
                    ping->op_param > (DEFAULT_PEER_TIMEOUT * MSEC_PER_SEC))
-                       timeout = msecs_to_jiffies(DEFAULT_PEER_TIMEOUT * MSEC_PER_SEC);
+                       timeout = cfs_time_seconds(DEFAULT_PEER_TIMEOUT);
                else
-                       timeout = msecs_to_jiffies(ping->op_param);
+                       timeout = nsecs_to_jiffies(ping->op_param * NSEC_PER_MSEC);
 
                rc = lnet_ping(ping->ping_id, timeout,
                               ping->ping_buf,
@@ -4043,7 +4090,7 @@ static int lnet_ping(struct lnet_process_id id, signed long timeout,
        int which;
        int unlinked = 0;
        int replied = 0;
-       const signed long a_long_time = msecs_to_jiffies(60 * MSEC_PER_SEC);
+       const signed long a_long_time = cfs_time_seconds(60);
        struct lnet_ping_buffer *pbuf;
        struct lnet_process_id tmpid;
        int i;