Whamcloud - gitweb
LU-11389 lnet: increase lnet transaction timeout
[fs/lustre-release.git] / lnet / lnet / api-ni.c
index aa35054..4c8decc 100644 (file)
@@ -100,6 +100,27 @@ module_param_call(lnet_health_sensitivity, sensitivity_set, param_get_int,
 MODULE_PARM_DESC(lnet_health_sensitivity,
                "Value to decrement the health value by on error");
 
+/*
+ * lnet_recovery_interval determines how often we should perform recovery
+ * on unhealthy interfaces.
+ */
+unsigned int lnet_recovery_interval = 1;
+static int recovery_interval_set(const char *val, cfs_kernel_param_arg_t *kp);
+static struct kernel_param_ops param_ops_recovery_interval = {
+       .set = recovery_interval_set,
+       .get = param_get_int,
+};
+#define param_check_recovery_interval(name, p) \
+               __param_check(name, p, int)
+#ifdef HAVE_KERNEL_PARAM_OPS
+module_param(lnet_recovery_interval, recovery_interval, S_IRUGO|S_IWUSR);
+#else
+module_param_call(lnet_recovery_interval, recovery_interval_set, param_get_int,
+                 &lnet_recovery_interval, S_IRUGO|S_IWUSR);
+#endif
+MODULE_PARM_DESC(lnet_recovery_interval,
+               "Interval to recover unhealthy interfaces in seconds");
+
 static int lnet_interfaces_max = LNET_INTERFACES_MAX_DEFAULT;
 static int intf_max_set(const char *val, cfs_kernel_param_arg_t *kp);
 
@@ -139,16 +160,43 @@ module_param_call(lnet_peer_discovery_disabled, discovery_set, param_get_int,
 MODULE_PARM_DESC(lnet_peer_discovery_disabled,
                "Set to 1 to disable peer discovery on this node.");
 
-unsigned lnet_transaction_timeout = 5;
-module_param(lnet_transaction_timeout, uint, 0444);
+unsigned lnet_transaction_timeout = 50;
+static int transaction_to_set(const char *val, cfs_kernel_param_arg_t *kp);
+static struct kernel_param_ops param_ops_transaction_timeout = {
+       .set = transaction_to_set,
+       .get = param_get_int,
+};
+
+#define param_check_transaction_timeout(name, p) \
+               __param_check(name, p, int)
+#ifdef HAVE_KERNEL_PARAM_OPS
+module_param(lnet_transaction_timeout, transaction_timeout, S_IRUGO|S_IWUSR);
+#else
+module_param_call(lnet_transaction_timeout, transaction_to_set, param_get_int,
+                 &lnet_transaction_timeout, S_IRUGO|S_IWUSR);
+#endif
 MODULE_PARM_DESC(lnet_transaction_timeout,
-               "Time in seconds to wait for a REPLY or an ACK");
+               "Maximum number of seconds to wait for a peer response.");
 
 unsigned lnet_retry_count = 0;
-module_param(lnet_retry_count, uint, 0444);
+static int retry_count_set(const char *val, cfs_kernel_param_arg_t *kp);
+static struct kernel_param_ops param_ops_retry_count = {
+       .set = retry_count_set,
+       .get = param_get_int,
+};
+
+#define param_check_retry_count(name, p) \
+               __param_check(name, p, int)
+#ifdef HAVE_KERNEL_PARAM_OPS
+module_param(lnet_retry_count, retry_count, S_IRUGO|S_IWUSR);
+#else
+module_param_call(lnet_retry_count, retry_count_set, param_get_int,
+                 &lnet_retry_count, S_IRUGO|S_IWUSR);
+#endif
 MODULE_PARM_DESC(lnet_retry_count,
                 "Maximum number of times to retry transmitting a message");
 
+
 unsigned lnet_lnd_timeout = LNET_LND_DEFAULT_TIMEOUT;
 
 /*
@@ -190,9 +238,11 @@ sensitivity_set(const char *val, cfs_kernel_param_arg_t *kp)
                return 0;
        }
 
-       if (value == *sensitivity) {
+       if (value > LNET_MAX_HEALTH_VALUE) {
                mutex_unlock(&the_lnet.ln_api_mutex);
-               return 0;
+               CERROR("Invalid health value. Maximum: %d value = %lu\n",
+                      LNET_MAX_HEALTH_VALUE, value);
+               return -EINVAL;
        }
 
        *sensitivity = value;
@@ -203,6 +253,42 @@ sensitivity_set(const char *val, cfs_kernel_param_arg_t *kp)
 }
 
 static int
+recovery_interval_set(const char *val, cfs_kernel_param_arg_t *kp)
+{
+       int rc;
+       unsigned *interval = (unsigned *)kp->arg;
+       unsigned long value;
+
+       rc = kstrtoul(val, 0, &value);
+       if (rc) {
+               CERROR("Invalid module parameter value for 'lnet_recovery_interval'\n");
+               return rc;
+       }
+
+       if (value < 1) {
+               CERROR("lnet_recovery_interval must be at least 1 second\n");
+               return -EINVAL;
+       }
+
+       /*
+        * The purpose of locking the api_mutex here is to ensure that
+        * the correct value ends up stored properly.
+        */
+       mutex_lock(&the_lnet.ln_api_mutex);
+
+       if (the_lnet.ln_state != LNET_STATE_RUNNING) {
+               mutex_unlock(&the_lnet.ln_api_mutex);
+               return 0;
+       }
+
+       *interval = value;
+
+       mutex_unlock(&the_lnet.ln_api_mutex);
+
+       return 0;
+}
+
+static int
 discovery_set(const char *val, cfs_kernel_param_arg_t *kp)
 {
        int rc;
@@ -253,6 +339,103 @@ discovery_set(const char *val, cfs_kernel_param_arg_t *kp)
 }
 
 static int
+transaction_to_set(const char *val, cfs_kernel_param_arg_t *kp)
+{
+       int rc;
+       unsigned *transaction_to = (unsigned *)kp->arg;
+       unsigned long value;
+
+       rc = kstrtoul(val, 0, &value);
+       if (rc) {
+               CERROR("Invalid module parameter value for 'lnet_transaction_timeout'\n");
+               return rc;
+       }
+
+       /*
+        * The purpose of locking the api_mutex here is to ensure that
+        * the correct value ends up stored properly.
+        */
+       mutex_lock(&the_lnet.ln_api_mutex);
+
+       if (the_lnet.ln_state != LNET_STATE_RUNNING) {
+               mutex_unlock(&the_lnet.ln_api_mutex);
+               return 0;
+       }
+
+       if (value < lnet_retry_count || value == 0) {
+               mutex_unlock(&the_lnet.ln_api_mutex);
+               CERROR("Invalid value for lnet_transaction_timeout (%lu). "
+                      "Has to be greater than lnet_retry_count (%u)\n",
+                      value, lnet_retry_count);
+               return -EINVAL;
+       }
+
+       if (value == *transaction_to) {
+               mutex_unlock(&the_lnet.ln_api_mutex);
+               return 0;
+       }
+
+       *transaction_to = value;
+       if (lnet_retry_count == 0)
+               lnet_lnd_timeout = value;
+       else
+               lnet_lnd_timeout = value / lnet_retry_count;
+
+       mutex_unlock(&the_lnet.ln_api_mutex);
+
+       return 0;
+}
+
+static int
+retry_count_set(const char *val, cfs_kernel_param_arg_t *kp)
+{
+       int rc;
+       unsigned *retry_count = (unsigned *)kp->arg;
+       unsigned long value;
+
+       rc = kstrtoul(val, 0, &value);
+       if (rc) {
+               CERROR("Invalid module parameter value for 'lnet_retry_count'\n");
+               return rc;
+       }
+
+       /*
+        * The purpose of locking the api_mutex here is to ensure that
+        * the correct value ends up stored properly.
+        */
+       mutex_lock(&the_lnet.ln_api_mutex);
+
+       if (the_lnet.ln_state != LNET_STATE_RUNNING) {
+               mutex_unlock(&the_lnet.ln_api_mutex);
+               return 0;
+       }
+
+       if (value > lnet_transaction_timeout) {
+               mutex_unlock(&the_lnet.ln_api_mutex);
+               CERROR("Invalid value for lnet_retry_count (%lu). "
+                      "Has to be smaller than lnet_transaction_timeout (%u)\n",
+                      value, lnet_transaction_timeout);
+               return -EINVAL;
+       }
+
+       if (value == *retry_count) {
+               mutex_unlock(&the_lnet.ln_api_mutex);
+               return 0;
+       }
+
+       *retry_count = value;
+
+       if (value == 0)
+               lnet_lnd_timeout = lnet_transaction_timeout;
+       else
+               lnet_lnd_timeout = lnet_transaction_timeout / value;
+
+       mutex_unlock(&the_lnet.ln_api_mutex);
+
+       return 0;
+}
+
+static int
 intf_max_set(const char *val, cfs_kernel_param_arg_t *kp)
 {
        int value, rc;
@@ -615,28 +798,70 @@ lnet_unregister_lnd(struct lnet_lnd *lnd)
 EXPORT_SYMBOL(lnet_unregister_lnd);
 
 void
+lnet_counters_get_common(struct lnet_counters_common *common)
+{
+       struct lnet_counters *ctr;
+       int i;
+
+       memset(common, 0, sizeof(*common));
+
+       lnet_net_lock(LNET_LOCK_EX);
+
+       cfs_percpt_for_each(ctr, i, the_lnet.ln_counters) {
+               common->lcc_msgs_max     += ctr->lct_common.lcc_msgs_max;
+               common->lcc_msgs_alloc   += ctr->lct_common.lcc_msgs_alloc;
+               common->lcc_errors       += ctr->lct_common.lcc_errors;
+               common->lcc_send_count   += ctr->lct_common.lcc_send_count;
+               common->lcc_recv_count   += ctr->lct_common.lcc_recv_count;
+               common->lcc_route_count  += ctr->lct_common.lcc_route_count;
+               common->lcc_drop_count   += ctr->lct_common.lcc_drop_count;
+               common->lcc_send_length  += ctr->lct_common.lcc_send_length;
+               common->lcc_recv_length  += ctr->lct_common.lcc_recv_length;
+               common->lcc_route_length += ctr->lct_common.lcc_route_length;
+               common->lcc_drop_length  += ctr->lct_common.lcc_drop_length;
+       }
+       lnet_net_unlock(LNET_LOCK_EX);
+}
+EXPORT_SYMBOL(lnet_counters_get_common);
+
+void
 lnet_counters_get(struct lnet_counters *counters)
 {
        struct lnet_counters *ctr;
+       struct lnet_counters_health *health = &counters->lct_health;
        int             i;
 
        memset(counters, 0, sizeof(*counters));
 
+       lnet_counters_get_common(&counters->lct_common);
+
        lnet_net_lock(LNET_LOCK_EX);
 
        cfs_percpt_for_each(ctr, i, the_lnet.ln_counters) {
-               counters->msgs_max     += ctr->msgs_max;
-               counters->msgs_alloc   += ctr->msgs_alloc;
-               counters->errors       += ctr->errors;
-               counters->send_count   += ctr->send_count;
-               counters->recv_count   += ctr->recv_count;
-               counters->route_count  += ctr->route_count;
-               counters->drop_count   += ctr->drop_count;
-               counters->send_length  += ctr->send_length;
-               counters->recv_length  += ctr->recv_length;
-               counters->route_length += ctr->route_length;
-               counters->drop_length  += ctr->drop_length;
-
+               health->lch_rst_alloc    += ctr->lct_health.lch_rst_alloc;
+               health->lch_resend_count += ctr->lct_health.lch_resend_count;
+               health->lch_response_timeout_count +=
+                               ctr->lct_health.lch_response_timeout_count;
+               health->lch_local_interrupt_count +=
+                               ctr->lct_health.lch_local_interrupt_count;
+               health->lch_local_dropped_count +=
+                               ctr->lct_health.lch_local_dropped_count;
+               health->lch_local_aborted_count +=
+                               ctr->lct_health.lch_local_aborted_count;
+               health->lch_local_no_route_count +=
+                               ctr->lct_health.lch_local_no_route_count;
+               health->lch_local_timeout_count +=
+                               ctr->lct_health.lch_local_timeout_count;
+               health->lch_local_error_count +=
+                               ctr->lct_health.lch_local_error_count;
+               health->lch_remote_dropped_count +=
+                               ctr->lct_health.lch_remote_dropped_count;
+               health->lch_remote_error_count +=
+                               ctr->lct_health.lch_remote_error_count;
+               health->lch_remote_timeout_count +=
+                               ctr->lct_health.lch_remote_timeout_count;
+               health->lch_network_timeout_count +=
+                               ctr->lct_health.lch_network_timeout_count;
        }
        lnet_net_unlock(LNET_LOCK_EX);
 }
@@ -1098,15 +1323,6 @@ lnet_islocalnet(__u32 net_id)
        return local;
 }
 
-bool
-lnet_is_ni_healthy_locked(struct lnet_ni *ni)
-{
-       if (ni->ni_state & LNET_NI_STATE_ACTIVE)
-               return true;
-
-       return false;
-}
-
 struct lnet_ni  *
 lnet_nid2ni_locked(lnet_nid_t nid, int cpt)
 {
@@ -1655,11 +1871,6 @@ lnet_ni_tq_credits(struct lnet_ni *ni)
 static void
 lnet_ni_unlink_locked(struct lnet_ni *ni)
 {
-       if (!list_empty(&ni->ni_cptlist)) {
-               list_del_init(&ni->ni_cptlist);
-               lnet_ni_decref_locked(ni, 0);
-       }
-
        /* move it to zombie list and nobody can find it anymore */
        LASSERT(!list_empty(&ni->ni_netlist));
        list_move(&ni->ni_netlist, &ni->ni_net->net_ni_zombie);
@@ -1688,7 +1899,7 @@ lnet_clear_zombies_nis_locked(struct lnet_net *net)
                list_del_init(&ni->ni_netlist);
                /* the ni should be in deleting state. If it's not it's
                 * a bug */
-               LASSERT(ni->ni_state & LNET_NI_STATE_DELETING);
+               LASSERT(ni->ni_state == LNET_NI_STATE_DELETING);
                cfs_percpt_for_each(ref, j, ni->ni_refs) {
                        if (*ref == 0)
                                continue;
@@ -1737,8 +1948,7 @@ lnet_shutdown_lndni(struct lnet_ni *ni)
 
        lnet_net_lock(LNET_LOCK_EX);
        lnet_ni_lock(ni);
-       ni->ni_state |= LNET_NI_STATE_DELETING;
-       ni->ni_state &= ~LNET_NI_STATE_ACTIVE;
+       ni->ni_state = LNET_NI_STATE_DELETING;
        lnet_ni_unlock(ni);
        lnet_ni_unlink_locked(ni);
        lnet_incr_dlc_seq();
@@ -1876,8 +2086,7 @@ lnet_startup_lndni(struct lnet_ni *ni, struct lnet_lnd_tunables *tun)
        }
 
        lnet_ni_lock(ni);
-       ni->ni_state |= LNET_NI_STATE_ACTIVE;
-       ni->ni_state &= ~LNET_NI_STATE_INIT;
+       ni->ni_state = LNET_NI_STATE_ACTIVE;
        lnet_ni_unlock(ni);
 
        /* We keep a reference on the loopback net through the loopback NI */
@@ -3117,6 +3326,102 @@ __u32 lnet_get_dlc_seq_locked(void)
        return atomic_read(&lnet_dlc_seq_no);
 }
 
+static void
+lnet_ni_set_healthv(lnet_nid_t nid, int value, bool all)
+{
+       struct lnet_net *net;
+       struct lnet_ni *ni;
+
+       lnet_net_lock(LNET_LOCK_EX);
+       list_for_each_entry(net, &the_lnet.ln_nets, net_list) {
+               list_for_each_entry(ni, &net->net_ni_list, ni_netlist) {
+                       if (ni->ni_nid == nid || all) {
+                               atomic_set(&ni->ni_healthv, value);
+                               if (list_empty(&ni->ni_recovery) &&
+                                   value < LNET_MAX_HEALTH_VALUE) {
+                                       CERROR("manually adding local NI %s to recovery\n",
+                                              libcfs_nid2str(ni->ni_nid));
+                                       list_add_tail(&ni->ni_recovery,
+                                                     &the_lnet.ln_mt_localNIRecovq);
+                                       lnet_ni_addref_locked(ni, 0);
+                               }
+                               if (!all) {
+                                       lnet_net_unlock(LNET_LOCK_EX);
+                                       return;
+                               }
+                       }
+               }
+       }
+       lnet_net_unlock(LNET_LOCK_EX);
+}
+
+static int
+lnet_get_local_ni_hstats(struct lnet_ioctl_local_ni_hstats *stats)
+{
+       int cpt, rc = 0;
+       struct lnet_ni *ni;
+       lnet_nid_t nid = stats->hlni_nid;
+
+       cpt = lnet_net_lock_current();
+       ni = lnet_nid2ni_locked(nid, cpt);
+
+       if (!ni) {
+               rc = -ENOENT;
+               goto unlock;
+       }
+
+       stats->hlni_local_interrupt = atomic_read(&ni->ni_hstats.hlt_local_interrupt);
+       stats->hlni_local_dropped = atomic_read(&ni->ni_hstats.hlt_local_dropped);
+       stats->hlni_local_aborted = atomic_read(&ni->ni_hstats.hlt_local_aborted);
+       stats->hlni_local_no_route = atomic_read(&ni->ni_hstats.hlt_local_no_route);
+       stats->hlni_local_timeout = atomic_read(&ni->ni_hstats.hlt_local_timeout);
+       stats->hlni_local_error = atomic_read(&ni->ni_hstats.hlt_local_error);
+       stats->hlni_health_value = atomic_read(&ni->ni_healthv);
+
+unlock:
+       lnet_net_unlock(cpt);
+
+       return rc;
+}
+
+static int
+lnet_get_local_ni_recovery_list(struct lnet_ioctl_recovery_list *list)
+{
+       struct lnet_ni *ni;
+       int i = 0;
+
+       lnet_net_lock(LNET_LOCK_EX);
+       list_for_each_entry(ni, &the_lnet.ln_mt_localNIRecovq, ni_recovery) {
+               list->rlst_nid_array[i] = ni->ni_nid;
+               i++;
+               if (i >= LNET_MAX_SHOW_NUM_NID)
+                       break;
+       }
+       lnet_net_unlock(LNET_LOCK_EX);
+       list->rlst_num_nids = i;
+
+       return 0;
+}
+
+static int
+lnet_get_peer_ni_recovery_list(struct lnet_ioctl_recovery_list *list)
+{
+       struct lnet_peer_ni *lpni;
+       int i = 0;
+
+       lnet_net_lock(LNET_LOCK_EX);
+       list_for_each_entry(lpni, &the_lnet.ln_mt_peerNIRecovq, lpni_recovery) {
+               list->rlst_nid_array[i] = lpni->lpni_nid;
+               i++;
+               if (i >= LNET_MAX_SHOW_NUM_NID)
+                       break;
+       }
+       lnet_net_unlock(LNET_LOCK_EX);
+       list->rlst_num_nids = i;
+
+       return 0;
+}
+
 /**
  * LNet ioctl handler.
  *
@@ -3327,6 +3632,33 @@ LNetCtl(unsigned int cmd, void *arg)
                return rc;
        }
 
+       case IOC_LIBCFS_GET_LOCAL_HSTATS: {
+               struct lnet_ioctl_local_ni_hstats *stats = arg;
+
+               if (stats->hlni_hdr.ioc_len < sizeof(*stats))
+                       return -EINVAL;
+
+               mutex_lock(&the_lnet.ln_api_mutex);
+               rc = lnet_get_local_ni_hstats(stats);
+               mutex_unlock(&the_lnet.ln_api_mutex);
+
+               return rc;
+       }
+
+       case IOC_LIBCFS_GET_RECOVERY_QUEUE: {
+               struct lnet_ioctl_recovery_list *list = arg;
+               if (list->rlst_hdr.ioc_len < sizeof(*list))
+                       return -EINVAL;
+
+               mutex_lock(&the_lnet.ln_api_mutex);
+               if (list->rlst_type == LNET_HEALTH_TYPE_LOCAL_NI)
+                       rc = lnet_get_local_ni_recovery_list(list);
+               else
+                       rc = lnet_get_peer_ni_recovery_list(list);
+               mutex_unlock(&the_lnet.ln_api_mutex);
+               return rc;
+       }
+
        case IOC_LIBCFS_ADD_PEER_NI: {
                struct lnet_ioctl_peer_cfg *cfg = arg;
 
@@ -3402,6 +3734,30 @@ LNetCtl(unsigned int cmd, void *arg)
                return rc;
        }
 
+       case IOC_LIBCFS_SET_HEALHV: {
+               struct lnet_ioctl_reset_health_cfg *cfg = arg;
+               int value;
+               if (cfg->rh_hdr.ioc_len < sizeof(*cfg))
+                       return -EINVAL;
+               if (cfg->rh_value < 0 ||
+                   cfg->rh_value > LNET_MAX_HEALTH_VALUE)
+                       value = LNET_MAX_HEALTH_VALUE;
+               else
+                       value = cfg->rh_value;
+               CDEBUG(D_NET, "Manually setting healthv to %d for %s:%s. all = %d\n",
+                      value, (cfg->rh_type == LNET_HEALTH_TYPE_LOCAL_NI) ?
+                      "local" : "peer", libcfs_nid2str(cfg->rh_nid), cfg->rh_all);
+               mutex_lock(&the_lnet.ln_api_mutex);
+               if (cfg->rh_type == LNET_HEALTH_TYPE_LOCAL_NI)
+                       lnet_ni_set_healthv(cfg->rh_nid, value,
+                                            cfg->rh_all);
+               else
+                       lnet_peer_ni_set_healthv(cfg->rh_nid, value,
+                                                 cfg->rh_all);
+               mutex_unlock(&the_lnet.ln_api_mutex);
+               return 0;
+       }
+
        case IOC_LIBCFS_NOTIFY_ROUTER: {
                time64_t deadline = ktime_get_real_seconds() - data->ioc_u64[0];
 
@@ -3873,3 +4229,16 @@ out:
 
        return rc;
 }
+
+/**
+ * Retrieve peer discovery status.
+ *
+ * \retval 1 if lnet_peer_discovery_disabled is 0
+ * \retval 0 if lnet_peer_discovery_disabled is 1
+ */
+int
+LNetGetPeerDiscoveryStatus(void)
+{
+       return !lnet_peer_discovery_disabled;
+}
+EXPORT_SYMBOL(LNetGetPeerDiscoveryStatus);