X-Git-Url: https://git.whamcloud.com/?p=fs%2Flustre-release.git;a=blobdiff_plain;f=lnet%2Flnet%2Fapi-ni.c;h=1c6a9313740f974db40cda0182c0226d35c867e8;hp=b2d4c2580ba5e61e3b1bdbea885168e7b017cf69;hb=9ee453928ab854f43016624578b95c76e73a6a47;hpb=60f6f2b480b482f2022cbea416d8bea87f848bec diff --git a/lnet/lnet/api-ni.c b/lnet/lnet/api-ni.c index b2d4c25..1c6a931 100644 --- a/lnet/lnet/api-ni.c +++ b/lnet/lnet/api-ni.c @@ -80,18 +80,18 @@ MODULE_PARM_DESC(lnet_numa_range, /* * lnet_health_sensitivity determines by how much we decrement the health - * value on sending error. The value defaults to 0, which means health - * checking is turned off by default. + * value on sending error. The value defaults to 100, which means health + * interface health is decremented by 100 points every failure. */ -unsigned int lnet_health_sensitivity = 0; +unsigned int lnet_health_sensitivity = 100; static int sensitivity_set(const char *val, cfs_kernel_param_arg_t *kp); +#ifdef HAVE_KERNEL_PARAM_OPS static struct kernel_param_ops param_ops_health_sensitivity = { .set = sensitivity_set, .get = param_get_int, }; #define param_check_health_sensitivity(name, p) \ __param_check(name, p, int) -#ifdef HAVE_KERNEL_PARAM_OPS module_param(lnet_health_sensitivity, health_sensitivity, S_IRUGO|S_IWUSR); #else module_param_call(lnet_health_sensitivity, sensitivity_set, param_get_int, @@ -100,6 +100,27 @@ module_param_call(lnet_health_sensitivity, sensitivity_set, param_get_int, MODULE_PARM_DESC(lnet_health_sensitivity, "Value to decrement the health value by on error"); +/* + * lnet_recovery_interval determines how often we should perform recovery + * on unhealthy interfaces. + */ +unsigned int lnet_recovery_interval = 1; +static int recovery_interval_set(const char *val, cfs_kernel_param_arg_t *kp); +#ifdef HAVE_KERNEL_PARAM_OPS +static struct kernel_param_ops param_ops_recovery_interval = { + .set = recovery_interval_set, + .get = param_get_int, +}; +#define param_check_recovery_interval(name, p) \ + __param_check(name, p, int) +module_param(lnet_recovery_interval, recovery_interval, S_IRUGO|S_IWUSR); +#else +module_param_call(lnet_recovery_interval, recovery_interval_set, param_get_int, + &lnet_recovery_interval, S_IRUGO|S_IWUSR); +#endif +MODULE_PARM_DESC(lnet_recovery_interval, + "Interval to recover unhealthy interfaces in seconds"); + static int lnet_interfaces_max = LNET_INTERFACES_MAX_DEFAULT; static int intf_max_set(const char *val, cfs_kernel_param_arg_t *kp); @@ -139,8 +160,31 @@ module_param_call(lnet_peer_discovery_disabled, discovery_set, param_get_int, MODULE_PARM_DESC(lnet_peer_discovery_disabled, "Set to 1 to disable peer discovery on this node."); -unsigned lnet_transaction_timeout = 5; +unsigned int lnet_drop_asym_route; +static int drop_asym_route_set(const char *val, cfs_kernel_param_arg_t *kp); + +static struct kernel_param_ops param_ops_drop_asym_route = { + .set = drop_asym_route_set, + .get = param_get_int, +}; + +#define param_check_drop_asym_route(name, p) \ + __param_check(name, p, int) +#ifdef HAVE_KERNEL_PARAM_OPS +module_param(lnet_drop_asym_route, drop_asym_route, 0644); +#else +module_param_call(lnet_drop_asym_route, drop_asym_route_set, param_get_int, + ¶m_ops_drop_asym_route, 0644); +#endif +MODULE_PARM_DESC(lnet_drop_asym_route, + "Set to 1 to drop asymmetrical route messages."); + +#define LNET_TRANSACTION_TIMEOUT_NO_HEALTH_DEFAULT 50 +#define LNET_TRANSACTION_TIMEOUT_HEALTH_DEFAULT 10 + +unsigned lnet_transaction_timeout = LNET_TRANSACTION_TIMEOUT_HEALTH_DEFAULT; static int transaction_to_set(const char *val, cfs_kernel_param_arg_t *kp); +#ifdef HAVE_KERNEL_PARAM_OPS static struct kernel_param_ops param_ops_transaction_timeout = { .set = transaction_to_set, .get = param_get_int, @@ -148,17 +192,18 @@ static struct kernel_param_ops param_ops_transaction_timeout = { #define param_check_transaction_timeout(name, p) \ __param_check(name, p, int) -#ifdef HAVE_KERNEL_PARAM_OPS module_param(lnet_transaction_timeout, transaction_timeout, S_IRUGO|S_IWUSR); #else module_param_call(lnet_transaction_timeout, transaction_to_set, param_get_int, &lnet_transaction_timeout, S_IRUGO|S_IWUSR); #endif -MODULE_PARM_DESC(lnet_peer_discovery_disabled, - "Set to 1 to disable peer discovery on this node."); +MODULE_PARM_DESC(lnet_transaction_timeout, + "Maximum number of seconds to wait for a peer response."); -unsigned lnet_retry_count = 0; +#define LNET_RETRY_COUNT_HEALTH_DEFAULT 3 +unsigned lnet_retry_count = LNET_RETRY_COUNT_HEALTH_DEFAULT; static int retry_count_set(const char *val, cfs_kernel_param_arg_t *kp); +#ifdef HAVE_KERNEL_PARAM_OPS static struct kernel_param_ops param_ops_retry_count = { .set = retry_count_set, .get = param_get_int, @@ -166,7 +211,6 @@ static struct kernel_param_ops param_ops_retry_count = { #define param_check_retry_count(name, p) \ __param_check(name, p, int) -#ifdef HAVE_KERNEL_PARAM_OPS module_param(lnet_retry_count, retry_count, S_IRUGO|S_IWUSR); #else module_param_call(lnet_retry_count, retry_count_set, param_get_int, @@ -212,14 +256,28 @@ sensitivity_set(const char *val, cfs_kernel_param_arg_t *kp) */ mutex_lock(&the_lnet.ln_api_mutex); - if (the_lnet.ln_state != LNET_STATE_RUNNING) { + if (value > LNET_MAX_HEALTH_VALUE) { mutex_unlock(&the_lnet.ln_api_mutex); - return 0; + CERROR("Invalid health value. Maximum: %d value = %lu\n", + LNET_MAX_HEALTH_VALUE, value); + return -EINVAL; } - if (value == *sensitivity) { - mutex_unlock(&the_lnet.ln_api_mutex); - return 0; + /* + * if we're turning on health then use the health timeout + * defaults. + */ + if (*sensitivity == 0 && value != 0) { + lnet_transaction_timeout = LNET_TRANSACTION_TIMEOUT_HEALTH_DEFAULT; + lnet_retry_count = LNET_RETRY_COUNT_HEALTH_DEFAULT; + /* + * if we're turning off health then use the no health timeout + * default. + */ + } else if (*sensitivity != 0 && value == 0) { + lnet_transaction_timeout = + LNET_TRANSACTION_TIMEOUT_NO_HEALTH_DEFAULT; + lnet_retry_count = 0; } *sensitivity = value; @@ -230,6 +288,37 @@ sensitivity_set(const char *val, cfs_kernel_param_arg_t *kp) } static int +recovery_interval_set(const char *val, cfs_kernel_param_arg_t *kp) +{ + int rc; + unsigned *interval = (unsigned *)kp->arg; + unsigned long value; + + rc = kstrtoul(val, 0, &value); + if (rc) { + CERROR("Invalid module parameter value for 'lnet_recovery_interval'\n"); + return rc; + } + + if (value < 1) { + CERROR("lnet_recovery_interval must be at least 1 second\n"); + return -EINVAL; + } + + /* + * The purpose of locking the api_mutex here is to ensure that + * the correct value ends up stored properly. + */ + mutex_lock(&the_lnet.ln_api_mutex); + + *interval = value; + + mutex_unlock(&the_lnet.ln_api_mutex); + + return 0; +} + +static int discovery_set(const char *val, cfs_kernel_param_arg_t *kp) { int rc; @@ -280,15 +369,16 @@ discovery_set(const char *val, cfs_kernel_param_arg_t *kp) } static int -transaction_to_set(const char *val, cfs_kernel_param_arg_t *kp) +drop_asym_route_set(const char *val, cfs_kernel_param_arg_t *kp) { int rc; - unsigned *transaction_to = (unsigned *)kp->arg; + unsigned int *drop_asym_route = (unsigned int *)kp->arg; unsigned long value; rc = kstrtoul(val, 0, &value); if (rc) { - CERROR("Invalid module parameter value for 'lnet_transaction_timeout'\n"); + CERROR("Invalid module parameter value for " + "'lnet_drop_asym_route'\n"); return rc; } @@ -298,11 +388,37 @@ transaction_to_set(const char *val, cfs_kernel_param_arg_t *kp) */ mutex_lock(&the_lnet.ln_api_mutex); - if (the_lnet.ln_state != LNET_STATE_RUNNING) { + if (value == *drop_asym_route) { mutex_unlock(&the_lnet.ln_api_mutex); return 0; } + *drop_asym_route = value; + + mutex_unlock(&the_lnet.ln_api_mutex); + + return 0; +} + +static int +transaction_to_set(const char *val, cfs_kernel_param_arg_t *kp) +{ + int rc; + unsigned *transaction_to = (unsigned *)kp->arg; + unsigned long value; + + rc = kstrtoul(val, 0, &value); + if (rc) { + CERROR("Invalid module parameter value for 'lnet_transaction_timeout'\n"); + return rc; + } + + /* + * The purpose of locking the api_mutex here is to ensure that + * the correct value ends up stored properly. + */ + mutex_lock(&the_lnet.ln_api_mutex); + if (value < lnet_retry_count || value == 0) { mutex_unlock(&the_lnet.ln_api_mutex); CERROR("Invalid value for lnet_transaction_timeout (%lu). " @@ -346,9 +462,10 @@ retry_count_set(const char *val, cfs_kernel_param_arg_t *kp) */ mutex_lock(&the_lnet.ln_api_mutex); - if (the_lnet.ln_state != LNET_STATE_RUNNING) { + if (lnet_health_sensitivity == 0) { mutex_unlock(&the_lnet.ln_api_mutex); - return 0; + CERROR("Can not set retry_count when health feature is turned off\n"); + return -EINVAL; } if (value > lnet_transaction_timeout) { @@ -359,11 +476,6 @@ retry_count_set(const char *val, cfs_kernel_param_arg_t *kp) return -EINVAL; } - if (value == *retry_count) { - mutex_unlock(&the_lnet.ln_api_mutex); - return 0; - } - *retry_count = value; if (value == 0) @@ -1020,6 +1132,7 @@ lnet_prepare(lnet_pid_t requested_pid) INIT_LIST_HEAD(&the_lnet.ln_mt_localNIRecovq); INIT_LIST_HEAD(&the_lnet.ln_mt_peerNIRecovq); init_waitqueue_head(&the_lnet.ln_dc_waitq); + LNetInvalidateEQHandle(&the_lnet.ln_mt_eqh); rc = lnet_descriptor_setup(); if (rc != 0) @@ -1088,6 +1201,8 @@ lnet_prepare(lnet_pid_t requested_pid) static int lnet_unprepare (void) { + int rc; + /* NB no LNET_LOCK since this is the last reference. All LND instances * have shut down already, so it is safe to unlink and free all * descriptors, even those that appear committed to a network op (eg MD @@ -1099,6 +1214,12 @@ lnet_unprepare (void) LASSERT(list_empty(&the_lnet.ln_test_peers)); LASSERT(list_empty(&the_lnet.ln_nets)); + if (!LNetEQHandleIsInvalid(the_lnet.ln_mt_eqh)) { + rc = LNetEQFree(the_lnet.ln_mt_eqh); + LNetInvalidateEQHandle(&the_lnet.ln_mt_eqh); + LASSERT(rc == 0); + } + lnet_portals_destroy(); if (the_lnet.ln_md_containers != NULL) { @@ -1415,6 +1536,28 @@ lnet_get_ni_count(void) return count; } +void +lnet_swap_pinginfo(struct lnet_ping_buffer *pbuf) +{ + struct lnet_ni_status *stat; + int nnis; + int i; + + __swab32s(&pbuf->pb_info.pi_magic); + __swab32s(&pbuf->pb_info.pi_features); + __swab32s(&pbuf->pb_info.pi_pid); + __swab32s(&pbuf->pb_info.pi_nnis); + nnis = pbuf->pb_info.pi_nnis; + if (nnis > pbuf->pb_nnis) + nnis = pbuf->pb_nnis; + for (i = 0; i < nnis; i++) { + stat = &pbuf->pb_info.pi_ni[i]; + __swab64s(&stat->ns_nid); + __swab32s(&stat->ns_status); + } + return; +} + int lnet_ping_info_validate(struct lnet_ping_info *pinfo) { @@ -1840,7 +1983,7 @@ lnet_clear_zombies_nis_locked(struct lnet_net *net) list_del_init(&ni->ni_netlist); /* the ni should be in deleting state. If it's not it's * a bug */ - LASSERT(ni->ni_state & LNET_NI_STATE_DELETING); + LASSERT(ni->ni_state == LNET_NI_STATE_DELETING); cfs_percpt_for_each(ref, j, ni->ni_refs) { if (*ref == 0) continue; @@ -1889,8 +2032,7 @@ lnet_shutdown_lndni(struct lnet_ni *ni) lnet_net_lock(LNET_LOCK_EX); lnet_ni_lock(ni); - ni->ni_state |= LNET_NI_STATE_DELETING; - ni->ni_state &= ~LNET_NI_STATE_ACTIVE; + ni->ni_state = LNET_NI_STATE_DELETING; lnet_ni_unlock(ni); lnet_ni_unlink_locked(ni); lnet_incr_dlc_seq(); @@ -2028,8 +2170,7 @@ lnet_startup_lndni(struct lnet_ni *ni, struct lnet_lnd_tunables *tun) } lnet_ni_lock(ni); - ni->ni_state |= LNET_NI_STATE_ACTIVE; - ni->ni_state &= ~LNET_NI_STATE_INIT; + ni->ni_state = LNET_NI_STATE_ACTIVE; lnet_ni_unlock(ni); /* We keep a reference on the loopback net through the loopback NI */ @@ -2335,12 +2476,9 @@ int lnet_lib_init(void) } the_lnet.ln_refcount = 0; - LNetInvalidateEQHandle(&the_lnet.ln_rc_eqh); INIT_LIST_HEAD(&the_lnet.ln_lnds); INIT_LIST_HEAD(&the_lnet.ln_net_zombie); - INIT_LIST_HEAD(&the_lnet.ln_rcd_zombie); INIT_LIST_HEAD(&the_lnet.ln_msg_resend); - INIT_LIST_HEAD(&the_lnet.ln_rcd_deathrow); /* The hash table size is the number of bits it takes to express the set * ln_num_routes, minus 1 (better to under estimate than over so we @@ -2456,10 +2594,6 @@ LNetNIInit(lnet_pid_t requested_pid) if (rc != 0) goto err_shutdown_lndnis; - rc = lnet_check_routes(); - if (rc != 0) - goto err_destroy_routes; - rc = lnet_rtrpools_alloc(im_a_router); if (rc != 0) goto err_destroy_routes; @@ -2478,6 +2612,12 @@ LNetNIInit(lnet_pid_t requested_pid) lnet_ping_target_update(pbuf, ping_mdh); + rc = LNetEQAlloc(0, lnet_mt_event_handler, &the_lnet.ln_mt_eqh); + if (rc != 0) { + CERROR("Can't allocate monitor thread EQ: %d\n", rc); + goto err_stop_ping; + } + rc = lnet_monitor_thr_start(); if (rc != 0) goto err_stop_ping; @@ -2549,7 +2689,7 @@ LNetNIFini() lnet_fault_fini(); - lnet_router_debugfs_init(); + lnet_router_debugfs_fini(); lnet_peer_discovery_stop(); lnet_push_target_fini(); lnet_monitor_thr_stop(); @@ -3402,12 +3542,6 @@ LNetCtl(unsigned int cmd, void *arg) config->cfg_nid, config->cfg_config_u.cfg_route. rtr_priority); - if (rc == 0) { - rc = lnet_check_routes(); - if (rc != 0) - lnet_del_route(config->cfg_net, - config->cfg_nid); - } mutex_unlock(&the_lnet.ln_api_mutex); return rc;