From: Mikhal Pershin Date: Wed, 9 Dec 2015 17:48:30 +0000 (+0300) Subject: LU-3285 osc: common client setup/cleanup X-Git-Tag: 2.10.56~64^2~16 X-Git-Url: https://git.whamcloud.com/?p=fs%2Flustre-release.git;a=commitdiff_plain;h=a1bd0f62ee08dcb97aae0e2221ebc85cd345e899 LU-3285 osc: common client setup/cleanup Make OSC code related to client setup/cleanup common for both OSB and MDC. Add new events in mdc_import_event(). Signed-off-by: Mikhal Pershin Change-Id: Ifcbd2011d0360d1671fd3193bb527d472361ebdd Reviewed-on: https://review.whamcloud.com/28014 Reviewed-by: Lai Siyao Reviewed-by: Jinshan Xiong Tested-by: Jenkins Tested-by: Maloo --- diff --git a/lustre/include/lustre_osc.h b/lustre/include/lustre_osc.h index d693db7..07b512f 100644 --- a/lustre/include/lustre_osc.h +++ b/lustre/include/lustre_osc.h @@ -489,9 +489,23 @@ int osc_cache_writeback_range(const struct lu_env *env, struct osc_object *obj, pgoff_t start, pgoff_t end, int hp, int discard); int osc_cache_wait_range(const struct lu_env *env, struct osc_object *obj, pgoff_t start, pgoff_t end); -void osc_io_unplug(const struct lu_env *env, struct client_obd *cli, - struct osc_object *osc); -int lru_queue_work(const struct lu_env *env, void *data); + +int osc_io_unplug0(const struct lu_env *env, struct client_obd *cli, + struct osc_object *osc, int async); + +static inline int osc_io_unplug_async(const struct lu_env *env, + struct client_obd *cli, + struct osc_object *osc) +{ + return osc_io_unplug0(env, cli, osc, 1); +} + +static inline void osc_io_unplug(const struct lu_env *env, + struct client_obd *cli, + struct osc_object *osc) +{ + (void)osc_io_unplug0(env, cli, osc, 0); +} void osc_object_set_contended(struct osc_object *obj); void osc_object_clear_contended(struct osc_object *obj); @@ -520,9 +534,18 @@ int osc_attr_update(const struct lu_env *env, struct cl_object *obj, const struct cl_attr *attr, unsigned valid); int osc_object_glimpse(const struct lu_env *env, const struct cl_object *obj, struct ost_lvb *lvb); +int osc_object_invalidate(const struct lu_env *env, struct osc_object *osc); /* osc_request.c */ void osc_init_grant(struct client_obd *cli, struct obd_connect_data *ocd); +int osc_setup_common(struct obd_device *obd, struct lustre_cfg *lcfg); +int osc_precleanup_common(struct obd_device *obd); +int osc_cleanup_common(struct obd_device *obd); +int osc_set_info_async(const struct lu_env *env, struct obd_export *exp, + u32 keylen, void *key, u32 vallen, void *val, + struct ptlrpc_request_set *set); +int osc_ldlm_resource_invalidate(struct cfs_hash *hs, struct cfs_hash_bd *bd, + struct hlist_node *hnode, void *arg); /***************************************************************************** * diff --git a/lustre/include/obd.h b/lustre/include/obd.h index ccd7fd9..89a427e 100644 --- a/lustre/include/obd.h +++ b/lustre/include/obd.h @@ -305,8 +305,11 @@ struct client_obd { struct mutex cl_mgc_mutex; struct local_oid_storage *cl_mgc_los; struct dt_object *cl_mgc_configs_dir; - atomic_t cl_mgc_refcount; struct obd_export *cl_mgc_mgsexp; + atomic_t cl_mgc_refcount; + /* in-flight control list and total RPCs counter */ + struct list_head cl_flight_waiters; + __u32 cl_rpcs_in_flight; /* checksumming for data sent over the network */ unsigned int cl_checksum:1, /* 0 = disabled, 1 = enabled */ diff --git a/lustre/ldlm/ldlm_lib.c b/lustre/ldlm/ldlm_lib.c index ea5af07..09c84b4 100644 --- a/lustre/ldlm/ldlm_lib.c +++ b/lustre/ldlm/ldlm_lib.c @@ -391,6 +391,9 @@ int client_obd_setup(struct obd_device *obddev, struct lustre_cfg *lcfg) atomic_long_set(&cli->cl_unstable_count, 0); INIT_LIST_HEAD(&cli->cl_shrink_list); + INIT_LIST_HEAD(&cli->cl_flight_waiters); + cli->cl_rpcs_in_flight = 0; + init_waitqueue_head(&cli->cl_destroy_waitq); atomic_set(&cli->cl_destroy_in_flight, 0); #ifdef ENABLE_CHECKSUM @@ -426,7 +429,7 @@ int client_obd_setup(struct obd_device *obddev, struct lustre_cfg *lcfg) cli->cl_max_rpcs_in_flight = OBD_MAX_RIF_MAX; else cli->cl_max_rpcs_in_flight = OBD_MAX_RIF_DEFAULT; - } + } spin_lock_init(&cli->cl_mod_rpcs_lock); spin_lock_init(&cli->cl_mod_rpcs_hist.oh_lock); diff --git a/lustre/mdc/mdc_request.c b/lustre/mdc/mdc_request.c index cc5c7d0..76e0247 100644 --- a/lustre/mdc/mdc_request.c +++ b/lustre/mdc/mdc_request.c @@ -2240,14 +2240,6 @@ static int mdc_set_info_async(const struct lu_env *env, keylen, key, vallen, val, set); RETURN(rc); } - if (KEY_IS(KEY_SPTLRPC_CONF)) { - sptlrpc_conf_client_adapt(exp->exp_obd); - RETURN(0); - } - if (KEY_IS(KEY_FLUSH_CTX)) { - sptlrpc_import_flush_my_ctx(imp); - RETURN(0); - } if (KEY_IS(KEY_CHANGELOG_CLEAR)) { rc = do_set_info_async(imp, MDS_SET_INFO, LUSTRE_MDS_VERSION, keylen, key, vallen, val, set); @@ -2265,13 +2257,8 @@ static int mdc_set_info_async(const struct lu_env *env, RETURN(0); } - /* TODO: these OSC-related keys are ignored for now */ - if (KEY_IS(KEY_CHECKSUM) || KEY_IS(KEY_CACHE_SET) || - KEY_IS(KEY_CACHE_LRU_SHRINK) || KEY_IS(KEY_GRANT_SHRINK)) - RETURN(0); - - CERROR("%s: Unknown key %s\n", exp->exp_obd->obd_name, (char *)key); - RETURN(-EINVAL); + rc = osc_set_info_async(env, exp, keylen, key, vallen, val, set); + RETURN(rc); } static int mdc_get_info(const struct lu_env *env, struct obd_export *exp, @@ -2350,22 +2337,19 @@ static int mdc_fsync(struct obd_export *exp, const struct lu_fid *fid, static int mdc_import_event(struct obd_device *obd, struct obd_import *imp, enum obd_import_event event) { + struct client_obd *cli = &obd->u.cli; int rc = 0; LASSERT(imp->imp_obd == obd); switch (event) { - case IMP_EVENT_DISCON: { - struct client_obd *cli = &obd->u.cli; - + case IMP_EVENT_DISCON: spin_lock(&cli->cl_loi_list_lock); cli->cl_avail_grant = 0; cli->cl_lost_grant = 0; spin_unlock(&cli->cl_loi_list_lock); break; - } - case IMP_EVENT_INACTIVE: { - struct client_obd *cli = &obd->u.cli; + case IMP_EVENT_INACTIVE: /* * Flush current sequence to make client obtain new one * from server in case of disconnect/reconnect. @@ -2377,12 +2361,28 @@ static int mdc_import_event(struct obd_device *obd, struct obd_import *imp, rc = obd_notify_observer(obd, obd, OBD_NOTIFY_INACTIVE); break; - } case IMP_EVENT_INVALIDATE: { struct ldlm_namespace *ns = obd->obd_namespace; + struct lu_env *env; + __u16 refcheck; ldlm_namespace_cleanup(ns, LDLM_FL_LOCAL_ONLY); + env = cl_env_get(&refcheck); + if (!IS_ERR(env)) { + /* Reset grants. All pages go to failing rpcs due to + * the invalid import. + */ + osc_io_unplug(env, cli, NULL); + + cfs_hash_for_each_nolock(ns->ns_rs_hash, + osc_ldlm_resource_invalidate, + env, 0); + cl_env_put(env, &refcheck); + ldlm_namespace_cleanup(ns, LDLM_FL_LOCAL_ONLY); + } else { + rc = PTR_ERR(env); + } break; } case IMP_EVENT_ACTIVE: @@ -2395,7 +2395,7 @@ static int mdc_import_event(struct obd_device *obd, struct obd_import *imp, struct obd_connect_data *ocd = &imp->imp_connect_data; if (OCD_HAS_FLAG(ocd, GRANT)) - osc_init_grant(&obd->u.cli, ocd); + osc_init_grant(cli, ocd); rc = obd_notify_observer(obd, obd, OBD_NOTIFY_OCD); break; @@ -2498,16 +2498,14 @@ static void mdc_llog_finish(struct obd_device *obd) int mdc_setup(struct obd_device *obd, struct lustre_cfg *cfg) { - int rc; + int rc; + ENTRY; - rc = ptlrpcd_addref(); + rc = osc_setup_common(obd, cfg); if (rc < 0) RETURN(rc); - rc = client_obd_setup(obd, cfg); - if (rc) - GOTO(err_ptlrpcd_decref, rc); #ifdef CONFIG_PROC_FS obd->obd_vars = lprocfs_mdc_obd_vars; lprocfs_obd_setup(obd, false); @@ -2524,26 +2522,27 @@ int mdc_setup(struct obd_device *obd, struct lustre_cfg *cfg) if (rc) { CERROR("%s: failed to setup llogging subsystems: rc = %d\n", obd->obd_name, rc); - GOTO(err_mdc_cleanup, rc); + GOTO(err_llog_cleanup, rc); } rc = mdc_changelog_cdev_init(obd); if (rc) { CERROR("%s: failed to setup changelog char device: rc = %d\n", obd->obd_name, rc); - GOTO(err_mdc_cleanup, rc); + GOTO(err_changelog_cleanup, rc); } - EXIT; -err_mdc_cleanup: - if (rc) - client_obd_cleanup(obd); + RETURN(rc); -err_ptlrpcd_decref: - if (rc) - ptlrpcd_decref(); +err_changelog_cleanup: + mdc_llog_finish(obd); +err_llog_cleanup: + ptlrpc_lprocfs_unregister_obd(obd); + lprocfs_obd_cleanup(obd); + lprocfs_free_md_stats(obd); - return rc; + osc_cleanup_common(obd); + return rc; } /* Initialize the default and maximum LOV EA sizes. This allows @@ -2574,6 +2573,8 @@ static int mdc_precleanup(struct obd_device *obd) { ENTRY; + osc_precleanup_common(obd); + /* Failsafe, ok if racy */ if (obd->obd_type->typ_refcnt <= 1) libcfs_kkuc_group_rem(0, KUC_GRP_HSM); @@ -2590,9 +2591,7 @@ static int mdc_precleanup(struct obd_device *obd) static int mdc_cleanup(struct obd_device *obd) { - ptlrpcd_decref(); - - return client_obd_cleanup(obd); + return osc_cleanup_common(obd); } int mdc_process_config(struct obd_device *obd, size_t len, void *buf) diff --git a/lustre/obdclass/genops.c b/lustre/obdclass/genops.c index fb3bd9a..f5f17c4 100644 --- a/lustre/obdclass/genops.c +++ b/lustre/obdclass/genops.c @@ -1986,14 +1986,14 @@ int obd_get_request_slot(struct client_obd *cli) int rc; spin_lock(&cli->cl_loi_list_lock); - if (cli->cl_r_in_flight < cli->cl_max_rpcs_in_flight) { - cli->cl_r_in_flight++; + if (cli->cl_rpcs_in_flight < cli->cl_max_rpcs_in_flight) { + cli->cl_rpcs_in_flight++; spin_unlock(&cli->cl_loi_list_lock); return 0; } init_waitqueue_head(&orsw.orsw_waitq); - list_add_tail(&orsw.orsw_entry, &cli->cl_loi_read_list); + list_add_tail(&orsw.orsw_entry, &cli->cl_flight_waiters); orsw.orsw_signaled = false; spin_unlock(&cli->cl_loi_list_lock); @@ -2009,7 +2009,7 @@ int obd_get_request_slot(struct client_obd *cli) if (rc != 0) { if (!orsw.orsw_signaled) { if (list_empty(&orsw.orsw_entry)) - cli->cl_r_in_flight--; + cli->cl_rpcs_in_flight--; else list_del(&orsw.orsw_entry); } @@ -2031,15 +2031,15 @@ void obd_put_request_slot(struct client_obd *cli) struct obd_request_slot_waiter *orsw; spin_lock(&cli->cl_loi_list_lock); - cli->cl_r_in_flight--; + cli->cl_rpcs_in_flight--; /* If there is free slot, wakeup the first waiter. */ - if (!list_empty(&cli->cl_loi_read_list) && - likely(cli->cl_r_in_flight < cli->cl_max_rpcs_in_flight)) { - orsw = list_entry(cli->cl_loi_read_list.next, + if (!list_empty(&cli->cl_flight_waiters) && + likely(cli->cl_rpcs_in_flight < cli->cl_max_rpcs_in_flight)) { + orsw = list_entry(cli->cl_flight_waiters.next, struct obd_request_slot_waiter, orsw_entry); list_del_init(&orsw->orsw_entry); - cli->cl_r_in_flight++; + cli->cl_rpcs_in_flight++; wake_up(&orsw->orsw_waitq); } spin_unlock(&cli->cl_loi_list_lock); @@ -2089,13 +2089,13 @@ int obd_set_max_rpcs_in_flight(struct client_obd *cli, __u32 max) /* We increase the max_rpcs_in_flight, then wakeup some waiters. */ for (i = 0; i < diff; i++) { - if (list_empty(&cli->cl_loi_read_list)) + if (list_empty(&cli->cl_flight_waiters)) break; - orsw = list_entry(cli->cl_loi_read_list.next, + orsw = list_entry(cli->cl_flight_waiters.next, struct obd_request_slot_waiter, orsw_entry); list_del_init(&orsw->orsw_entry); - cli->cl_r_in_flight++; + cli->cl_rpcs_in_flight++; wake_up(&orsw->orsw_waitq); } spin_unlock(&cli->cl_loi_list_lock); diff --git a/lustre/osc/osc_cache.c b/lustre/osc/osc_cache.c index 33cf890..2231bff 100644 --- a/lustre/osc/osc_cache.c +++ b/lustre/osc/osc_cache.c @@ -2316,8 +2316,8 @@ __must_hold(&cli->cl_loi_list_lock) } } -static int osc_io_unplug0(const struct lu_env *env, struct client_obd *cli, - struct osc_object *osc, int async) +int osc_io_unplug0(const struct lu_env *env, struct client_obd *cli, + struct osc_object *osc, int async) { int rc = 0; @@ -2335,18 +2335,7 @@ static int osc_io_unplug0(const struct lu_env *env, struct client_obd *cli, } return rc; } - -static int osc_io_unplug_async(const struct lu_env *env, - struct client_obd *cli, struct osc_object *osc) -{ - return osc_io_unplug0(env, cli, osc, 1); -} - -void osc_io_unplug(const struct lu_env *env, struct client_obd *cli, - struct osc_object *osc) -{ - (void)osc_io_unplug0(env, cli, osc, 0); -} +EXPORT_SYMBOL(osc_io_unplug0); int osc_prep_async_page(struct osc_object *osc, struct osc_page *ops, struct page *page, loff_t offset) diff --git a/lustre/osc/osc_internal.h b/lustre/osc/osc_internal.h index 9d00c5c..7534443 100644 --- a/lustre/osc/osc_internal.h +++ b/lustre/osc/osc_internal.h @@ -44,6 +44,7 @@ extern struct ptlrpc_request_pool *osc_rq_pool; void osc_wake_cache_waiters(struct client_obd *cli); int osc_shrink_grant_to_target(struct client_obd *cli, __u64 target_bytes); void osc_update_next_shrink(struct client_obd *cli); +int lru_queue_work(const struct lu_env *env, void *data); extern struct ptlrpc_request_set *PTLRPCD_SET; diff --git a/lustre/osc/osc_object.c b/lustre/osc/osc_object.c index 77f6b03..4597201 100644 --- a/lustre/osc/osc_object.c +++ b/lustre/osc/osc_object.c @@ -467,5 +467,5 @@ int osc_object_invalidate(const struct lu_env *env, struct osc_object *osc) RETURN(0); } - +EXPORT_SYMBOL(osc_object_invalidate); /** @} osc */ diff --git a/lustre/osc/osc_request.c b/lustre/osc/osc_request.c index d34c30e..d16a7af 100644 --- a/lustre/osc/osc_request.c +++ b/lustre/osc/osc_request.c @@ -742,11 +742,6 @@ static void osc_update_grant(struct client_obd *cli, struct ost_body *body) } } -static int osc_set_info_async(const struct lu_env *env, struct obd_export *exp, - u32 keylen, void *key, - u32 vallen, void *val, - struct ptlrpc_request_set *set); - static int osc_shrink_grant_interpret(const struct lu_env *env, struct ptlrpc_request *req, void *aa, int rc) @@ -2516,10 +2511,9 @@ out: return err; } -static int osc_set_info_async(const struct lu_env *env, struct obd_export *exp, - u32 keylen, void *key, - u32 vallen, void *val, - struct ptlrpc_request_set *set) +int osc_set_info_async(const struct lu_env *env, struct obd_export *exp, + u32 keylen, void *key, u32 vallen, void *val, + struct ptlrpc_request_set *set) { struct ptlrpc_request *req; struct obd_device *obd = exp->exp_obd; @@ -2635,6 +2629,7 @@ static int osc_set_info_async(const struct lu_env *env, struct obd_export *exp, RETURN(0); } +EXPORT_SYMBOL(osc_set_info_async); static int osc_reconnect(const struct lu_env *env, struct obd_export *exp, struct obd_device *obd, @@ -2695,8 +2690,8 @@ static int osc_disconnect(struct obd_export *exp) return rc; } -static int osc_ldlm_resource_invalidate(struct cfs_hash *hs, - struct cfs_hash_bd *bd, struct hlist_node *hnode, void *arg) +int osc_ldlm_resource_invalidate(struct cfs_hash *hs, struct cfs_hash_bd *bd, + struct hlist_node *hnode, void *arg) { struct lu_env *env = arg; struct ldlm_resource *res = cfs_hash_object(hs, hnode); @@ -2725,6 +2720,7 @@ static int osc_ldlm_resource_invalidate(struct cfs_hash *hs, RETURN(0); } +EXPORT_SYMBOL(osc_ldlm_resource_invalidate); static int osc_import_event(struct obd_device *obd, struct obd_import *imp, @@ -2832,15 +2828,12 @@ static int brw_queue_work(const struct lu_env *env, void *data) RETURN(0); } -int osc_setup(struct obd_device *obd, struct lustre_cfg *lcfg) +int osc_setup_common(struct obd_device *obd, struct lustre_cfg *lcfg) { struct client_obd *cli = &obd->u.cli; - struct obd_type *type; - void *handler; - int rc; - int adding; - int added; - int req_count; + void *handler; + int rc; + ENTRY; rc = ptlrpcd_addref(); @@ -2851,9 +2844,10 @@ int osc_setup(struct obd_device *obd, struct lustre_cfg *lcfg) if (rc) GOTO(out_ptlrpcd, rc); + handler = ptlrpcd_alloc_work(cli->cl_import, brw_queue_work, cli); if (IS_ERR(handler)) - GOTO(out_client_setup, rc = PTR_ERR(handler)); + GOTO(out_ptlrpcd_work, rc = PTR_ERR(handler)); cli->cl_writeback_work = handler; handler = ptlrpcd_alloc_work(cli->cl_import, lru_queue_work, cli); @@ -2867,6 +2861,40 @@ int osc_setup(struct obd_device *obd, struct lustre_cfg *lcfg) cli->cl_grant_shrink_interval = GRANT_SHRINK_INTERVAL; + INIT_LIST_HEAD(&cli->cl_grant_shrink_list); + RETURN(rc); + +out_ptlrpcd_work: + if (cli->cl_writeback_work != NULL) { + ptlrpcd_destroy_work(cli->cl_writeback_work); + cli->cl_writeback_work = NULL; + } + if (cli->cl_lru_work != NULL) { + ptlrpcd_destroy_work(cli->cl_lru_work); + cli->cl_lru_work = NULL; + } + client_obd_cleanup(obd); +out_ptlrpcd: + ptlrpcd_decref(); + RETURN(rc); +} +EXPORT_SYMBOL(osc_setup_common); + +int osc_setup(struct obd_device *obd, struct lustre_cfg *lcfg) +{ + struct client_obd *cli = &obd->u.cli; + struct obd_type *type; + int adding; + int added; + int req_count; + int rc; + + ENTRY; + + rc = osc_setup_common(obd, lcfg); + if (rc < 0) + RETURN(rc); + #ifdef CONFIG_PROC_FS obd->obd_vars = lprocfs_osc_obd_vars; #endif @@ -2921,24 +2949,9 @@ int osc_setup(struct obd_device *obd, struct lustre_cfg *lcfg) spin_unlock(&osc_shrink_lock); RETURN(0); - -out_ptlrpcd_work: - if (cli->cl_writeback_work != NULL) { - ptlrpcd_destroy_work(cli->cl_writeback_work); - cli->cl_writeback_work = NULL; - } - if (cli->cl_lru_work != NULL) { - ptlrpcd_destroy_work(cli->cl_lru_work); - cli->cl_lru_work = NULL; - } -out_client_setup: - client_obd_cleanup(obd); -out_ptlrpcd: - ptlrpcd_decref(); - RETURN(rc); } -static int osc_precleanup(struct obd_device *obd) +int osc_precleanup_common(struct obd_device *obd) { struct client_obd *cli = &obd->u.cli; ENTRY; @@ -2964,12 +2977,22 @@ static int osc_precleanup(struct obd_device *obd) } obd_cleanup_client_import(obd); + RETURN(0); +} +EXPORT_SYMBOL(osc_precleanup_common); + +static int osc_precleanup(struct obd_device *obd) +{ + ENTRY; + + osc_precleanup_common(obd); + ptlrpc_lprocfs_unregister_obd(obd); lprocfs_obd_cleanup(obd); RETURN(0); } -int osc_cleanup(struct obd_device *obd) +int osc_cleanup_common(struct obd_device *obd) { struct client_obd *cli = &obd->u.cli; int rc; @@ -2999,6 +3022,7 @@ int osc_cleanup(struct obd_device *obd) ptlrpcd_decref(); RETURN(rc); } +EXPORT_SYMBOL(osc_cleanup_common); int osc_process_config_base(struct obd_device *obd, struct lustre_cfg *lcfg) { @@ -3015,7 +3039,7 @@ static struct obd_ops osc_obd_ops = { .o_owner = THIS_MODULE, .o_setup = osc_setup, .o_precleanup = osc_precleanup, - .o_cleanup = osc_cleanup, + .o_cleanup = osc_cleanup_common, .o_add_conn = client_import_add_conn, .o_del_conn = client_import_del_conn, .o_connect = client_connect_import,