From bcdb90a56079ac9b7187f67f9c6052d4692d532b Mon Sep 17 00:00:00 2001 From: jcl Date: Thu, 5 Sep 2013 09:49:00 +0200 Subject: [PATCH] LU-3828 mdt: hsm tunable can be set permanently HSM tunable are now always visible, even if hsm is off, so HSM tunning in MGS conf can be set at MDT mount. Rename hsm /proc entry archive_id in default_archive_id Signed-off-by: JC Lafoucriere Change-Id: Ic89de22e11f989ecd2ce01f10764ec6a61a7e870 Reviewed-on: http://review.whamcloud.com/7557 Reviewed-by: John L. Hammond Tested-by: Hudson Tested-by: Maloo Tested-by: Malcolm Cowe Reviewed-by: Faccini Bruno Reviewed-by: Oleg Drokin --- lustre/include/lustre_param.h | 27 +++++++-------- lustre/mdt/mdt_coordinator.c | 55 +++++++++++++++++++++---------- lustre/mdt/mdt_handler.c | 13 ++++++-- lustre/mdt/mdt_hsm_cdt_client.c | 2 +- lustre/mdt/mdt_internal.h | 67 +++++++++++++++++++++---------------- lustre/mdt/mdt_lproc.c | 28 ++++++++++------ lustre/tests/sanity-hsm.sh | 73 +++++++++++++++++++++++++++++------------ 7 files changed, 171 insertions(+), 94 deletions(-) diff --git a/lustre/include/lustre_param.h b/lustre/include/lustre_param.h index f201515..87d0f48 100644 --- a/lustre/include/lustre_param.h +++ b/lustre/include/lustre_param.h @@ -100,21 +100,22 @@ int do_lcfg(char *cfgname, lnet_nid_t nid, int cmd, #define PARAM_ID_UPCALL "identity_upcall=" /* identity upcall */ /* Prefixes for parameters handled by obd's proc methods (XXX_process_config) */ -#define PARAM_OST "ost." -#define PARAM_OSC "osc." -#define PARAM_MDT "mdt." -#define PARAM_MDD "mdd." -#define PARAM_MDC "mdc." -#define PARAM_LLITE "llite." -#define PARAM_LOV "lov." +#define PARAM_OST "ost." +#define PARAM_OSC "osc." +#define PARAM_MDT "mdt." +#define PARAM_HSM "mdt.hsm." +#define PARAM_MDD "mdd." +#define PARAM_MDC "mdc." +#define PARAM_LLITE "llite." +#define PARAM_LOV "lov." #define PARAM_LOD "lod." #define PARAM_OSP "osp." -#define PARAM_SYS "sys." /* global */ -#define PARAM_SRPC "srpc." -#define PARAM_SRPC_FLVR "srpc.flavor." -#define PARAM_SRPC_UDESC "srpc.udesc.cli2mdt" -#define PARAM_SEC "security." -#define PARAM_QUOTA "quota." /* global */ +#define PARAM_SYS "sys." /* global */ +#define PARAM_SRPC "srpc." +#define PARAM_SRPC_FLVR "srpc.flavor." +#define PARAM_SRPC_UDESC "srpc.udesc.cli2mdt" +#define PARAM_SEC "security." +#define PARAM_QUOTA "quota." /* global */ /** @} param */ diff --git a/lustre/mdt/mdt_coordinator.c b/lustre/mdt/mdt_coordinator.c index fbde2c5..991a03e 100644 --- a/lustre/mdt/mdt_coordinator.c +++ b/lustre/mdt/mdt_coordinator.c @@ -385,7 +385,7 @@ static int mdt_coordinator_cb(const struct lu_env *env, * \retval 0 success * \retval -ve failure */ -static int hsm_cdt_procfs_init(struct mdt_device *mdt) +int hsm_cdt_procfs_init(struct mdt_device *mdt) { struct coordinator *cdt = &mdt->mdt_coordinator; int rc = 0; @@ -407,6 +407,29 @@ static int hsm_cdt_procfs_init(struct mdt_device *mdt) } /** + * remove /proc entries for coordinator + * \param mdt [IN] + */ +void hsm_cdt_procfs_fini(struct mdt_device *mdt) +{ + struct coordinator *cdt = &mdt->mdt_coordinator; + + LASSERT(cdt->cdt_state == CDT_STOPPED); + if (cdt->cdt_proc_dir != NULL) + lprocfs_remove(&cdt->cdt_proc_dir); +} + +/** + * get vector of hsm cdt /proc vars + * \param none + * \retval var vector + */ +struct lprocfs_vars *hsm_cdt_get_proc_vars(void) +{ + return lprocfs_mdt_hsm_vars; +} + +/** * coordinator thread * \param data [IN] obd device * \retval 0 success @@ -427,10 +450,6 @@ static int mdt_coordinator(void *data) CDEBUG(D_HSM, "%s: coordinator thread starting, pid=%d\n", mdt_obd_name(mdt), current_pid()); - /* - * create /proc entries for coordinator - */ - hsm_cdt_procfs_init(mdt); /* timeouted cookie vector initialization */ hsd.max_cookie = 0; hsd.cookie_cnt = 0; @@ -876,6 +895,15 @@ int mdt_hsm_cdt_init(struct mdt_device *mdt) hsm_init_ucred(mdt_ucred(cdt_mti)); + /* default values for /proc tunnables + * can be override by MGS conf */ + cdt->cdt_default_archive_id = 1; + cdt->cdt_delay = 60; + cdt->cdt_loop_period = 10; + cdt->cdt_max_request = 3; + cdt->cdt_policy = CDT_DEFAULT_POLICY; + cdt->cdt_timeout = 3600; + RETURN(0); } @@ -924,17 +952,14 @@ int mdt_hsm_cdt_start(struct mdt_device *mdt) CLASSERT(1 << (CDT_POLICY_SHIFT_COUNT - 1) == CDT_POLICY_LAST); cdt->cdt_policy = CDT_DEFAULT_POLICY; + cdt->cdt_state = CDT_INIT; atomic_set(&cdt->cdt_compound_id, cfs_time_current_sec()); /* just need to be larger than previous one */ /* cdt_last_cookie is protected by cdt_llog_lock */ cdt->cdt_last_cookie = cfs_time_current_sec(); - cdt->cdt_loop_period = 10; - cdt->cdt_delay = 60; - cdt->cdt_timeout = 3600; - cdt->cdt_max_request = 3; - cdt->cdt_archive_id = 1; + atomic_set(&cdt->cdt_request_count, 0); /* to avoid deadlock when start is made through /proc @@ -988,10 +1013,6 @@ int mdt_hsm_cdt_stop(struct mdt_device *mdt) RETURN(-EALREADY); } - /* remove proc entries */ - if (cdt->cdt_proc_dir != NULL) - lprocfs_remove(&cdt->cdt_proc_dir); - if (cdt->cdt_state != CDT_STOPPING) { /* stop coordinator thread before cleaning */ cdt->cdt_thread.t_flags = SVC_STOPPING; @@ -1930,7 +1951,7 @@ GENERATE_PROC_METHOD(cdt_loop_period) GENERATE_PROC_METHOD(cdt_delay) GENERATE_PROC_METHOD(cdt_timeout) GENERATE_PROC_METHOD(cdt_max_request) -GENERATE_PROC_METHOD(cdt_archive_id) +GENERATE_PROC_METHOD(cdt_default_archive_id) /* * procfs write method for MDT/hsm_control @@ -2016,8 +2037,8 @@ static struct lprocfs_vars lprocfs_mdt_hsm_vars[] = { { "agents", NULL, NULL, NULL, &mdt_hsm_agent_fops, 0 }, { "agent_actions", NULL, NULL, NULL, &mdt_agent_actions_fops, 0444 }, - { "archive_id", lprocfs_rd_hsm_cdt_archive_id, - lprocfs_wr_hsm_cdt_archive_id, + { "default_archive_id", lprocfs_rd_hsm_cdt_default_archive_id, + lprocfs_wr_hsm_cdt_default_archive_id, NULL, NULL, 0 }, { "grace_delay", lprocfs_rd_hsm_cdt_delay, lprocfs_wr_hsm_cdt_delay, diff --git a/lustre/mdt/mdt_handler.c b/lustre/mdt/mdt_handler.c index dc960e5..5de7d78 100644 --- a/lustre/mdt/mdt_handler.c +++ b/lustre/mdt/mdt_handler.c @@ -5166,9 +5166,16 @@ static int mdt_process_config(const struct lu_env *env, lprocfs_mdt_init_vars(&lvars); rc = class_process_proc_param(PARAM_MDT, lvars.obd_vars, cfg, obd); - if (rc > 0 || rc == -ENOSYS) - /* we don't understand; pass it on */ - rc = next->ld_ops->ldo_process_config(env, next, cfg); + if (rc > 0 || rc == -ENOSYS) { + /* is it an HSM var ? */ + rc = class_process_proc_param(PARAM_HSM, + hsm_cdt_get_proc_vars(), + cfg, obd); + if (rc > 0 || rc == -ENOSYS) + /* we don't understand; pass it on */ + rc = next->ld_ops->ldo_process_config(env, next, + cfg); + } if (old_cfg != NULL) lustre_cfg_free(cfg); diff --git a/lustre/mdt/mdt_hsm_cdt_client.c b/lustre/mdt/mdt_hsm_cdt_client.c index 285818b..ac8da871 100644 --- a/lustre/mdt/mdt_hsm_cdt_client.c +++ b/lustre/mdt/mdt_hsm_cdt_client.c @@ -363,7 +363,7 @@ int mdt_hsm_add_actions(struct mdt_thread_info *mti, if (mh.mh_arch_id != 0) archive_id = mh.mh_arch_id; else - archive_id = cdt->cdt_archive_id; + archive_id = cdt->cdt_default_archive_id; } /* if restore, take an exclusive lock on layout */ diff --git a/lustre/mdt/mdt_internal.h b/lustre/mdt/mdt_internal.h index f4483de..1317255 100644 --- a/lustre/mdt/mdt_internal.h +++ b/lustre/mdt/mdt_internal.h @@ -110,35 +110,41 @@ enum cdt_states { CDT_STOPPED = 0, * cdt_request_lock */ struct coordinator { - struct ptlrpc_thread cdt_thread; /**< coordinator thread */ - struct lu_env cdt_env; /**< coordinator lustre - * env */ - struct lu_context cdt_session; /** session for lu_ucred */ - struct proc_dir_entry *cdt_proc_dir; /**< cdt /proc directory */ - __u64 cdt_policy; /**< flags to defined - * policy */ - enum cdt_states cdt_state; /**< state */ - atomic_t cdt_compound_id; /**< compound id counter */ - __u64 cdt_last_cookie; /**< last cookie allocated */ - struct mutex cdt_llog_lock; /**< protect llog access */ - struct rw_semaphore cdt_agent_lock; /**< protect agent list */ - struct rw_semaphore cdt_request_lock; /**< protect request list */ - struct mutex cdt_restore_lock; /**< protect restore list */ - cfs_time_t cdt_loop_period; /**< llog scan period */ - cfs_time_t cdt_delay; /**< request grace delay */ - cfs_time_t cdt_timeout; /**< request timeout */ - __u32 cdt_archive_id; /** archive id used when - * none are specified */ - __u64 cdt_max_request; /**< max count of started - * requests */ - atomic_t cdt_request_count; /**< current count of - * started requests */ - struct list_head cdt_requests; /**< list of started - * requests */ - struct list_head cdt_agents; /**< list of register - * agents */ - struct list_head cdt_restore_hdl; /**< list of restore lock - * handles */ + struct ptlrpc_thread cdt_thread; /**< coordinator thread */ + struct lu_env cdt_env; /**< coordinator lustre + * env */ + struct lu_context cdt_session; /** session for lu_ucred */ + struct proc_dir_entry *cdt_proc_dir; /**< cdt /proc directory */ + __u64 cdt_policy; /**< policy flags */ + enum cdt_states cdt_state; /**< state */ + atomic_t cdt_compound_id; /**< compound id + * counter */ + __u64 cdt_last_cookie; /**< last cookie + * allocated */ + struct mutex cdt_llog_lock; /**< protect llog + * access */ + struct rw_semaphore cdt_agent_lock; /**< protect agent list */ + struct rw_semaphore cdt_request_lock; /**< protect request + * list */ + struct mutex cdt_restore_lock; /**< protect restore + * list */ + cfs_time_t cdt_loop_period; /**< llog scan period */ + cfs_time_t cdt_delay; /**< request grace + * delay */ + cfs_time_t cdt_timeout; /**< request timeout */ + __u32 cdt_default_archive_id; /** archive id used + * when none are + * specified */ + __u64 cdt_max_request; /**< max count of started + * requests */ + atomic_t cdt_request_count; /**< current count of + * started requests */ + struct list_head cdt_requests; /**< list of started + * requests */ + struct list_head cdt_agents; /**< list of register + * agents */ + struct list_head cdt_restore_hdl; /**< list of restore lock + * handles */ }; /* mdt state flag bits */ @@ -1012,6 +1018,9 @@ int lprocfs_wr_hsm_cdt_control(struct file *file, const char *buffer, unsigned long count, void *data); int lprocfs_rd_hsm_cdt_control(char *page, char **start, off_t off, int count, int *eof, void *data); +int hsm_cdt_procfs_init(struct mdt_device *mdt); +void hsm_cdt_procfs_fini(struct mdt_device *mdt); +struct lprocfs_vars *hsm_cdt_get_proc_vars(void); /* md_hsm helpers */ struct mdt_object *mdt_hsm_get_md_hsm(struct mdt_thread_info *mti, const struct lu_fid *fid, diff --git a/lustre/mdt/mdt_lproc.c b/lustre/mdt/mdt_lproc.c index 0ae4fd0..6953802 100644 --- a/lustre/mdt/mdt_lproc.c +++ b/lustre/mdt/mdt_lproc.c @@ -225,7 +225,14 @@ int mdt_procfs_init(struct mdt_device *mdt, const char *name) lprocfs_mdt_init_vars(&lvars); rc = lprocfs_obd_setup(obd, lvars.obd_vars); if (rc) { - CERROR("Can't init lprocfs, rc %d\n", rc); + CERROR("%s: cannot create proc entries: rc = %d\n", + mdt_obd_name(mdt), rc); + return rc; + } + rc = hsm_cdt_procfs_init(mdt); + if (rc) { + CERROR("%s: cannot create hsm proc entries: rc = %d\n", + mdt_obd_name(mdt), rc); return rc; } ptlrpc_lprocfs_register_obd(obd); @@ -256,15 +263,16 @@ void mdt_procfs_fini(struct mdt_device *mdt) { struct obd_device *obd = mdt2obd_dev(mdt); - if (obd->obd_proc_exports_entry) { - lprocfs_remove_proc_entry("clear", obd->obd_proc_exports_entry); - obd->obd_proc_exports_entry = NULL; - } - lprocfs_free_per_client_stats(obd); - lprocfs_obd_cleanup(obd); - ptlrpc_lprocfs_unregister_obd(obd); - lprocfs_free_md_stats(obd); - lprocfs_free_obd_stats(obd); + if (obd->obd_proc_exports_entry) { + lprocfs_remove_proc_entry("clear", obd->obd_proc_exports_entry); + obd->obd_proc_exports_entry = NULL; + } + lprocfs_free_per_client_stats(obd); + lprocfs_obd_cleanup(obd); + hsm_cdt_procfs_fini(mdt); + ptlrpc_lprocfs_unregister_obd(obd); + lprocfs_free_md_stats(obd); + lprocfs_free_obd_stats(obd); lprocfs_job_stats_fini(obd); } diff --git a/lustre/tests/sanity-hsm.sh b/lustre/tests/sanity-hsm.sh index f1563030..ea4c4c3 100644 --- a/lustre/tests/sanity-hsm.sh +++ b/lustre/tests/sanity-hsm.sh @@ -237,7 +237,11 @@ get_hsm_param() { set_hsm_param() { local param=$1 local value=$2 - do_facet $SINGLEMDS $LCTL set_param -n $HSM_PARAM.$param=$value + local opt=$3 + if [[ "$value" != "" ]]; then + value="=$value" + fi + do_facet $SINGLEMDS $LCTL set_param $opt -n $HSM_PARAM.$param$value return $? } @@ -280,15 +284,11 @@ cdt_clear_non_blocking_restore() { } cdt_clear_mount_state() { - # /!\ conf_param and set_param syntax differ +> we cannot use - # $MDT_PARAM - do_facet $SINGLEMDS $LCTL conf_param -d $FSNAME-MDT0000.mdt.hsm_control + do_facet $SINGLEMDS $LCTL set_param -d -P $MDT_PARAM.hsm_control } cdt_set_mount_state() { - # /!\ conf_param and set_param syntax differ +> we cannot use - # $MDT_PARAM - do_facet $SINGLEMDS $LCTL conf_param $FSNAME-MDT0000.mdt.hsm_control=$1 + do_facet $SINGLEMDS $LCTL set_param -P $MDT_PARAM.hsm_control=$1 } cdt_check_state() { @@ -337,8 +337,7 @@ get_hsm_flags() { local f=$1 local u=$2 - if [[ $u == "user" ]] - then + if [[ $u == "user" ]]; then local st=$($RUNAS $LFS hsm_state $f) else local st=$($LFS hsm_state $f) @@ -380,15 +379,13 @@ check_hsm_flags_user() { copy_file() { local f= - if [[ -d $2 ]] - then + if [[ -d $2 ]]; then f=$2/$(basename $1) else f=$2 fi - if [[ "$3" != 1 ]] - then + if [[ "$3" != 1 ]]; then f=${f/$DIR/$DIR2} fi rm -f $f @@ -797,7 +794,7 @@ test_10d() { wait_request_state $fid ARCHIVE SUCCEED local ar=$(get_hsm_archive_id $f) - local dflt=$(get_hsm_param archive_id) + local dflt=$(get_hsm_param default_archive_id) [[ $ar == $dflt ]] || error "archived file is not on default archive: $ar != $dflt" @@ -1716,8 +1713,7 @@ restore_and_check_size() { n=$(stat -c "%s" $f) # we echo in both cases to show stat is not # hang - if [[ $n != $s ]] - then + if [[ $n != $s ]]; then echo "size seen is $n != $s" err=1 else @@ -1727,8 +1723,7 @@ restore_and_check_size() { sleep 10 cpt=$((cpt + 1)) done - if [[ $cpt -lt 10 ]] - then + if [[ $cpt -lt 10 ]]; then echo " restore is too long" else echo " "done @@ -2251,8 +2246,7 @@ double_verify_reset_hsm_param() { # restore value set_hsm_param $p $save - if [[ $rc == 0 ]] - then + if [[ $rc == 0 ]]; then error "we must not be able to set $HSM_PARAM.$p to 0" fi } @@ -2262,7 +2256,7 @@ test_100() { double_verify_reset_hsm_param grace_delay double_verify_reset_hsm_param request_timeout double_verify_reset_hsm_param max_requests - double_verify_reset_hsm_param archive_id + double_verify_reset_hsm_param default_archive_id } run_test 100 "Set coordinator /proc tunables" @@ -3047,6 +3041,43 @@ test_300() { } run_test 300 "On disk coordinator state kept between MDT umount/mount" +test_301() { + local ai=$(get_hsm_param default_archive_id) + local new=$((ai + 1)) + + set_hsm_param default_archive_id $new -P + fail $SINGLEMDS + local res=$(get_hsm_param default_archive_id) + + # clear value + set_hsm_param default_archive_id "" "-P -d" + + [[ $new == $res ]] || error "Value after MDS restart is $res != $new" +} +run_test 301 "HSM tunnable are persistent" + +test_302() { + local ai=$(get_hsm_param default_archive_id) + local new=$((ai + 1)) + + # stop coordinator + cdt_shutdown + + set_hsm_param default_archive_id $new -P + fail $SINGLEMDS + + # check cdt is on + cdt_check_state enabled + + local res=$(get_hsm_param default_archive_id) + + # clear value + set_hsm_param default_archive_id "" "-P -d" + + [[ $new == $res ]] || error "Value after MDS restart is $res != $new" +} +run_test 302 "HSM tunnable are persistent when CDT is off" + copytool_cleanup complete $SECONDS -- 1.8.3.1