From bc25ea34379b5abacffbbb41d0ecb5dfcd676cd0 Mon Sep 17 00:00:00 2001 From: Artem Blagodarenko Date: Thu, 11 Apr 2013 11:31:43 +0400 Subject: [PATCH] LU-3155 mgs: set_param -P option that sets value permanently set_param and conf_param have different syntaxes. Also conf_param has unimplemented paths and no wildcarding support. This patch adds set_param -P option, that replaces the whole conf_param "direct" proc access with a simple upcall-type mechanism from the MGC. Option conf_param is saved now for compatibility. Xyratex-bug-id: MRP-272 Signed-off-by: Artem Blagodarenko Change-Id: I19ec3f14e339f780ef1403f2902f6baa9e789614 Reviewed-on: http://review.whamcloud.com/6025 Tested-by: Hudson Tested-by: Maloo Reviewed-by: Andreas Dilger Reviewed-by: Emoly Liu --- lustre/include/lustre_cfg.h | 2 + lustre/include/lustre_disk.h | 2 + lustre/include/obd_class.h | 17 +++-- lustre/llite/llite_lib.c | 5 +- lustre/mgc/mgc_request.c | 131 ++++++++++++++++++++++++++---------- lustre/mgs/mgs_handler.c | 137 ++++++++++++++++++++----------------- lustre/mgs/mgs_internal.h | 1 + lustre/mgs/mgs_llog.c | 69 +++++++++++++------ lustre/obdclass/obd_config.c | 54 ++++++++++++--- lustre/tests/conf-sanity.sh | 59 ++++++++++++++++ lustre/utils/lctl.c | 11 +-- lustre/utils/llog_reader.c | 15 +++-- lustre/utils/lustre_cfg.c | 157 +++++++++++++++++++++++++++++++++---------- 13 files changed, 485 insertions(+), 175 deletions(-) diff --git a/lustre/include/lustre_cfg.h b/lustre/include/lustre_cfg.h index d86ba77..bde93b6 100644 --- a/lustre/include/lustre_cfg.h +++ b/lustre/include/lustre_cfg.h @@ -88,6 +88,8 @@ enum lcfg_command_type { LCFG_SET_LDLM_TIMEOUT = 0x00ce030, /**< set ldlm_timeout */ LCFG_PRE_CLEANUP = 0x00cf031, /**< call type-specific pre * cleanup cleanup */ + LCFG_SET_PARAM = 0x00ce032, /**< use set_param syntax to set + *a proc parameters */ }; struct lustre_cfg_bufs { diff --git a/lustre/include/lustre_disk.h b/lustre/include/lustre_disk.h index 48406b5..de1ec28 100644 --- a/lustre/include/lustre_disk.h +++ b/lustre/include/lustre_disk.h @@ -98,6 +98,8 @@ #define LDD_F_IR_CAPABLE 0x2000 /** the MGS refused to register the target. */ #define LDD_F_ERROR 0x4000 +/** process at lctl conf_param */ +#define LDD_F_PARAM2 0x8000 /* opc for target register */ #define LDD_F_OPC_REG 0x10000000 diff --git a/lustre/include/obd_class.h b/lustre/include/obd_class.h index 029b21e..5e91034 100644 --- a/lustre/include/obd_class.h +++ b/lustre/include/obd_class.h @@ -183,20 +183,25 @@ int class_config_dump_llog(const struct lu_env *env, struct llog_ctxt *ctxt, char *name, struct config_llog_instance *cfg); enum { - CONFIG_T_CONFIG = 0, - CONFIG_T_SPTLRPC = 1, - CONFIG_T_RECOVER = 2, - CONFIG_T_MAX = 3 + CONFIG_T_CONFIG = 0, + CONFIG_T_SPTLRPC = 1, + CONFIG_T_RECOVER = 2, + CONFIG_T_PARAMS = 3, + CONFIG_T_MAX = 4 }; +#define PARAMS_FILENAME "params" +#define LCTL_UPCALL "lctl" + /* list of active configuration logs */ struct config_llog_data { struct ldlm_res_id cld_resid; struct config_llog_instance cld_cfg; cfs_list_t cld_list_chain; cfs_atomic_t cld_refcount; - struct config_llog_data *cld_sptlrpc;/* depended sptlrpc log */ - struct config_llog_data *cld_recover; /* imperative recover log */ + struct config_llog_data *cld_sptlrpc;/* depended sptlrpc log */ + struct config_llog_data *cld_params; /* common parameters log */ + struct config_llog_data *cld_recover;/* imperative recover log */ struct obd_export *cld_mgcexp; struct mutex cld_lock; int cld_type; diff --git a/lustre/llite/llite_lib.c b/lustre/llite/llite_lib.c index e2dd3a2..7dd1ee4 100644 --- a/lustre/llite/llite_lib.c +++ b/lustre/llite/llite_lib.c @@ -1088,7 +1088,7 @@ void lu_context_keys_dump(void); void ll_put_super(struct super_block *sb) { - struct config_llog_instance cfg; + struct config_llog_instance cfg, params_cfg; struct obd_device *obd; struct lustre_sb_info *lsi = s2lsi(sb); struct ll_sb_info *sbi = ll_s2sbi(sb); @@ -1103,6 +1103,9 @@ void ll_put_super(struct super_block *sb) cfg.cfg_instance = sb; lustre_end_log(sb, profilenm, &cfg); + params_cfg.cfg_instance = sb; + lustre_end_log(sb, PARAMS_FILENAME, ¶ms_cfg); + if (sbi->ll_md_exp) { obd = class_exp2obd(sbi->ll_md_exp); if (obd) diff --git a/lustre/mgc/mgc_request.c b/lustre/mgc/mgc_request.c index e503c1b..f91f8a1 100644 --- a/lustre/mgc/mgc_request.c +++ b/lustre/mgc/mgc_request.c @@ -56,7 +56,7 @@ static int mgc_name2resid(char *name, int len, struct ldlm_res_id *res_id, { __u64 resname = 0; - if (len > 8) { + if (len > sizeof(resname)) { CERROR("name too long: %s\n", name); return -EINVAL; } @@ -75,7 +75,8 @@ static int mgc_name2resid(char *name, int len, struct ldlm_res_id *res_id, case CONFIG_T_SPTLRPC: resname = 0; break; - case CONFIG_T_RECOVER: + case CONFIG_T_RECOVER: + case CONFIG_T_PARAMS: resname = type; break; default: @@ -97,15 +98,18 @@ EXPORT_SYMBOL(mgc_fsname2resid); int mgc_logname2resid(char *logname, struct ldlm_res_id *res_id, int type) { - char *name_end; - int len; - - /* logname consists of "fsname-nodetype". - * e.g. "lustre-MDT0001", "SUN-000-client" */ - name_end = strrchr(logname, '-'); - LASSERT(name_end); - len = name_end - logname; - return mgc_name2resid(logname, len, res_id, type); + char *name_end; + int len; + + /* logname consists of "fsname-nodetype". + * e.g. "lustre-MDT0001", "SUN-000-client" + * there is an exception: llog "params" */ + name_end = strrchr(logname, '-'); + if (!name_end) + len = strlen(logname); + else + len = name_end - logname; + return mgc_name2resid(logname, len, res_id, type); } /********************** config llog list **********************/ @@ -143,6 +147,8 @@ static void config_log_put(struct config_llog_data *cld) config_log_put(cld->cld_recover); if (cld->cld_sptlrpc) config_log_put(cld->cld_sptlrpc); + if (cld->cld_params) + config_log_put(cld->cld_params); if (cld_is_sptlrpc(cld)) sptlrpc_conf_log_stop(cld->cld_logname); @@ -278,6 +284,19 @@ static struct config_llog_data *config_recover_log_add(struct obd_device *obd, return cld; } +static struct config_llog_data *config_params_log_add(struct obd_device *obd, + struct config_llog_instance *cfg, struct super_block *sb) +{ + struct config_llog_instance lcfg = *cfg; + struct config_llog_data *cld; + + lcfg.cfg_instance = sb; + + cld = do_config_log_add(obd, PARAMS_FILENAME, CONFIG_T_PARAMS, + &lcfg, sb); + + return cld; +} /** Add this log to the list of active logs watched by an MGC. * Active means we're watching for updates. @@ -288,12 +307,14 @@ static int config_log_add(struct obd_device *obd, char *logname, struct config_llog_instance *cfg, struct super_block *sb) { - struct lustre_sb_info *lsi = s2lsi(sb); - struct config_llog_data *cld; - struct config_llog_data *sptlrpc_cld; - char seclogname[32]; - char *ptr; - ENTRY; + struct lustre_sb_info *lsi = s2lsi(sb); + struct config_llog_data *cld; + struct config_llog_data *sptlrpc_cld; + struct config_llog_data *params_cld; + char seclogname[32]; + char *ptr; + int rc; + ENTRY; CDEBUG(D_MGC, "adding config log %s:%p\n", logname, cfg->cfg_instance); @@ -316,32 +337,49 @@ static int config_log_add(struct obd_device *obd, char *logname, CONFIG_T_SPTLRPC, NULL, NULL); if (IS_ERR(sptlrpc_cld)) { CERROR("can't create sptlrpc log: %s\n", seclogname); - RETURN(PTR_ERR(sptlrpc_cld)); + GOTO(out_err, rc = PTR_ERR(sptlrpc_cld)); } } + params_cld = config_params_log_add(obd, cfg, sb); + if (IS_ERR(params_cld)) { + rc = PTR_ERR(params_cld); + CERROR("%s: can't create params log: rc = %d\n", + obd->obd_name, rc); + GOTO(out_err1, rc); + } - cld = do_config_log_add(obd, logname, CONFIG_T_CONFIG, cfg, sb); - if (IS_ERR(cld)) { - CERROR("can't create log: %s\n", logname); - config_log_put(sptlrpc_cld); - RETURN(PTR_ERR(cld)); - } + cld = do_config_log_add(obd, logname, CONFIG_T_CONFIG, cfg, sb); + if (IS_ERR(cld)) { + CERROR("can't create log: %s\n", logname); + GOTO(out_err2, rc = PTR_ERR(cld)); + } - cld->cld_sptlrpc = sptlrpc_cld; + cld->cld_sptlrpc = sptlrpc_cld; + cld->cld_params = params_cld; LASSERT(lsi->lsi_lmd); if (!(lsi->lsi_lmd->lmd_flags & LMD_FLG_NOIR)) { struct config_llog_data *recover_cld; *strrchr(seclogname, '-') = 0; recover_cld = config_recover_log_add(obd, seclogname, cfg, sb); - if (IS_ERR(recover_cld)) { - config_log_put(cld); - RETURN(PTR_ERR(recover_cld)); - } - cld->cld_recover = recover_cld; - } + if (IS_ERR(recover_cld)) + GOTO(out_err3, rc = PTR_ERR(recover_cld)); + cld->cld_recover = recover_cld; + } - RETURN(0); + RETURN(0); + +out_err3: + config_log_put(cld); + +out_err2: + config_log_put(params_cld); + +out_err1: + config_log_put(sptlrpc_cld); + +out_err: + RETURN(rc); } DEFINE_MUTEX(llog_process_lock); @@ -352,6 +390,7 @@ static int config_log_end(char *logname, struct config_llog_instance *cfg) { struct config_llog_data *cld; struct config_llog_data *cld_sptlrpc = NULL; + struct config_llog_data *cld_params = NULL; struct config_llog_data *cld_recover = NULL; int rc = 0; ENTRY; @@ -391,11 +430,20 @@ static int config_log_end(char *logname, struct config_llog_instance *cfg) spin_lock(&config_list_lock); cld_sptlrpc = cld->cld_sptlrpc; cld->cld_sptlrpc = NULL; + cld_params = cld->cld_params; + cld->cld_params = NULL; spin_unlock(&config_list_lock); if (cld_sptlrpc) config_log_put(cld_sptlrpc); + if (cld_params) { + mutex_lock(&cld_params->cld_lock); + cld_params->cld_stopping = 1; + mutex_unlock(&cld_params->cld_lock); + config_log_put(cld_params); + } + /* drop the ref from the find */ config_log_put(cld); /* drop the start ref */ @@ -527,8 +575,8 @@ static int mgc_requeue_thread(void *data) cld_prev = cld; cld->cld_lostlock = 0; - if (likely(!stopped)) - do_requeue(cld); + if (likely(!stopped)) + do_requeue(cld); spin_lock(&config_list_lock); } @@ -1706,7 +1754,7 @@ static int mgc_process_cfg_log(struct obd_device *mgc, LCONSOLE_ERROR_MSG(0x13a, "Failed to get MGS " "log %s and no local copy." "\n", cld->cld_logname); - GOTO(out_pop, rc = -ENOTCONN); + GOTO(out_pop, rc = -ENOENT); } CDEBUG(D_MGC, "Failed to get MGS log %s, using local " "copy for now, will try to update later.\n", @@ -1908,6 +1956,19 @@ static int mgc_process_config(struct obd_device *obd, obd_count len, void *buf) if (rc) CERROR("Cannot process recover llog %d\n", rc); } + + if (rc == 0 && cld->cld_params != NULL) { + rc = mgc_process_log(obd, cld->cld_params); + if (rc == -ENOENT) { + CDEBUG(D_MGC, "There is no params" + "config file yet\n"); + rc = 0; + } + /* params log is optional */ + if (rc) + CERROR("%s: can't process params llog: rc = %d\n", + obd->obd_name, rc); + } config_log_put(cld); break; diff --git a/lustre/mgs/mgs_handler.c b/lustre/mgs/mgs_handler.c index 782c6d5..9c6af86 100644 --- a/lustre/mgs/mgs_handler.c +++ b/lustre/mgs/mgs_handler.c @@ -132,14 +132,20 @@ out_cfg: return rc; } -static int mgs_completion_ast_config(struct ldlm_lock *lock, __u64 flags, - void *cbdata) +enum ast_type { + AST_CONFIG = 1, + AST_PARAMS = 2, + AST_IR = 3 +}; + +static int mgs_completion_ast_generic(struct ldlm_lock *lock, __u64 flags, + void *cbdata, enum ast_type type) { ENTRY; if (!(flags & (LDLM_FL_BLOCK_WAIT | LDLM_FL_BLOCK_GRANTED | LDLM_FL_BLOCK_CONV))) { - struct fs_db *fsdb; + struct fs_db *fsdb; /* l_ast_data is used as a marker to avoid cancel ldlm lock * twice. See LU-2317. */ @@ -151,8 +157,22 @@ static int mgs_completion_ast_config(struct ldlm_lock *lock, __u64 flags, if (fsdb != NULL) { struct lustre_handle lockh; - /* clear the bit before lock put */ - clear_bit(FSDB_REVOKING_LOCK, &fsdb->fsdb_flags); + switch(type) { + case AST_CONFIG: + /* clear the bit before lock put */ + clear_bit(FSDB_REVOKING_LOCK, + &fsdb->fsdb_flags); + break; + case AST_PARAMS: + clear_bit(FSDB_REVOKING_PARAMS, + &fsdb->fsdb_flags); + break; + case AST_IR: + mgs_ir_notify_complete(fsdb); + break; + default: + LBUG(); + } ldlm_lock2handle(lock, &lockh); ldlm_lock_decref_and_cancel(&lockh, LCK_EX); @@ -162,80 +182,77 @@ static int mgs_completion_ast_config(struct ldlm_lock *lock, __u64 flags, RETURN(ldlm_completion_ast(lock, flags, cbdata)); } -static int mgs_completion_ast_ir(struct ldlm_lock *lock, __u64 flags, - void *cbdata) +static int mgs_completion_ast_config(struct ldlm_lock *lock, __u64 flags, + void *cbdata) { - ENTRY; - - if (!(flags & (LDLM_FL_BLOCK_WAIT | LDLM_FL_BLOCK_GRANTED | - LDLM_FL_BLOCK_CONV))) { - struct fs_db *fsdb; - - /* l_ast_data is used as a marker to avoid cancel ldlm lock - * twice. See LU-1259. */ - lock_res_and_lock(lock); - fsdb = (struct fs_db *)lock->l_ast_data; - lock->l_ast_data = NULL; - unlock_res_and_lock(lock); - - if (fsdb != NULL) { - struct lustre_handle lockh; - - mgs_ir_notify_complete(fsdb); + return mgs_completion_ast_generic(lock, flags, cbdata, AST_CONFIG); +} - ldlm_lock2handle(lock, &lockh); - ldlm_lock_decref_and_cancel(&lockh, LCK_EX); - } - } +static int mgs_completion_ast_params(struct ldlm_lock *lock, __u64 flags, + void *cbdata) +{ + return mgs_completion_ast_generic(lock, flags, cbdata, AST_PARAMS); +} - RETURN(ldlm_completion_ast(lock, flags, cbdata)); +static int mgs_completion_ast_ir(struct ldlm_lock *lock, __u64 flags, + void *cbdata) +{ + return mgs_completion_ast_generic(lock, flags, cbdata, AST_IR); } void mgs_revoke_lock(struct mgs_device *mgs, struct fs_db *fsdb, int type) { - ldlm_completion_callback cp = NULL; - struct lustre_handle lockh = { 0 }; - struct ldlm_res_id res_id; + ldlm_completion_callback cp = NULL; + struct lustre_handle lockh = { 0 }; + struct ldlm_res_id res_id; __u64 flags = LDLM_FL_ATOMIC_CB; - int rc; - ENTRY; - - LASSERT(fsdb->fsdb_name[0] != '\0'); - rc = mgc_fsname2resid(fsdb->fsdb_name, &res_id, type); - LASSERT(rc == 0); + int rc; + ENTRY; - switch (type) { - case CONFIG_T_CONFIG: - cp = mgs_completion_ast_config; + LASSERT(fsdb->fsdb_name[0] != '\0'); + rc = mgc_fsname2resid(fsdb->fsdb_name, &res_id, type); + LASSERT(rc == 0); + switch (type) { + case CONFIG_T_CONFIG: + cp = mgs_completion_ast_config; if (test_and_set_bit(FSDB_REVOKING_LOCK, &fsdb->fsdb_flags)) - rc = -EALREADY; - break; - case CONFIG_T_RECOVER: - cp = mgs_completion_ast_ir; - default: - break; - } + rc = -EALREADY; + break; + case CONFIG_T_PARAMS: + cp = mgs_completion_ast_params; + if (test_and_set_bit(FSDB_REVOKING_PARAMS, &fsdb->fsdb_flags)) + rc = -EALREADY; + break; + case CONFIG_T_RECOVER: + cp = mgs_completion_ast_ir; + default: + break; + } - if (!rc) { - LASSERT(cp != NULL); + if (!rc) { + LASSERT(cp != NULL); rc = ldlm_cli_enqueue_local(mgs->mgs_obd->obd_namespace, &res_id, LDLM_PLAIN, NULL, LCK_EX, &flags, ldlm_blocking_ast, cp, NULL, fsdb, 0, LVB_T_NONE, NULL, &lockh); - if (rc != ELDLM_OK) { - CERROR("can't take cfg lock for "LPX64"/"LPX64"(%d)\n", - le64_to_cpu(res_id.name[0]), - le64_to_cpu(res_id.name[1]), rc); + if (rc != ELDLM_OK) { + CERROR("can't take cfg lock for "LPX64"/"LPX64"(%d)\n", + le64_to_cpu(res_id.name[0]), + le64_to_cpu(res_id.name[1]), rc); - if (type == CONFIG_T_CONFIG) + if (type == CONFIG_T_CONFIG) clear_bit(FSDB_REVOKING_LOCK, - &fsdb->fsdb_flags); - } - /* lock has been cancelled in completion_ast. */ - } + &fsdb->fsdb_flags); + + if (type == CONFIG_T_PARAMS) + clear_bit(FSDB_REVOKING_PARAMS, + &fsdb->fsdb_flags); + } + /* lock has been cancelled in completion_ast. */ + } - RETURN_EXIT; + RETURN_EXIT; } /* rc=0 means ok diff --git a/lustre/mgs/mgs_internal.h b/lustre/mgs/mgs_internal.h index fe5fcb0..06bcc99 100644 --- a/lustre/mgs/mgs_internal.h +++ b/lustre/mgs/mgs_internal.h @@ -117,6 +117,7 @@ struct mgs_tgt_srpc_conf { #define FSDB_MGS_SELF (3) /* for '_mgs', used by sptlrpc */ #define FSDB_OSCNAME18 (4) /* old 1.8 style OSC naming */ #define FSDB_UDESC (5) /* sptlrpc user desc, will be obsolete */ +#define FSDB_REVOKING_PARAMS (6) /* DLM lock is being revoked */ struct fs_db { char fsdb_name[9]; diff --git a/lustre/mgs/mgs_llog.c b/lustre/mgs/mgs_llog.c index 1d4f58c..6303c4c 100644 --- a/lustre/mgs/mgs_llog.c +++ b/lustre/mgs/mgs_llog.c @@ -2711,8 +2711,13 @@ static int mgs_wlp_lcfg(const struct lu_env *env, return rc; lustre_cfg_bufs_reset(bufs, tgtname); - lustre_cfg_bufs_set_string(bufs, 1, ptr); - lcfg = lustre_cfg_new(LCFG_PARAM, bufs); + lustre_cfg_bufs_set_string(bufs, 1, ptr); + if (mti->mti_flags & LDD_F_PARAM2) + lustre_cfg_bufs_set_string(bufs, 2, LCTL_UPCALL); + + lcfg = lustre_cfg_new((mti->mti_flags & LDD_F_PARAM2) ? + LCFG_SET_PARAM : LCFG_PARAM, bufs); + if (!lcfg) return -ENOMEM; rc = mgs_write_log_direct(env, mgs, fsdb, logname,lcfg,tgtname,comment); @@ -2720,6 +2725,22 @@ static int mgs_wlp_lcfg(const struct lu_env *env, return rc; } +static int mgs_write_log_param2(const struct lu_env *env, + struct mgs_device *mgs, + struct fs_db *fsdb, + struct mgs_target_info *mti, char *ptr) +{ + struct lustre_cfg_bufs bufs; + int rc = 0; + ENTRY; + + CDEBUG(D_MGS, "next param '%s'\n", ptr); + rc = mgs_wlp_lcfg(env, mgs, fsdb, mti, PARAMS_FILENAME, &bufs, + mti->mti_svname, ptr); + + RETURN(rc); +} + /* write global variable settings into log */ static int mgs_write_log_sys(const struct lu_env *env, struct mgs_device *mgs, struct fs_db *fsdb, @@ -3738,10 +3759,14 @@ int mgs_setparam(const struct lu_env *env, struct mgs_device *mgs, } CDEBUG(D_MGS, "setparam fs='%s' device='%s'\n", fsname, devname); - rc = mgs_find_or_make_fsdb(env, mgs, fsname, &fsdb); - if (rc) - RETURN(rc); - if (!test_bit(FSDB_MGS_SELF, &fsdb->fsdb_flags) && + rc = mgs_find_or_make_fsdb(env, mgs, + lcfg->lcfg_command == LCFG_SET_PARAM ? + PARAMS_FILENAME : fsname, &fsdb); + if (rc) + RETURN(rc); + + if (lcfg->lcfg_command != LCFG_SET_PARAM && + !test_bit(FSDB_MGS_SELF, &fsdb->fsdb_flags) && test_bit(FSDB_LOG_EMPTY, &fsdb->fsdb_flags)) { CERROR("No filesystem targets for %s. cfg_device from lctl " "is '%s'\n", fsname, devname); @@ -3771,20 +3796,26 @@ int mgs_setparam(const struct lu_env *env, struct mgs_device *mgs, if (server_make_name(rc, mti->mti_stripe_index, mti->mti_fsname, mti->mti_svname)) GOTO(out, rc = -EINVAL); + /* + * Revoke lock so everyone updates. Should be alright if + * someone was already reading while we were updating the logs, + * so we don't really need to hold the lock while we're + * writing (above). + */ + if (lcfg->lcfg_command == LCFG_SET_PARAM) { + mti->mti_flags = rc | LDD_F_PARAM2; + mutex_lock(&fsdb->fsdb_mutex); + rc = mgs_write_log_param2(env, mgs, fsdb, mti, mti->mti_params); + mutex_unlock(&fsdb->fsdb_mutex); + mgs_revoke_lock(mgs, fsdb, CONFIG_T_PARAMS); + } else { + mti->mti_flags = rc | LDD_F_PARAM; + mutex_lock(&fsdb->fsdb_mutex); + rc = mgs_write_log_param(env, mgs, fsdb, mti, mti->mti_params); + mutex_unlock(&fsdb->fsdb_mutex); + mgs_revoke_lock(mgs, fsdb, CONFIG_T_CONFIG); + } - mti->mti_flags = rc | LDD_F_PARAM; - - mutex_lock(&fsdb->fsdb_mutex); - rc = mgs_write_log_param(env, mgs, fsdb, mti, mti->mti_params); - mutex_unlock(&fsdb->fsdb_mutex); - - /* - * Revoke lock so everyone updates. Should be alright if - * someone was already reading while we were updating the logs, - * so we don't really need to hold the lock while we're - * writing (above). - */ - mgs_revoke_lock(mgs, fsdb, CONFIG_T_CONFIG); out: OBD_FREE_PTR(mti); RETURN(rc); diff --git a/lustre/obdclass/obd_config.c b/lustre/obdclass/obd_config.c index c663e5d..c2ca70c 100644 --- a/lustre/obdclass/obd_config.c +++ b/lustre/obdclass/obd_config.c @@ -1049,6 +1049,45 @@ struct lustre_cfg *lustre_cfg_rename(struct lustre_cfg *cfg, } EXPORT_SYMBOL(lustre_cfg_rename); +static int process_param2_config(struct lustre_cfg *lcfg) +{ + char *param = lustre_cfg_string(lcfg, 1); + char *upcall = lustre_cfg_string(lcfg, 2); + char *argv[] = { + [0] = "/usr/sbin/lctl", + [1] = "set_param", + [2] = param, + [3] = NULL + }; + struct timeval start; + struct timeval end; + int rc; + ENTRY; + + /* Add upcall processing here. Now only lctl is supported */ + if (strcmp(upcall, LCTL_UPCALL) != 0) { + CERROR("Unsupported upcall %s\n", upcall); + RETURN(-EINVAL); + } + + cfs_gettimeofday(&start); + rc = USERMODEHELPER(argv[0], argv, NULL); + cfs_gettimeofday(&end); + + if (rc < 0) { + CERROR("lctl: error invoking upcall %s %s %s: rc = %d; " + "time %ldus\n", argv[0], argv[1], argv[2], rc, + cfs_timeval_sub(&end, &start, NULL)); + } else { + CDEBUG(D_HA, "lctl: invoked upcall %s %s %s, time %ldus\n", + argv[0], argv[1], argv[2], + cfs_timeval_sub(&end, &start, NULL)); + rc = 0; + } + + RETURN(rc); +} + void lustre_register_quota_process_config(int (*qpc)(struct lustre_cfg *lcfg)) { quota_process_config = qpc; @@ -1164,11 +1203,14 @@ int class_process_config(struct lustre_cfg *lcfg) err = (*quota_process_config)(lcfg); GOTO(out, err); } - /* Fall through */ - break; - } - } + break; + } + case LCFG_SET_PARAM: { + err = process_param2_config(lcfg); + GOTO(out, 0); + } + } /* Commands that require a device */ obd = class_name2obd(lustre_cfg_string(lcfg, 0)); if (obd == NULL) { @@ -1205,24 +1247,20 @@ int class_process_config(struct lustre_cfg *lcfg) case LCFG_POOL_NEW: { err = obd_pool_new(obd, lustre_cfg_string(lcfg, 2)); GOTO(out, err = 0); - break; } case LCFG_POOL_ADD: { err = obd_pool_add(obd, lustre_cfg_string(lcfg, 2), lustre_cfg_string(lcfg, 3)); GOTO(out, err = 0); - break; } case LCFG_POOL_REM: { err = obd_pool_rem(obd, lustre_cfg_string(lcfg, 2), lustre_cfg_string(lcfg, 3)); GOTO(out, err = 0); - break; } case LCFG_POOL_DEL: { err = obd_pool_del(obd, lustre_cfg_string(lcfg, 2)); GOTO(out, err = 0); - break; } default: { err = obd_process_config(obd, sizeof(*lcfg), lcfg); diff --git a/lustre/tests/conf-sanity.sh b/lustre/tests/conf-sanity.sh index d1010cc..e96cc8f 100644 --- a/lustre/tests/conf-sanity.sh +++ b/lustre/tests/conf-sanity.sh @@ -4157,6 +4157,65 @@ test_75() { # LU-2374 } run_test 75 "The order of --index should be irrelevant" +test_76() { + [[ $(lustre_version_code mgs) -ge $(version_code 2.4.52) ]] || + { skip "Need MDS version at least 2.4.52" && return 0; } + setup + local MDMB_PARAM="osc.*.max_dirty_mb" + echo "Change MGS params" + local MAX_DIRTY_MB=$($LCTL get_param -n $MDMB_PARAM | + head -1) + echo "max_dirty_mb: $MAX_DIRTY_MB" + local NEW_MAX_DIRTY_MB=$((MAX_DIRTY_MB + MAX_DIRTY_MB)) + echo "new_max_dirty_mb: $NEW_MAX_DIRTY_MB" + do_facet mgs $LCTL set_param -P $MDMB_PARAM=$NEW_MAX_DIRTY_MB + wait_update $HOSTNAME "lctl get_param -n $MDMB_PARAM | + head -1" $NEW_MAX_DIRTY_MB + MAX_DIRTY_MB=$($LCTL get_param -n $MDMB_PARAM | head -1) + echo "$MAX_DIRTY_MB" + [ $MAX_DIRTY_MB = $NEW_MAX_DIRTY_MB ] || + error "error while apply max_dirty_mb" + + echo "Check the value is stored after remount" + stopall + setupall + wait_update $HOSTNAME "lctl get_param -n $MDMB_PARAM | + head -1" $NEW_MAX_DIRTY_MB + MAX_DIRTY_MB=$($LCTL get_param -n $MDMB_PARAM | head -1) + [ $MAX_DIRTY_MB = $NEW_MAX_DIRTY_MB ] || + error "max_dirty_mb is not saved after remount" + + echo "Change OST params" + CLIENT_PARAM="obdfilter.*.client_cache_count" + local CLIENT_CACHE_COUNT + CLIENT_CACHE_COUNT=$(do_facet ost1 $LCTL get_param -n $CLIENT_PARAM | + head -1) + echo "client_cache_count: $CLIENT_CACHE_COUNT" + NEW_CLIENT_CACHE_COUNT=$((CLIENT_CACHE_COUNT+CLIENT_CACHE_COUNT)) + echo "new_client_cache_count: $NEW_CLIENT_CACHE_COUNT" + do_facet mgs $LCTL set_param -P $CLIENT_PARAM=$NEW_CLIENT_CACHE_COUNT + wait_update $(facet_host ost1) "lctl get_param -n $CLIENT_PARAM | + head -1" $NEW_CLIENT_CACHE_COUNT + CLIENT_CACHE_COUNT=$(do_facet ost1 $LCTL get_param -n $CLIENT_PARAM | + head -1) + echo "$CLIENT_CACHE_COUNT" + [ $CLIENT_CACHE_COUNT = $NEW_CLIENT_CACHE_COUNT ] || + error "error while apply client_cache_count" + + echo "Check the value is stored after remount" + stopall + setupall + wait_update $(facet_host ost1) "lctl get_param -n $CLIENT_PARAM | + head -1" $NEW_CLIENT_CACHE_COUNT + CLIENT_CACHE_COUNT=$(do_facet ost1 $LCTL get_param -n $CLIENT_PARAM | + head -1) + echo "$CLIENT_CACHE_COUNT" + [ $CLIENT_CACHE_COUNT = $NEW_CLIENT_CACHE_COUNT ] || + error "client_cache_count is not saved after remount" + stopall +} +run_test 76 "set permanent params set_param -P" + if ! combined_mgs_mds ; then stop mgs fi diff --git a/lustre/utils/lctl.c b/lustre/utils/lctl.c index 4363223..4a95edf 100644 --- a/lustre/utils/lctl.c +++ b/lustre/utils/lctl.c @@ -150,10 +150,13 @@ command_t cmdlist[] = { " -F When -N specified, add '/', '@' or '=' for directories,\n" " symlinks and writeable files, respectively."}, {"set_param", jt_lcfg_setparam, 0, "set the Lustre or LNET parameter\n" - "usage: set_param [-n] \n" - "Set the value of the Lustre or LNET parameter at the specified path\n" - " -n Disable printing of the key name when printing values."}, - {"list_param", jt_lcfg_listparam, 0, + "usage: set_param [-n] [-P] [-d]" + "\n" + "Set the value of the Lustre or LNET parameter at the specified path.\n" + " -n Disable printing of the key name when printing values.\n" + " -P Set the parameter permanently, filesystem-wide.\n" + " -d Remove the permanent setting (only with -P option).\n"}, + {"list_param", jt_lcfg_listparam, 0, "list the Lustre or LNET parameter name\n" "usage: list_param [-F|-R] \n" "List the name of Lustre or LNET parameter from the specified path.\n" diff --git a/lustre/utils/llog_reader.c b/lustre/utils/llog_reader.c index 596ab6e..3c994fc 100644 --- a/lustre/utils/llog_reader.c +++ b/lustre/utils/llog_reader.c @@ -361,11 +361,16 @@ void print_lustre_cfg(struct lustre_cfg *lcfg, int *skip) print_1_cfg(lcfg); break; } - case(LCFG_PARAM):{ - printf("param "); - print_1_cfg(lcfg); - break; - } + case(LCFG_PARAM):{ + printf("param "); + print_1_cfg(lcfg); + break; + } + case(LCFG_SET_PARAM):{ + printf("set_param "); + print_1_cfg(lcfg); + break; + } case(LCFG_SPTLRPC_CONF):{ printf("sptlrpc_conf "); print_1_cfg(lcfg); diff --git a/lustre/utils/lustre_cfg.c b/lustre/utils/lustre_cfg.c index 5b4c0ec..e709115 100644 --- a/lustre/utils/lustre_cfg.c +++ b/lustre/utils/lustre_cfg.c @@ -503,6 +503,83 @@ int jt_lcfg_param(int argc, char **argv) return rc; } +struct param_opts { + unsigned int po_only_path:1; + unsigned int po_show_path:1; + unsigned int po_show_type:1; + unsigned int po_recursive:1; + unsigned int po_params2:1; + unsigned int po_delete:1; +}; + +/* Param set to single log file, used by all clients and servers. + * This should be loaded after the individual config logs. + * Called from set param with -P option. + */ +static int jt_lcfg_mgsparam2(int argc, char **argv, struct param_opts *popt) +{ + int rc, i; + int first_param; + struct lustre_cfg_bufs bufs; + struct lustre_cfg *lcfg; + char *buf = NULL; + int len; + + first_param = optind; + if (first_param < 0 || first_param >= argc) + return CMD_HELP; + + for (i = first_param, rc = 0; i < argc; i++) { + lustre_cfg_bufs_reset(&bufs, NULL); + /* This same command would be executed on all nodes, many + * of which should fail (silently) because they don't have + * that proc file existing locally. There would be no + * preprocessing on the MGS to try to figure out which + * parameter files to add this to, there would be nodes + * processing on the cluster nodes to try to figure out + * if they are the intended targets. They will blindly + * try to set the parameter, and ENOTFOUND means it wasn't + * for them. + * Target name "general" means call on all targets. It is + * left here in case some filtering will be added in + * future. + */ + lustre_cfg_bufs_set_string(&bufs, 0, "general"); + + len = strlen(argv[i]); + + /* put an '=' on the end in case it doesn't have one */ + if (popt->po_delete && argv[i][len - 1] != '=') { + buf = malloc(len + 1); + sprintf(buf, "%s=", argv[i]); + } else { + buf = argv[i]; + } + lustre_cfg_bufs_set_string(&bufs, 1, buf); + + lcfg = lustre_cfg_new(LCFG_SET_PARAM, &bufs); + if (IS_ERR(lcfg)) { + fprintf(stderr, "error: allocating lcfg for %s: %s\n", + jt_cmdname(argv[0]), strerror(PTR_ERR(lcfg))); + if (rc == 0) + rc = PTR_ERR(lcfg); + } else { + int rc2 = lcfg_mgs_ioctl(argv[0], OBD_DEV_ID, lcfg); + if (rc2 != 0) { + fprintf(stderr, "error: executing %s: %s\n", + jt_cmdname(argv[0]), strerror(rc2)); + if (rc == 0) + rc = rc2; + } + lustre_cfg_free(lcfg); + } + if (buf != argv[i]) + free(buf); + } + + return rc; +} + /* Param set in config log on MGS */ /* conf_param key=value */ /* Note we can actually send mgc conf_params from clients, but currently @@ -565,12 +642,12 @@ int jt_lcfg_mgsparam(int argc, char **argv) /* Display the path in the same format as sysctl * For eg. obdfilter.lustre-OST0000.stats */ -static char *display_name(char *filename, int show_type) +static char *display_name(char *filename, unsigned int po_show_type) { char *tmp; struct stat st; - if (show_type) { + if (po_show_type) { if (lstat(filename, &st) < 0) return NULL; } @@ -592,7 +669,7 @@ static char *display_name(char *filename, int show_type) *tmp = '.'; /* append the indicator to entries */ - if (show_type) { + if (po_show_type) { if (S_ISDIR(st.st_mode)) strcat(filename, "/"); else if (S_ISLNK(st.st_mode)) @@ -683,29 +760,22 @@ static void lprocfs_param_pattern(const char *cmd, const char *path, char *buf, } } -struct param_opts { - int only_path:1; - int show_path:1; - int show_type:1; - int recursive:1; -}; - static int listparam_cmdline(int argc, char **argv, struct param_opts *popt) { int ch; - popt->show_path = 1; - popt->only_path = 1; - popt->show_type = 0; - popt->recursive = 0; + popt->po_show_path = 1; + popt->po_only_path = 1; + popt->po_show_type = 0; + popt->po_recursive = 0; while ((ch = getopt(argc, argv, "FR")) != -1) { switch (ch) { case 'F': - popt->show_type = 1; + popt->po_show_type = 1; break; case 'R': - popt->recursive = 1; + popt->po_recursive = 1; break; default: return -1; @@ -722,7 +792,7 @@ static int listparam_display(struct param_opts *popt, char *pattern) glob_t glob_info; char filename[PATH_MAX + 1]; /* extra 1 byte for file type */ - rc = glob(pattern, GLOB_BRACE | (popt->recursive ? GLOB_MARK : 0), + rc = glob(pattern, GLOB_BRACE | (popt->po_recursive ? GLOB_MARK : 0), NULL, &glob_info); if (rc) { fprintf(stderr, "error: list_param: %s: %s\n", @@ -743,7 +813,7 @@ static int listparam_display(struct param_opts *popt, char *pattern) else last = 0; strcpy(filename, glob_info.gl_pathv[i]); - valuename = display_name(filename, popt->show_type); + valuename = display_name(filename, popt->po_show_type); if (valuename) printf("%s\n", valuename); if (last) { @@ -765,7 +835,7 @@ int jt_lcfg_listparam(int argc, char **argv) char *path; rc = listparam_cmdline(argc, argv, &popt); - if (rc == argc && popt.recursive) { + if (rc == argc && popt.po_recursive) { rc--; /* we know at least "-R" is a parameter */ argv[rc] = "*"; } else if (rc < 0 || rc >= argc) { @@ -791,20 +861,20 @@ static int getparam_cmdline(int argc, char **argv, struct param_opts *popt) { int ch; - popt->show_path = 1; - popt->only_path = 0; - popt->show_type = 0; - popt->recursive = 0; + popt->po_show_path = 1; + popt->po_only_path = 0; + popt->po_show_type = 0; + popt->po_recursive = 0; while ((ch = getopt(argc, argv, "nNF")) != -1) { switch (ch) { case 'N': - popt->only_path = 1; + popt->po_only_path = 1; break; case 'n': - popt->show_path = 0; + popt->po_show_path = 0; case 'F': - popt->show_type = 1; + popt->po_show_type = 1; break; default: return -1; @@ -837,7 +907,7 @@ static int getparam_display(struct param_opts *popt, char *pattern) memset(buf, 0, PAGE_CACHE_SIZE); /* As listparam_display is used to show param name (with type), * here "if (only_path)" is ignored.*/ - if (popt->show_path) { + if (popt->po_show_path) { strcpy(filename, glob_info.gl_pathv[i]); valuename = display_name(filename, 0); } @@ -864,7 +934,7 @@ static int getparam_display(struct param_opts *popt, char *pattern) /* Print the output in the format path=value if the * value contains no new line character or cab be * occupied in a line, else print value on new line */ - if (valuename && popt->show_path) { + if (valuename && popt->po_show_path) { int longbuf = strnchr(buf, rc - 1, '\n') != NULL || rc > 60; printf("%s=%s", valuename, longbuf ? "\n" : buf); @@ -909,7 +979,7 @@ int jt_lcfg_getparam(int argc, char **argv) lprocfs_param_pattern(argv[0], path, pattern, sizeof(pattern)); - if (popt.only_path) + if (popt.po_only_path) rc2 = listparam_display(&popt, pattern); else rc2 = getparam_display(&popt, pattern); @@ -924,16 +994,24 @@ static int setparam_cmdline(int argc, char **argv, struct param_opts *popt) { int ch; - popt->show_path = 1; - popt->only_path = 0; - popt->show_type = 0; - popt->recursive = 0; + popt->po_show_path = 1; + popt->po_only_path = 0; + popt->po_show_type = 0; + popt->po_recursive = 0; + popt->po_params2 = 0; + popt->po_delete = 0; - while ((ch = getopt(argc, argv, "n")) != -1) { + while ((ch = getopt(argc, argv, "nPd")) != -1) { switch (ch) { case 'n': - popt->show_path = 0; + popt->po_show_path = 0; break; + case 'P': + popt->po_params2 = 1; + break; + case 'd': + popt->po_delete = 1; + break; default: return -1; } @@ -958,7 +1036,7 @@ static int setparam_display(struct param_opts *popt, char *pattern, char *value) for (i = 0; i < glob_info.gl_pathc; i++) { char *valuename = NULL; - if (popt->show_path) { + if (popt->po_show_path) { strcpy(filename, glob_info.gl_pathv[i]); valuename = display_name(filename, 0); if (valuename) @@ -966,7 +1044,7 @@ static int setparam_display(struct param_opts *popt, char *pattern, char *value) } /* Write the new value to the file */ fd = open(glob_info.gl_pathv[i], O_WRONLY); - if (fd > 0) { + if (fd >= 0) { rc = write(fd, value, strlen(value)); if (rc < 0) fprintf(stderr, "error: set_param: setting " @@ -996,6 +1074,11 @@ int jt_lcfg_setparam(int argc, char **argv) if (rc < 0 || rc >= argc) return CMD_HELP; + if (popt.po_params2) + /* We can't delete parameters that were + * set with old conf_param interface */ + return jt_lcfg_mgsparam2(argc, argv, &popt); + for (i = rc, rc = 0; i < argc; i++) { int rc2; -- 1.8.3.1