X-Git-Url: https://git.whamcloud.com/?p=fs%2Flustre-release.git;a=blobdiff_plain;f=lustre%2Fmgc%2Fmgc_request.c;h=4a982263d0c875fb4cc41e84df6ea18b538f0132;hp=0d568d38557abce5987c9c4b165ac3c641c7e6af;hb=e34950d29f1168a8a85f26e862eaec888b94350e;hpb=ffe234a077576f0f66f6fe9c165be180133be89a diff --git a/lustre/mgc/mgc_request.c b/lustre/mgc/mgc_request.c index 0d568d3..4a98226 100644 --- a/lustre/mgc/mgc_request.c +++ b/lustre/mgc/mgc_request.c @@ -15,11 +15,7 @@ * * You should have received a copy of the GNU General Public License * version 2 along with this program; If not, see - * http://www.sun.com/software/products/lustre/docs/GPLv2.pdf - * - * Please contact Sun Microsystems, Inc., 4150 Network Circle, Santa Clara, - * CA 95054 USA or visit www.sun.com if you need additional information or - * have any questions. + * http://www.gnu.org/licenses/gpl-2.0.html * * GPL HEADER END */ @@ -27,11 +23,10 @@ * Copyright (c) 2007, 2010, Oracle and/or its affiliates. All rights reserved. * Use is subject to license terms. * - * Copyright (c) 2011, 2015, Intel Corporation. + * Copyright (c) 2011, 2017, Intel Corporation. */ /* * This file is part of Lustre, http://www.lustre.org/ - * Lustre is a trademark of Sun Microsystems, Inc. * * lustre/mgc/mgc_request.c * @@ -43,6 +38,7 @@ #include #include +#include #include #include @@ -52,11 +48,12 @@ #include #include #include +#include #include "mgc_internal.h" static int mgc_name2resid(char *name, int len, struct ldlm_res_id *res_id, - int type) + enum mgs_cfg_type type) { __u64 resname = 0; @@ -75,25 +72,27 @@ static int mgc_name2resid(char *name, int len, struct ldlm_res_id *res_id, res_id->name[0] = cpu_to_le64(resname); /* XXX: unfortunately, sptlprc and config llog share one lock */ switch(type) { - case CONFIG_T_CONFIG: - case CONFIG_T_SPTLRPC: + case MGS_CFG_T_CONFIG: + case MGS_CFG_T_SPTLRPC: resname = 0; break; - case CONFIG_T_RECOVER: - case CONFIG_T_PARAMS: - case CONFIG_T_NODEMAP: + case MGS_CFG_T_RECOVER: + case MGS_CFG_T_PARAMS: + case MGS_CFG_T_NODEMAP: + case MGS_CFG_T_BARRIER: resname = type; break; default: LBUG(); } res_id->name[1] = cpu_to_le64(resname); - CDEBUG(D_MGC, "log %s to resid "LPX64"/"LPX64" (%.8s)\n", name, + CDEBUG(D_MGC, "log %s to resid %#llx/%#llx (%.8s)\n", name, res_id->name[0], res_id->name[1], (char *)&res_id->name[0]); return 0; } -int mgc_fsname2resid(char *fsname, struct ldlm_res_id *res_id, int type) +int mgc_fsname2resid(char *fsname, struct ldlm_res_id *res_id, + enum mgs_cfg_type type) { /* fsname is at most 8 chars long, maybe contain "-". * e.g. "lustre", "SUN-000" */ @@ -101,8 +100,8 @@ int mgc_fsname2resid(char *fsname, struct ldlm_res_id *res_id, int type) } EXPORT_SYMBOL(mgc_fsname2resid); -static int mgc_logname2resid(char *logname, struct ldlm_res_id *res_id, - int type) +int mgc_logname2resid(char *logname, struct ldlm_res_id *res_id, + enum mgs_cfg_type type) { char *name_end; int len; @@ -117,17 +116,18 @@ static int mgc_logname2resid(char *logname, struct ldlm_res_id *res_id, len = name_end - logname; return mgc_name2resid(logname, len, res_id, type); } +EXPORT_SYMBOL(mgc_logname2resid); /********************** config llog list **********************/ -static struct list_head config_llog_list = LIST_HEAD_INIT(config_llog_list); -static DEFINE_SPINLOCK(config_list_lock); +static LIST_HEAD(config_llog_list); +static DEFINE_SPINLOCK(config_list_lock); /* protects config_llog_list */ /* Take a reference to a config log */ static int config_log_get(struct config_llog_data *cld) { ENTRY; atomic_inc(&cld->cld_refcount); - CDEBUG(D_INFO, "log %s refs %d\n", cld->cld_logname, + CDEBUG(D_INFO, "log %s (%p) refs %d\n", cld->cld_logname, cld, atomic_read(&cld->cld_refcount)); RETURN(0); } @@ -138,7 +138,10 @@ static void config_log_put(struct config_llog_data *cld) { ENTRY; - CDEBUG(D_INFO, "log %s refs %d\n", cld->cld_logname, + if (unlikely(!cld)) + RETURN_EXIT; + + CDEBUG(D_INFO, "log %s(%p) refs %d\n", cld->cld_logname, cld, atomic_read(&cld->cld_refcount)); LASSERT(atomic_read(&cld->cld_refcount) > 0); @@ -149,16 +152,15 @@ static void config_log_put(struct config_llog_data *cld) CDEBUG(D_MGC, "dropping config log %s\n", cld->cld_logname); - if (cld->cld_recover) - config_log_put(cld->cld_recover); - if (cld->cld_sptlrpc) - config_log_put(cld->cld_sptlrpc); - if (cld->cld_params) - config_log_put(cld->cld_params); - if (cld->cld_nodemap) - config_log_put(cld->cld_nodemap); - if (cld_is_sptlrpc(cld)) + config_log_put(cld->cld_barrier); + config_log_put(cld->cld_recover); + config_log_put(cld->cld_params); + config_log_put(cld->cld_nodemap); + config_log_put(cld->cld_sptlrpc); + if (cld_is_sptlrpc(cld)) { + cld->cld_stopping = 1; sptlrpc_conf_log_stop(cld->cld_logname); + } class_export_put(cld->cld_mgcexp); OBD_FREE(cld, sizeof(*cld) + strlen(cld->cld_logname) + 1); @@ -172,30 +174,27 @@ static struct config_llog_data *config_log_find(char *logname, struct config_llog_instance *cfg) { - struct config_llog_data *cld; - struct config_llog_data *found = NULL; - void * instance; - ENTRY; + struct config_llog_data *cld; + struct config_llog_data *found = NULL; + unsigned long cfg_instance; - LASSERT(logname != NULL); + ENTRY; + LASSERT(logname != NULL); - instance = cfg ? cfg->cfg_instance : NULL; + cfg_instance = cfg ? cfg->cfg_instance : 0; spin_lock(&config_list_lock); list_for_each_entry(cld, &config_llog_list, cld_list_chain) { - /* check if instance equals */ - if (instance != cld->cld_cfg.cfg_instance) - continue; + /* check if cfg_instance is the one we want */ + if (cfg_instance != cld->cld_cfg.cfg_instance) + continue; /* instance may be NULL, should check name */ if (strcmp(logname, cld->cld_logname) == 0) { found = cld; + config_log_get(found); break; } } - if (found) { - atomic_inc(&found->cld_refcount); - LASSERT(found->cld_stopping == 0 || cld_is_sptlrpc(found) == 0); - } spin_unlock(&config_list_lock); RETURN(found); } @@ -203,7 +202,7 @@ struct config_llog_data *config_log_find(char *logname, static struct config_llog_data *do_config_log_add(struct obd_device *obd, char *logname, - int type, + enum mgs_cfg_type type, struct config_llog_instance *cfg, struct super_block *sb) { @@ -212,13 +211,19 @@ struct config_llog_data *do_config_log_add(struct obd_device *obd, ENTRY; - CDEBUG(D_MGC, "do adding config log %s:%p\n", logname, - cfg ? cfg->cfg_instance : NULL); + CDEBUG(D_MGC, "do adding config log %s-%016lx\n", logname, + cfg ? cfg->cfg_instance : 0); OBD_ALLOC(cld, sizeof(*cld) + strlen(logname) + 1); if (!cld) RETURN(ERR_PTR(-ENOMEM)); + rc = mgc_logname2resid(logname, &cld->cld_resid, type); + if (rc) { + OBD_FREE(cld, sizeof(*cld) + strlen(cld->cld_logname) + 1); + RETURN(ERR_PTR(rc)); + } + strcpy(cld->cld_logname, logname); if (cfg) cld->cld_cfg = *cfg; @@ -234,23 +239,14 @@ struct config_llog_data *do_config_log_add(struct obd_device *obd, /* Keep the mgc around until we are done */ cld->cld_mgcexp = class_export_get(obd->obd_self_export); - if (cld_is_sptlrpc(cld)) { + if (cld_is_sptlrpc(cld)) sptlrpc_conf_log_start(logname); - cld->cld_cfg.cfg_obdname = obd->obd_name; - } - - rc = mgc_logname2resid(logname, &cld->cld_resid, type); spin_lock(&config_list_lock); list_add(&cld->cld_list_chain, &config_llog_list); spin_unlock(&config_list_lock); - if (rc) { - config_log_put(cld); - RETURN(ERR_PTR(rc)); - } - - if (cld_is_sptlrpc(cld) || cld_is_nodemap(cld)) { + if (cld_is_sptlrpc(cld) || cld_is_nodemap(cld) || cld_is_barrier(cld)) { rc = mgc_process_log(obd, cld); if (rc && rc != -ENOENT) CERROR("%s: failed processing log, type %d: rc = %d\n", @@ -261,51 +257,55 @@ struct config_llog_data *do_config_log_add(struct obd_device *obd, } static struct config_llog_data *config_recover_log_add(struct obd_device *obd, - char *fsname, - struct config_llog_instance *cfg, - struct super_block *sb) + char *fsname, + struct config_llog_instance *cfg, + struct super_block *sb) { - struct config_llog_instance lcfg = *cfg; - struct lustre_sb_info *lsi = s2lsi(sb); - struct config_llog_data *cld; - char logname[32]; + struct config_llog_instance lcfg = *cfg; + struct config_llog_data *cld; + char logname[32]; - if (IS_OST(lsi)) - return NULL; + if (IS_OST(s2lsi(sb))) + return NULL; /* for osp-on-ost, see lustre_start_osp() */ - if (IS_MDT(lsi) && lcfg.cfg_instance) + if (IS_MDT(s2lsi(sb)) && lcfg.cfg_instance) return NULL; - /* we have to use different llog for clients and mdts for cmd - * where only clients are notified if one of cmd server restarts */ - LASSERT(strlen(fsname) < sizeof(logname) / 2); - strcpy(logname, fsname); - if (IS_SERVER(lsi)) { /* mdt */ - LASSERT(lcfg.cfg_instance == NULL); - lcfg.cfg_instance = sb; - strcat(logname, "-mdtir"); - } else { - LASSERT(lcfg.cfg_instance != NULL); - strcat(logname, "-cliir"); - } + /* We have to use different llog for clients and MDTs for DNE, + * where only clients are notified if one of DNE server restarts. + */ + LASSERT(strlen(fsname) < sizeof(logname) / 2); + strncpy(logname, fsname, sizeof(logname)); + if (IS_SERVER(s2lsi(sb))) { /* mdt */ + LASSERT(lcfg.cfg_instance == 0); + lcfg.cfg_instance = ll_get_cfg_instance(sb); + strncat(logname, "-mdtir", sizeof(logname)); + } else { + LASSERT(lcfg.cfg_instance != 0); + strncat(logname, "-cliir", sizeof(logname)); + } - cld = do_config_log_add(obd, logname, CONFIG_T_RECOVER, &lcfg, sb); - return cld; + cld = do_config_log_add(obd, logname, MGS_CFG_T_RECOVER, &lcfg, sb); + return cld; } -static struct config_llog_data *config_params_log_add(struct obd_device *obd, - struct config_llog_instance *cfg, struct super_block *sb) +static struct config_llog_data * +config_log_find_or_add(struct obd_device *obd, char *logname, + struct super_block *sb, enum mgs_cfg_type type, + struct config_llog_instance *cfg) { - struct config_llog_instance lcfg = *cfg; - struct config_llog_data *cld; + struct config_llog_instance lcfg = *cfg; + struct config_llog_data *cld; - lcfg.cfg_instance = sb; + /* Note class_config_llog_handler() depends on getting "obd" back */ + lcfg.cfg_instance = sb ? ll_get_cfg_instance(sb) : (unsigned long)obd; - cld = do_config_log_add(obd, PARAMS_FILENAME, CONFIG_T_PARAMS, - &lcfg, sb); + cld = config_log_find(logname, &lcfg); + if (unlikely(cld != NULL)) + return cld; - return cld; + return do_config_log_add(obd, logname, type, &lcfg, sb); } /** Add this log to the list of active logs watched by an MGC. @@ -313,21 +313,24 @@ static struct config_llog_data *config_params_log_add(struct obd_device *obd, * We have one active log per "mount" - client instance or servername. * Each instance may be at a different point in the log. */ -static int config_log_add(struct obd_device *obd, char *logname, - struct config_llog_instance *cfg, - struct super_block *sb) +static struct config_llog_data * +config_log_add(struct obd_device *obd, char *logname, + struct config_llog_instance *cfg, struct super_block *sb) { - struct lustre_sb_info *lsi = s2lsi(sb); - struct config_llog_data *cld; - struct config_llog_data *sptlrpc_cld; - struct config_llog_data *params_cld; - struct config_llog_data *nodemap_cld; - char seclogname[32]; - char *ptr; - int rc; + struct lustre_sb_info *lsi = s2lsi(sb); + struct config_llog_data *cld = NULL; + struct config_llog_data *sptlrpc_cld = NULL; + struct config_llog_data *params_cld = NULL; + struct config_llog_data *nodemap_cld = NULL; + struct config_llog_data *barrier_cld = NULL; + char seclogname[32]; + char *ptr; + int rc; + bool locked = false; ENTRY; - CDEBUG(D_MGC, "adding config log %s:%p\n", logname, cfg->cfg_instance); + CDEBUG(D_MGC, "add config log %s-%016lx\n", logname, + cfg->cfg_instance); /* * for each regular log, the depended sptlrpc log name is @@ -336,26 +339,26 @@ static int config_log_add(struct obd_device *obd, char *logname, ptr = strrchr(logname, '-'); if (ptr == NULL || ptr - logname > 8) { CERROR("logname %s is too long\n", logname); - RETURN(-EINVAL); + RETURN(ERR_PTR(-EINVAL)); } memcpy(seclogname, logname, ptr - logname); strcpy(seclogname + (ptr - logname), "-sptlrpc"); - sptlrpc_cld = config_log_find(seclogname, NULL); - if (sptlrpc_cld == NULL) { - sptlrpc_cld = do_config_log_add(obd, seclogname, - CONFIG_T_SPTLRPC, NULL, NULL); + if (cfg->cfg_sub_clds & CONFIG_SUB_SPTLRPC) { + sptlrpc_cld = config_log_find_or_add(obd, seclogname, NULL, + MGS_CFG_T_SPTLRPC, cfg); if (IS_ERR(sptlrpc_cld)) { - CERROR("can't create sptlrpc log: %s\n", seclogname); - GOTO(out, rc = PTR_ERR(sptlrpc_cld)); + CERROR("%s: can't create sptlrpc log %s: rc = %ld\n", + obd->obd_name, seclogname, PTR_ERR(sptlrpc_cld)); + RETURN(sptlrpc_cld); } } - nodemap_cld = config_log_find(LUSTRE_NODEMAP_NAME, NULL); - if (!nodemap_cld && IS_SERVER(lsi) && !IS_MGS(lsi)) { - nodemap_cld = do_config_log_add(obd, LUSTRE_NODEMAP_NAME, - CONFIG_T_NODEMAP, NULL, NULL); + if (!IS_MGS(lsi) && cfg->cfg_sub_clds & CONFIG_SUB_NODEMAP) { + nodemap_cld = config_log_find_or_add(obd, LUSTRE_NODEMAP_NAME, + NULL, MGS_CFG_T_NODEMAP, + cfg); if (IS_ERR(nodemap_cld)) { rc = PTR_ERR(nodemap_cld); CERROR("%s: cannot create nodemap log: rc = %d\n", @@ -364,63 +367,113 @@ static int config_log_add(struct obd_device *obd, char *logname, } } - params_cld = config_params_log_add(obd, cfg, sb); - if (IS_ERR(params_cld)) { - rc = PTR_ERR(params_cld); - CERROR("%s: can't create params log: rc = %d\n", - obd->obd_name, rc); - GOTO(out_nodemap, rc); + if (cfg->cfg_sub_clds & CONFIG_SUB_PARAMS) { + params_cld = config_log_find_or_add(obd, PARAMS_FILENAME, sb, + MGS_CFG_T_PARAMS, cfg); + if (IS_ERR(params_cld)) { + rc = PTR_ERR(params_cld); + CERROR("%s: can't create params log: rc = %d\n", + obd->obd_name, rc); + GOTO(out_nodemap, rc); + } } - cld = do_config_log_add(obd, logname, CONFIG_T_CONFIG, cfg, sb); + if (IS_MDT(s2lsi(sb)) && cfg->cfg_sub_clds & CONFIG_SUB_BARRIER) { + snprintf(seclogname + (ptr - logname), sizeof(seclogname) - 1, + "-%s", BARRIER_FILENAME); + barrier_cld = config_log_find_or_add(obd, seclogname, sb, + MGS_CFG_T_BARRIER, cfg); + if (IS_ERR(barrier_cld)) { + rc = PTR_ERR(barrier_cld); + CERROR("%s: can't create barrier log: rc = %d\n", + obd->obd_name, rc); + GOTO(out_params, rc); + } + } + + cld = do_config_log_add(obd, logname, MGS_CFG_T_CONFIG, cfg, sb); if (IS_ERR(cld)) { - CERROR("can't create log: %s\n", logname); - GOTO(out_params, rc = PTR_ERR(cld)); + rc = PTR_ERR(cld); + CERROR("%s: can't create log: rc = %d\n", + obd->obd_name, rc); + GOTO(out_barrier, rc = PTR_ERR(cld)); } LASSERT(lsi->lsi_lmd); - if (!(lsi->lsi_lmd->lmd_flags & LMD_FLG_NOIR)) { + if (!(lsi->lsi_lmd->lmd_flags & LMD_FLG_NOIR) && + cfg->cfg_sub_clds & CONFIG_SUB_RECOVER) { struct config_llog_data *recover_cld; + ptr = strrchr(seclogname, '-'); if (ptr != NULL) { *ptr = 0; - } - else { + } else { CERROR("%s: sptlrpc log name not correct, %s: " "rc = %d\n", obd->obd_name, seclogname, -EINVAL); - config_log_put(cld); - RETURN(-EINVAL); + GOTO(out_cld, rc = -EINVAL); } + recover_cld = config_recover_log_add(obd, seclogname, cfg, sb); - if (IS_ERR(recover_cld)) - GOTO(out_cld, rc = PTR_ERR(recover_cld)); + if (IS_ERR(recover_cld)) { + rc = PTR_ERR(recover_cld); + CERROR("%s: can't create recover log: rc = %d\n", + obd->obd_name, rc); + GOTO(out_cld, rc); + } + + mutex_lock(&cld->cld_lock); + locked = true; cld->cld_recover = recover_cld; } - cld->cld_sptlrpc = sptlrpc_cld; + if (!locked) + mutex_lock(&cld->cld_lock); cld->cld_params = params_cld; + cld->cld_barrier = barrier_cld; cld->cld_nodemap = nodemap_cld; + cld->cld_sptlrpc = sptlrpc_cld; + mutex_unlock(&cld->cld_lock); - RETURN(0); + RETURN(cld); out_cld: config_log_put(cld); - +out_barrier: + config_log_put(barrier_cld); out_params: config_log_put(params_cld); - out_nodemap: config_log_put(nodemap_cld); - out_sptlrpc: config_log_put(sptlrpc_cld); -out: - return rc; + return ERR_PTR(rc); } DEFINE_MUTEX(llog_process_lock); +static inline void config_mark_cld_stop_nolock(struct config_llog_data *cld) +{ + ENTRY; + + spin_lock(&config_list_lock); + cld->cld_stopping = 1; + spin_unlock(&config_list_lock); + + CDEBUG(D_INFO, "lockh %#llx\n", cld->cld_lockh.cookie); + if (!ldlm_lock_addref_try(&cld->cld_lockh, LCK_CR)) + ldlm_lock_decref_and_cancel(&cld->cld_lockh, LCK_CR); +} + +static inline void config_mark_cld_stop(struct config_llog_data *cld) +{ + if (cld) { + mutex_lock(&cld->cld_lock); + config_mark_cld_stop_nolock(cld); + mutex_unlock(&cld->cld_lock); + } +} + /** Stop watching for updates on this log. */ static int config_log_end(char *logname, struct config_llog_instance *cfg) @@ -430,6 +483,7 @@ static int config_log_end(char *logname, struct config_llog_instance *cfg) struct config_llog_data *cld_params = NULL; struct config_llog_data *cld_recover = NULL; struct config_llog_data *cld_nodemap = NULL; + struct config_llog_data *cld_barrier = NULL; int rc = 0; ENTRY; @@ -453,44 +507,32 @@ static int config_log_end(char *logname, struct config_llog_instance *cfg) RETURN(rc); } - cld->cld_stopping = 1; - cld_recover = cld->cld_recover; cld->cld_recover = NULL; - mutex_unlock(&cld->cld_lock); - - if (cld_recover) { - mutex_lock(&cld_recover->cld_lock); - cld_recover->cld_stopping = 1; - mutex_unlock(&cld_recover->cld_lock); - config_log_put(cld_recover); - } - - spin_lock(&config_list_lock); - cld_sptlrpc = cld->cld_sptlrpc; - cld->cld_sptlrpc = NULL; cld_params = cld->cld_params; cld->cld_params = NULL; cld_nodemap = cld->cld_nodemap; cld->cld_nodemap = NULL; - spin_unlock(&config_list_lock); - - if (cld_sptlrpc) - config_log_put(cld_sptlrpc); + cld_barrier = cld->cld_barrier; + cld->cld_barrier = NULL; + cld_sptlrpc = cld->cld_sptlrpc; + cld->cld_sptlrpc = NULL; - if (cld_params) { - mutex_lock(&cld_params->cld_lock); - cld_params->cld_stopping = 1; - mutex_unlock(&cld_params->cld_lock); - config_log_put(cld_params); - } + config_mark_cld_stop_nolock(cld); + mutex_unlock(&cld->cld_lock); - if (cld_nodemap) { - mutex_lock(&cld_nodemap->cld_lock); - cld_nodemap->cld_stopping = 1; - mutex_unlock(&cld_nodemap->cld_lock); - config_log_put(cld_nodemap); - } + config_mark_cld_stop(cld_recover); + config_log_put(cld_recover); + config_mark_cld_stop(cld_params); + config_log_put(cld_params); + config_mark_cld_stop(cld_barrier); + config_log_put(cld_barrier); + /* don't explicitly set cld_stopping on sptlrpc lock here, as other + * targets may be active, it will be done in config_log_put if necessary + */ + config_log_put(cld_sptlrpc); + /* don't set cld_stopping on nm lock as other targets may be active */ + config_log_put(cld_nodemap); /* drop the ref from the find */ config_log_put(cld); @@ -502,22 +544,26 @@ static int config_log_end(char *logname, struct config_llog_instance *cfg) RETURN(rc); } -#ifdef CONFIG_PROC_FS int lprocfs_mgc_rd_ir_state(struct seq_file *m, void *data) { struct obd_device *obd = data; struct obd_import *imp; struct obd_connect_data *ocd; struct config_llog_data *cld; + int rc = 0; + ENTRY; + LASSERT(obd); + with_imp_locked(obd, imp, rc) { + ocd = &imp->imp_connect_data; - LASSERT(obd != NULL); - LPROCFS_CLIMP_CHECK(obd); - imp = obd->u.cli.cl_import; - ocd = &imp->imp_connect_data; + seq_printf(m, "imperative_recovery: %s\n", + OCD_HAS_FLAG(ocd, IMP_RECOV) ? + "ENABLED" : "DISABLED"); + } + if (rc) + RETURN(rc); - seq_printf(m, "imperative_recovery: %s\n", - OCD_HAS_FLAG(ocd, IMP_RECOV) ? "ENABLED" : "DISABLED"); seq_printf(m, "client_state:\n"); spin_lock(&config_list_lock); @@ -530,10 +576,8 @@ int lprocfs_mgc_rd_ir_state(struct seq_file *m, void *data) } spin_unlock(&config_list_lock); - LPROCFS_CLIMP_EXIT(obd); RETURN(0); } -#endif /* reenqueue any lost locks */ #define RQ_RUNNING 0x1 @@ -553,10 +597,13 @@ static void do_requeue(struct config_llog_data *cld) LASSERT(atomic_read(&cld->cld_refcount) > 0); - /* Do not run mgc_process_log on a disconnected export or an + /* + * Do not run mgc_process_log on a disconnected export or an * export which is being disconnected. Take the client - * semaphore to make the check non-racy. */ - down_read(&cld->cld_mgcexp->exp_obd->u.cli.cl_sem); + * semaphore to make the check non-racy. + */ + down_read_nested(&cld->cld_mgcexp->exp_obd->u.cli.cl_sem, + OBD_CLI_SEM_MGC); if (cld->cld_mgcexp->exp_obd->u.cli.cl_conn_count != 0) { CDEBUG(D_MGC, "updating log %s\n", cld->cld_logname); rc = mgc_process_log(cld->cld_mgcexp->exp_obd, cld); @@ -571,13 +618,6 @@ static void do_requeue(struct config_llog_data *cld) EXIT; } -/* this timeout represents how many seconds MGC should wait before - * requeue config and recover lock to the MGS. We need to randomize this - * in order to not flood the MGS. - */ -#define MGC_TIMEOUT_MIN_SECONDS 5 -#define MGC_TIMEOUT_RAND_CENTISEC 0x1ff /* ~500 */ - static int mgc_requeue_thread(void *data) { int rc = 0; @@ -589,11 +629,8 @@ static int mgc_requeue_thread(void *data) /* Keep trying failed locks periodically */ spin_lock(&config_list_lock); rq_state |= RQ_RUNNING; - while (1) { - struct l_wait_info lwi; + while (!(rq_state & RQ_STOP)) { struct config_llog_data *cld, *cld_prev; - int rand = cfs_rand() & MGC_TIMEOUT_RAND_CENTISEC; - int stopped = !!(rq_state & RQ_STOP); int to; /* Any new or requeued lostlocks will change the state */ @@ -606,65 +643,57 @@ static int mgc_requeue_thread(void *data) } /* Always wait a few seconds to allow the server who - caused the lock revocation to finish its setup, plus some - random so everyone doesn't try to reconnect at once. */ - to = msecs_to_jiffies(MGC_TIMEOUT_MIN_SECONDS * MSEC_PER_SEC); - /* rand is centi-seconds */ - to += msecs_to_jiffies(rand * MSEC_PER_SEC / 100); - lwi = LWI_TIMEOUT(to, NULL, NULL); - l_wait_event(rq_waitq, rq_state & (RQ_STOP | RQ_PRECLEANUP), - &lwi); - - /* - * iterate & processing through the list. for each cld, process - * its depending sptlrpc cld firstly (if any) and then itself. - * - * it's guaranteed any item in the list must have - * reference > 0; and if cld_lostlock is set, at - * least one reference is taken by the previous enqueue. - */ - cld_prev = NULL; + * caused the lock revocation to finish its setup, plus some + * random so everyone doesn't try to reconnect at once. + */ + to = mgc_requeue_timeout_min == 0 ? 1 : mgc_requeue_timeout_min; + to = cfs_time_seconds(mgc_requeue_timeout_min) + + prandom_u32_max(cfs_time_seconds(to)); + wait_event_idle_timeout(rq_waitq, + rq_state & (RQ_STOP | RQ_PRECLEANUP), to); + + /* + * iterate & processing through the list. for each cld, process + * its depending sptlrpc cld firstly (if any) and then itself. + * + * it's guaranteed any item in the list must have + * reference > 0; and if cld_lostlock is set, at + * least one reference is taken by the previous enqueue. + */ + cld_prev = NULL; spin_lock(&config_list_lock); rq_state &= ~RQ_PRECLEANUP; list_for_each_entry(cld, &config_llog_list, - cld_list_chain) { - if (!cld->cld_lostlock) + cld_list_chain) { + if (!cld->cld_lostlock || cld->cld_stopping) continue; + /* hold reference to avoid being freed during + * subsequent processing. */ + config_log_get(cld); + cld->cld_lostlock = 0; spin_unlock(&config_list_lock); - LASSERT(atomic_read(&cld->cld_refcount) > 0); - - /* Whether we enqueued again or not in mgc_process_log, - * we're done with the ref from the old enqueue */ - if (cld_prev) - config_log_put(cld_prev); + config_log_put(cld_prev); cld_prev = cld; - cld->cld_lostlock = 0; - if (likely(!stopped)) + if (likely(!(rq_state & RQ_STOP))) { do_requeue(cld); - - spin_lock(&config_list_lock); + spin_lock(&config_list_lock); + } else { + spin_lock(&config_list_lock); + break; + } } spin_unlock(&config_list_lock); - if (cld_prev) - config_log_put(cld_prev); - - /* break after scanning the list so that we can drop - * refcount to losing lock clds */ - if (unlikely(stopped)) { - spin_lock(&config_list_lock); - break; - } + config_log_put(cld_prev); /* Wait a bit to see if anyone else needs a requeue */ - lwi = (struct l_wait_info) { 0 }; - l_wait_event(rq_waitq, rq_state & (RQ_NOW | RQ_STOP), - &lwi); + wait_event_idle(rq_waitq, rq_state & (RQ_NOW | RQ_STOP)); spin_lock(&config_list_lock); } + /* spinlock and while guarantee RQ_NOW and RQ_LATER are not set */ rq_state &= ~RQ_RUNNING; spin_unlock(&config_list_lock); @@ -679,6 +708,7 @@ static int mgc_requeue_thread(void *data) We are responsible for dropping the config log reference from here on out. */ static void mgc_requeue_add(struct config_llog_data *cld) { + bool wakeup = false; ENTRY; CDEBUG(D_INFO, "log %s: requeue (r=%d sp=%d st=%x)\n", @@ -686,31 +716,28 @@ static void mgc_requeue_add(struct config_llog_data *cld) cld->cld_stopping, rq_state); LASSERT(atomic_read(&cld->cld_refcount) > 0); - mutex_lock(&cld->cld_lock); - if (cld->cld_stopping || cld->cld_lostlock) { - mutex_unlock(&cld->cld_lock); - RETURN_EXIT; - } - /* this refcount will be released in mgc_requeue_thread. */ - config_log_get(cld); - cld->cld_lostlock = 1; - mutex_unlock(&cld->cld_lock); + /* lets cancel an existent lock to mark cld as "lostlock" */ + CDEBUG(D_INFO, "lockh %#llx\n", cld->cld_lockh.cookie); + if (!ldlm_lock_addref_try(&cld->cld_lockh, LCK_CR)) + ldlm_lock_decref_and_cancel(&cld->cld_lockh, LCK_CR); - /* Hold lock for rq_state */ + mutex_lock(&cld->cld_lock); spin_lock(&config_list_lock); - if (rq_state & RQ_STOP) { - spin_unlock(&config_list_lock); - cld->cld_lostlock = 0; - config_log_put(cld); - } else { + if (!(rq_state & RQ_STOP) && !cld->cld_stopping) { + cld->cld_lostlock = 1; rq_state |= RQ_NOW; - spin_unlock(&config_list_lock); - wake_up(&rq_waitq); + wakeup = true; } + spin_unlock(&config_list_lock); + mutex_unlock(&cld->cld_lock); + if (wakeup) + wake_up(&rq_waitq); + EXIT; } /********************** class fns **********************/ +#ifdef HAVE_SERVER_SUPPORT static int mgc_local_llog_init(const struct lu_env *env, struct obd_device *obd, struct obd_device *disk) @@ -770,7 +797,7 @@ static int mgc_fs_setup(const struct lu_env *env, struct obd_device *obd, rc = local_oid_storage_init(env, lsi->lsi_dt_dev, &fid, &cli->cl_mgc_los); if (rc) - RETURN(rc); + GOTO(out_mutex, rc); rc = dt_root_get(env, lsi->lsi_dt_dev, &rfid); if (rc) @@ -784,7 +811,7 @@ static int mgc_fs_setup(const struct lu_env *env, struct obd_device *obd, dto = local_file_find_or_create(env, cli->cl_mgc_los, root, MOUNT_CONFIGS_DIR, S_IFDIR | S_IRUGO | S_IWUSR | S_IXUGO); - lu_object_put_nocache(env, &root->do_lu); + dt_object_put_nocache(env, root); if (IS_ERR(dto)) GOTO(out_los, rc = PTR_ERR(dto)); @@ -803,12 +830,13 @@ static int mgc_fs_setup(const struct lu_env *env, struct obd_device *obd, EXIT; out_llog: if (rc) { - lu_object_put(env, &cli->cl_mgc_configs_dir->do_lu); + dt_object_put(env, cli->cl_mgc_configs_dir); cli->cl_mgc_configs_dir = NULL; } out_los: if (rc < 0) { local_oid_storage_fini(env, cli->cl_mgc_los); +out_mutex: cli->cl_mgc_los = NULL; mutex_unlock(&cli->cl_mgc_mutex); } @@ -824,7 +852,7 @@ static int mgc_fs_cleanup(const struct lu_env *env, struct obd_device *obd) mgc_local_llog_fini(env, obd); - lu_object_put_nocache(env, &cli->cl_mgc_configs_dir->do_lu); + dt_object_put_nocache(env, cli->cl_mgc_configs_dir); cli->cl_mgc_configs_dir = NULL; local_oid_storage_fini(env, cli->cl_mgc_los); @@ -835,6 +863,7 @@ static int mgc_fs_cleanup(const struct lu_env *env, struct obd_device *obd) RETURN(0); } +#endif /* HAVE_SERVER_SUPPORT */ static int mgc_llog_init(const struct lu_env *env, struct obd_device *obd) { @@ -912,7 +941,7 @@ static int mgc_cleanup(struct obd_device *obd) /* COMPAT_146 - old config logs may have added profiles we don't know about */ - if (obd->obd_type->typ_refcnt <= 1) + if (atomic_read(&obd->obd_type->typ_refcnt) <= 1) /* Only for the last mgc */ class_del_profiles(); @@ -943,11 +972,9 @@ static int mgc_setup(struct obd_device *obd, struct lustre_cfg *lcfg) GOTO(err_cleanup, rc); } -#ifdef CONFIG_PROC_FS - obd->obd_vars = lprocfs_mgc_obd_vars; - lprocfs_obd_setup(obd); -#endif - sptlrpc_lprocfs_cliobd_attach(obd); + rc = mgc_tunables_init(obd); + if (rc) + GOTO(err_sysfs, rc); if (atomic_inc_return(&mgc_count) == 1) { rq_state = 0; @@ -960,7 +987,7 @@ static int mgc_setup(struct obd_device *obd, struct lustre_cfg *lcfg) CERROR("%s: cannot start requeue thread: rc = %d; " "no more log updates\n", obd->obd_name, rc); - GOTO(err_cleanup, rc); + GOTO(err_sysfs, rc); } /* rc is the task_struct pointer of mgc_requeue_thread. */ rc = 0; @@ -969,6 +996,8 @@ static int mgc_setup(struct obd_device *obd, struct lustre_cfg *lcfg) RETURN(rc); +err_sysfs: + lprocfs_obd_cleanup(obd); err_cleanup: client_obd_cleanup(obd); err_decref: @@ -1007,6 +1036,9 @@ static int mgc_blocking_ast(struct ldlm_lock *lock, struct ldlm_lock_desc *desc, /* held at mgc_process_log(). */ LASSERT(atomic_read(&cld->cld_refcount) > 0); + + lock->l_ast_data = NULL; + cld->cld_lockh.cookie = 0; /* Are we done with this log? */ if (cld->cld_stopping) { CDEBUG(D_MGC, "log %s: stopping, won't requeue\n", @@ -1041,49 +1073,13 @@ static int mgc_blocking_ast(struct ldlm_lock *lock, struct ldlm_lock_desc *desc, #define MGC_ENQUEUE_LIMIT (INITIAL_CONNECT_TIMEOUT + (AT_OFF ? 0 : at_min) \ + PING_INTERVAL) #define MGC_TARGET_REG_LIMIT 10 +#define MGC_TARGET_REG_LIMIT_MAX RECONNECT_DELAY_MAX #define MGC_SEND_PARAM_LIMIT 10 -/* Send parameter to MGS*/ -static int mgc_set_mgs_param(struct obd_export *exp, - struct mgs_send_param *msp) -{ - struct ptlrpc_request *req; - struct mgs_send_param *req_msp, *rep_msp; - int rc; - ENTRY; - - req = ptlrpc_request_alloc_pack(class_exp2cliimp(exp), - &RQF_MGS_SET_INFO, LUSTRE_MGS_VERSION, - MGS_SET_INFO); - if (!req) - RETURN(-ENOMEM); - - req_msp = req_capsule_client_get(&req->rq_pill, &RMF_MGS_SEND_PARAM); - if (!req_msp) { - ptlrpc_req_finished(req); - RETURN(-ENOMEM); - } - - memcpy(req_msp, msp, sizeof(*req_msp)); - ptlrpc_request_set_replen(req); - - /* Limit how long we will wait for the enqueue to complete */ - req->rq_delay_limit = MGC_SEND_PARAM_LIMIT; - rc = ptlrpc_queue_wait(req); - if (!rc) { - rep_msp = req_capsule_server_get(&req->rq_pill, &RMF_MGS_SEND_PARAM); - memcpy(msp, rep_msp, sizeof(*rep_msp)); - } - - ptlrpc_req_finished(req); - - RETURN(rc); -} - /* Take a config lock so we can get cancel notifications */ static int mgc_enqueue(struct obd_export *exp, enum ldlm_type type, union ldlm_policy_data *policy, enum ldlm_mode mode, - __u64 *flags, void *bl_cb, void *cp_cb, void *gl_cb, + __u64 *flags, ldlm_glimpse_callback glimpse_callback, void *data, __u32 lvb_len, void *lvb_swabber, struct lustre_handle *lockh) { @@ -1093,13 +1089,17 @@ static int mgc_enqueue(struct obd_export *exp, enum ldlm_type type, .ei_mode = mode, .ei_cb_bl = mgc_blocking_ast, .ei_cb_cp = ldlm_completion_ast, + .ei_cb_gl = glimpse_callback, }; struct ptlrpc_request *req; int short_limit = cld_is_sptlrpc(cld); int rc; ENTRY; - CDEBUG(D_MGC, "Enqueue for %s (res "LPX64")\n", cld->cld_logname, + if (!exp) + RETURN(-EBADR); + + CDEBUG(D_MGC, "Enqueue for %s (res %#llx)\n", cld->cld_logname, cld->cld_resid.name[0]); /* We need a callback for every lockholder, so don't try to @@ -1113,12 +1113,11 @@ static int mgc_enqueue(struct obd_export *exp, enum ldlm_type type, req_capsule_set_size(&req->rq_pill, &RMF_DLM_LVB, RCL_SERVER, 0); ptlrpc_request_set_replen(req); - /* check if this is server or client */ - if (cld->cld_cfg.cfg_sb) { - struct lustre_sb_info *lsi = s2lsi(cld->cld_cfg.cfg_sb); - if (lsi && IS_SERVER(lsi)) - short_limit = 1; - } + /* check if this is server or client */ + if (cld->cld_cfg.cfg_sb && + IS_SERVER(s2lsi(cld->cld_cfg.cfg_sb))) + short_limit = 1; + /* Limit how long we will wait for the enqueue to complete */ req->rq_delay_limit = short_limit ? 5 : MGC_ENQUEUE_LIMIT; rc = ldlm_cli_enqueue(exp, &req, &einfo, &cld->cld_resid, NULL, flags, @@ -1150,45 +1149,57 @@ static void mgc_notify_active(struct obd_device *unused) /* TODO: Help the MGS rebuild nidtbl. -jay */ } +#ifdef HAVE_SERVER_SUPPORT /* Send target_reg message to MGS */ static int mgc_target_register(struct obd_export *exp, - struct mgs_target_info *mti) + struct mgs_target_info *mti) { - struct ptlrpc_request *req; - struct mgs_target_info *req_mti, *rep_mti; - int rc; - ENTRY; + struct ptlrpc_request *req; + struct mgs_target_info *req_mti, *rep_mti; + int rc; + ENTRY; - req = ptlrpc_request_alloc_pack(class_exp2cliimp(exp), - &RQF_MGS_TARGET_REG, LUSTRE_MGS_VERSION, - MGS_TARGET_REG); - if (req == NULL) - RETURN(-ENOMEM); + req = ptlrpc_request_alloc_pack(class_exp2cliimp(exp), + &RQF_MGS_TARGET_REG, LUSTRE_MGS_VERSION, + MGS_TARGET_REG); + if (req == NULL) + RETURN(-ENOMEM); - req_mti = req_capsule_client_get(&req->rq_pill, &RMF_MGS_TARGET_INFO); - if (!req_mti) { - ptlrpc_req_finished(req); - RETURN(-ENOMEM); - } + req_mti = req_capsule_client_get(&req->rq_pill, &RMF_MGS_TARGET_INFO); + if (!req_mti) { + ptlrpc_req_finished(req); + RETURN(-ENOMEM); + } - memcpy(req_mti, mti, sizeof(*req_mti)); - ptlrpc_request_set_replen(req); - CDEBUG(D_MGC, "register %s\n", mti->mti_svname); - /* Limit how long we will wait for the enqueue to complete */ - req->rq_delay_limit = MGC_TARGET_REG_LIMIT; + memcpy(req_mti, mti, sizeof(*req_mti)); + ptlrpc_request_set_replen(req); + CDEBUG(D_MGC, "register %s\n", mti->mti_svname); + /* Limit how long we will wait for the enqueue to complete */ + req->rq_delay_limit = MGC_TARGET_REG_LIMIT; + + /* if the target needs to regenerate the config log in MGS, it's better + * to use some longer limit to let MGC have time to change connection to + * another MGS (or try again with the same MGS) for the target (server) + * will fail and exit if the request expired due to delay limit. */ + if (mti->mti_flags & (LDD_F_UPDATE | LDD_F_NEED_INDEX)) + req->rq_delay_limit = MGC_TARGET_REG_LIMIT_MAX; rc = ptlrpc_queue_wait(req); - if (!rc) { - rep_mti = req_capsule_server_get(&req->rq_pill, - &RMF_MGS_TARGET_INFO); - memcpy(mti, rep_mti, sizeof(*rep_mti)); - CDEBUG(D_MGC, "register %s got index = %d\n", - mti->mti_svname, mti->mti_stripe_index); + if (ptlrpc_client_replied(req)) { + rep_mti = req_capsule_server_get(&req->rq_pill, + &RMF_MGS_TARGET_INFO); + if (rep_mti) + memcpy(mti, rep_mti, sizeof(*rep_mti)); + } + if (!rc) { + CDEBUG(D_MGC, "register %s got index = %d\n", + mti->mti_svname, mti->mti_stripe_index); } ptlrpc_req_finished(req); RETURN(rc); } +#endif /* HAVE_SERVER_SUPPORT */ static int mgc_set_info_async(const struct lu_env *env, struct obd_export *exp, u32 keylen, void *key, @@ -1198,35 +1209,41 @@ static int mgc_set_info_async(const struct lu_env *env, struct obd_export *exp, int rc = -EINVAL; ENTRY; - /* Turn off initial_recov after we try all backup servers once */ - if (KEY_IS(KEY_INIT_RECOV_BACKUP)) { - struct obd_import *imp = class_exp2cliimp(exp); - int value; - if (vallen != sizeof(int)) - RETURN(-EINVAL); - value = *(int *)val; - CDEBUG(D_MGC, "InitRecov %s %d/d%d:i%d:r%d:or%d:%s\n", - imp->imp_obd->obd_name, value, - imp->imp_deactive, imp->imp_invalid, - imp->imp_replayable, imp->imp_obd->obd_replayable, - ptlrpc_import_state_name(imp->imp_state)); - /* Resurrect if we previously died */ - if ((imp->imp_state != LUSTRE_IMP_FULL && - imp->imp_state != LUSTRE_IMP_NEW) || value > 1) - ptlrpc_reconnect_import(imp); - RETURN(0); - } - /* FIXME move this to mgc_process_config */ - if (KEY_IS(KEY_REGISTER_TARGET)) { - struct mgs_target_info *mti; - if (vallen != sizeof(struct mgs_target_info)) - RETURN(-EINVAL); - mti = (struct mgs_target_info *)val; - CDEBUG(D_MGC, "register_target %s %#x\n", - mti->mti_svname, mti->mti_flags); - rc = mgc_target_register(exp, mti); - RETURN(rc); - } + /* Turn off initial_recov after we try all backup servers once */ + if (KEY_IS(KEY_INIT_RECOV_BACKUP)) { + struct obd_import *imp = class_exp2cliimp(exp); + int value; + if (vallen != sizeof(int)) + RETURN(-EINVAL); + value = *(int *)val; + CDEBUG(D_MGC, "InitRecov %s %d/d%d:i%d:r%d:or%d:%s\n", + imp->imp_obd->obd_name, value, + imp->imp_deactive, imp->imp_invalid, + imp->imp_replayable, imp->imp_obd->obd_replayable, + ptlrpc_import_state_name(imp->imp_state)); + /* Resurrect the import immediately if + * 1. we previously got disconnected, + * 2. value > 1 (at the same node with MGS) + * */ + if (imp->imp_state == LUSTRE_IMP_DISCON || value > 1) + ptlrpc_reconnect_import(imp); + + RETURN(0); + } + +#ifdef HAVE_SERVER_SUPPORT + /* FIXME move this to mgc_process_config */ + if (KEY_IS(KEY_REGISTER_TARGET)) { + struct mgs_target_info *mti; + + if (vallen != sizeof(struct mgs_target_info)) + RETURN(-EINVAL); + mti = (struct mgs_target_info *)val; + CDEBUG(D_MGC, "register_target %s %#x\n", + mti->mti_svname, mti->mti_flags); + rc = mgc_target_register(exp, mti); + RETURN(rc); + } if (KEY_IS(KEY_SET_FS)) { struct super_block *sb = (struct super_block *)val; @@ -1242,13 +1259,7 @@ static int mgc_set_info_async(const struct lu_env *env, struct obd_export *exp, rc = mgc_fs_cleanup(env, exp->exp_obd); RETURN(rc); } - if (KEY_IS(KEY_SET_INFO)) { - struct mgs_send_param *msp; - - msp = (struct mgs_send_param *)val; - rc = mgc_set_mgs_param(exp, msp); - RETURN(rc); - } +#endif if (KEY_IS(KEY_MGSSEC)) { struct client_obd *cli = &exp->exp_obd->u.cli; struct sptlrpc_flavor flvr; @@ -1357,7 +1368,7 @@ static int mgc_import_event(struct obd_device *obd, } enum { - CONFIG_READ_NRPAGES_INIT = 1 << (20 - PAGE_CACHE_SHIFT), + CONFIG_READ_NRPAGES_INIT = 1 << (20 - PAGE_SHIFT), CONFIG_READ_NRPAGES = 4 }; @@ -1366,79 +1377,80 @@ static int mgc_apply_recover_logs(struct obd_device *mgc, __u64 max_version, void *data, int datalen, bool mne_swab) { - struct config_llog_instance *cfg = &cld->cld_cfg; - struct lustre_sb_info *lsi = s2lsi(cfg->cfg_sb); - struct mgs_nidtbl_entry *entry; - struct lustre_cfg *lcfg; - struct lustre_cfg_bufs bufs; - u64 prev_version = 0; - char *inst; - char *buf; - int bufsz; - int pos; - int rc = 0; - int off = 0; - ENTRY; + struct config_llog_instance *cfg = &cld->cld_cfg; + struct mgs_nidtbl_entry *entry; + struct lustre_cfg *lcfg; + struct lustre_cfg_bufs bufs; + u64 prev_version = 0; + char inst[MTI_NAME_MAXLEN + 1]; + char *buf; + int bufsz; + int pos = 0; + int rc = 0; + int off = 0; + unsigned long dynamic_nids; - LASSERT(cfg->cfg_instance != NULL); - LASSERT(cfg->cfg_sb == cfg->cfg_instance); + ENTRY; + LASSERT(cfg->cfg_instance != 0); + LASSERT(ll_get_cfg_instance(cfg->cfg_sb) == cfg->cfg_instance); - OBD_ALLOC(inst, PAGE_CACHE_SIZE); - if (inst == NULL) - RETURN(-ENOMEM); + /* get dynamic nids setting */ + dynamic_nids = mgc->obd_dynamic_nids; - if (!IS_SERVER(lsi)) { - pos = snprintf(inst, PAGE_CACHE_SIZE, "%p", cfg->cfg_instance); - if (pos >= PAGE_CACHE_SIZE) { - OBD_FREE(inst, PAGE_CACHE_SIZE); + if (!IS_SERVER(s2lsi(cfg->cfg_sb))) { + pos = snprintf(inst, sizeof(inst), "%016lx", cfg->cfg_instance); + if (pos >= PAGE_SIZE) return -E2BIG; - } - } else { +#ifdef HAVE_SERVER_SUPPORT + } else { + struct lustre_sb_info *lsi = s2lsi(cfg->cfg_sb); + LASSERT(IS_MDT(lsi)); rc = server_name2svname(lsi->lsi_svname, inst, NULL, - PAGE_CACHE_SIZE); - if (rc) { - OBD_FREE(inst, PAGE_CACHE_SIZE); + sizeof(inst)); + if (rc) RETURN(-EINVAL); - } - pos = strlen(inst); - } +#endif /* HAVE_SERVER_SUPPORT */ + } + + OBD_ALLOC(buf, PAGE_SIZE); + if (!buf) + return -ENOMEM; + bufsz = PAGE_SIZE; + pos = 0; + + while (datalen > 0) { + int entry_len = sizeof(*entry); + int is_ost; + struct obd_device *obd; + struct obd_import *imp; + char *obdname; + char *cname; + char *params; + char *uuid; + + rc = -EINVAL; + if (datalen < sizeof(*entry)) + break; + + entry = (typeof(entry))(data + off); - ++pos; - buf = inst + pos; - bufsz = PAGE_CACHE_SIZE - pos; - - while (datalen > 0) { - int entry_len = sizeof(*entry); - int is_ost, i; - struct obd_device *obd; - char *obdname; - char *cname; - char *params; - char *uuid; - - rc = -EINVAL; - if (datalen < sizeof(*entry)) - break; - - entry = (typeof(entry))(data + off); - - /* sanity check */ - if (entry->mne_nid_type != 0) /* only support type 0 for ipv4 */ - break; - if (entry->mne_nid_count == 0) /* at least one nid entry */ - break; - if (entry->mne_nid_size != sizeof(lnet_nid_t)) - break; - - entry_len += entry->mne_nid_count * entry->mne_nid_size; - if (datalen < entry_len) /* must have entry_len at least */ - break; + /* sanity check */ + if (entry->mne_nid_type != 0) /* only support type 0 for ipv4 */ + break; + if (entry->mne_nid_count == 0) /* at least one nid entry */ + break; + if (entry->mne_nid_size != sizeof(lnet_nid_t)) + break; + + entry_len += entry->mne_nid_count * entry->mne_nid_size; + if (datalen < entry_len) /* must have entry_len at least */ + break; /* Keep this swab for normal mixed endian handling. LU-1644 */ if (mne_swab) lustre_swab_mgs_nidtbl_entry(entry); - if (entry->mne_length > PAGE_CACHE_SIZE) { + if (entry->mne_length > PAGE_SIZE) { CERROR("MNE too large (%u)\n", entry->mne_length); break; } @@ -1446,123 +1458,155 @@ static int mgc_apply_recover_logs(struct obd_device *mgc, if (entry->mne_length < entry_len) break; - off += entry->mne_length; - datalen -= entry->mne_length; - if (datalen < 0) - break; + off += entry->mne_length; + datalen -= entry->mne_length; + if (datalen < 0) + break; - if (entry->mne_version > max_version) { - CERROR("entry index(%lld) is over max_index(%lld)\n", - entry->mne_version, max_version); - break; - } + if (entry->mne_version > max_version) { + CERROR("entry index(%lld) is over max_index(%lld)\n", + entry->mne_version, max_version); + break; + } - if (prev_version >= entry->mne_version) { - CERROR("index unsorted, prev %lld, now %lld\n", - prev_version, entry->mne_version); - break; - } - prev_version = entry->mne_version; + if (prev_version >= entry->mne_version) { + CERROR("index unsorted, prev %lld, now %lld\n", + prev_version, entry->mne_version); + break; + } + prev_version = entry->mne_version; - /* - * Write a string with format "nid::instance" to - * lustre//--/import. - */ + /* + * Write a string with format "nid::instance" to + * lustre//--/import. + */ - is_ost = entry->mne_type == LDD_F_SV_TYPE_OST; - memset(buf, 0, bufsz); - obdname = buf; - pos = 0; - - /* lustre-OST0001-osc- */ - strcpy(obdname, cld->cld_logname); - cname = strrchr(obdname, '-'); - if (cname == NULL) { - CERROR("mgc %s: invalid logname %s\n", - mgc->obd_name, obdname); - break; - } + is_ost = entry->mne_type == LDD_F_SV_TYPE_OST; + memset(buf, 0, bufsz); + obdname = buf; + pos = 0; + + /* lustre-OST0001-osc- */ + strcpy(obdname, cld->cld_logname); + cname = strrchr(obdname, '-'); + if (cname == NULL) { + CERROR("mgc %s: invalid logname %s\n", + mgc->obd_name, obdname); + break; + } - pos = cname - obdname; - obdname[pos] = 0; - pos += sprintf(obdname + pos, "-%s%04x", - is_ost ? "OST" : "MDT", entry->mne_index); + pos = cname - obdname; + obdname[pos] = 0; + pos += sprintf(obdname + pos, "-%s%04x", + is_ost ? "OST" : "MDT", entry->mne_index); - cname = is_ost ? "osc" : "mdc", - pos += sprintf(obdname + pos, "-%s-%s", cname, inst); - lustre_cfg_bufs_reset(&bufs, obdname); + cname = is_ost ? "osc" : "mdc", + pos += snprintf(obdname + pos, bufsz, "-%s-%s", cname, inst); + lustre_cfg_bufs_reset(&bufs, obdname); - /* find the obd by obdname */ - obd = class_name2obd(obdname); - if (obd == NULL) { - CDEBUG(D_INFO, "mgc %s: cannot find obdname %s\n", - mgc->obd_name, obdname); + /* find the obd by obdname */ + obd = class_name2obd(obdname); + if (obd == NULL) { + CDEBUG(D_INFO, "mgc %s: cannot find obdname %s\n", + mgc->obd_name, obdname); rc = 0; - /* this is a safe race, when the ost is starting up...*/ - continue; - } - - /* osc.import = "connection=::" */ - ++pos; - params = buf + pos; - pos += sprintf(params, "%s.import=%s", cname, "connection="); - uuid = buf + pos; + /* this is a safe race, when the ost is starting up...*/ + continue; + } - down_read(&obd->u.cli.cl_sem); - if (obd->u.cli.cl_import == NULL) { + /* osc.import = "connection=::" */ + ++pos; + params = buf + pos; + pos += sprintf(params, "%s.import=%s", cname, "connection="); + uuid = buf + pos; + + with_imp_locked(obd, imp, rc) { + /* iterate all nids to find one */ + /* find uuid by nid */ + /* create import entries if they don't exist */ + rc = client_import_add_nids_to_conn( + imp, entry->u.nids, entry->mne_nid_count, + (struct obd_uuid *)uuid); + + if (rc == -ENOENT && dynamic_nids) { + /* create a new connection for this import */ + char *primary_nid = + libcfs_nid2str(entry->u.nids[0]); + int prim_nid_len = strlen(primary_nid) + 1; + struct obd_uuid server_uuid; + + if (prim_nid_len > UUID_MAX) + goto fail; + strncpy(server_uuid.uuid, primary_nid, + prim_nid_len); + + CDEBUG(D_INFO, "Adding a connection for %s\n", + primary_nid); + + rc = client_import_dyn_add_conn( + imp, &server_uuid, entry->u.nids[0], 1); + if (rc < 0) { + CERROR("%s: Failed to add new connection with NID '%s' to import: rc = %d\n", + obd->obd_name, primary_nid, rc); + goto fail; + } + rc = client_import_add_nids_to_conn( + imp, entry->u.nids, + entry->mne_nid_count, + (struct obd_uuid *)uuid); + if (rc < 0) { + CERROR("%s: failed to lookup UUID: rc = %d\n", + obd->obd_name, rc); + goto fail; + } + } +fail:; + } + if (rc == -ENODEV) { /* client does not connect to the OST yet */ - up_read(&obd->u.cli.cl_sem); rc = 0; continue; } - /* iterate all nids to find one */ - /* find uuid by nid */ - rc = -ENOENT; - for (i = 0; i < entry->mne_nid_count; i++) { - rc = client_import_find_conn(obd->u.cli.cl_import, - entry->u.nids[i], - (struct obd_uuid *)uuid); - if (rc == 0) - break; + if (rc < 0 && rc != -ENOSPC) { + CERROR("mgc: cannot find UUID by nid '%s': rc = %d\n", + libcfs_nid2str(entry->u.nids[0]), rc); + break; } - up_read(&obd->u.cli.cl_sem); - if (rc < 0) { - CERROR("mgc: cannot find uuid by nid %s\n", - libcfs_nid2str(entry->u.nids[0])); - break; - } - - CDEBUG(D_INFO, "Find uuid %s by nid %s\n", - uuid, libcfs_nid2str(entry->u.nids[0])); + CDEBUG(D_INFO, "Found UUID '%s' by NID '%s'\n", + uuid, libcfs_nid2str(entry->u.nids[0])); - pos += strlen(uuid); - pos += sprintf(buf + pos, "::%u", entry->mne_instance); - LASSERT(pos < bufsz); + pos += strlen(uuid); + pos += sprintf(buf + pos, "::%u", entry->mne_instance); + LASSERT(pos < bufsz); - lustre_cfg_bufs_set_string(&bufs, 1, params); + lustre_cfg_bufs_set_string(&bufs, 1, params); - lcfg = lustre_cfg_new(LCFG_PARAM, &bufs); - if (lcfg == NULL) { + OBD_ALLOC(lcfg, lustre_cfg_len(bufs.lcfg_bufcount, + bufs.lcfg_buflen)); + if (!lcfg) { rc = -ENOMEM; break; } + lustre_cfg_init(lcfg, LCFG_PARAM, &bufs); - CDEBUG(D_INFO, "ir apply logs "LPD64"/"LPD64" for %s -> %s\n", - prev_version, max_version, obdname, params); + CDEBUG(D_INFO, "ir apply logs %lld/%lld for %s -> %s\n", + prev_version, max_version, obdname, params); - rc = class_process_config(lcfg); - lustre_cfg_free(lcfg); - if (rc) - CDEBUG(D_INFO, "process config for %s error %d\n", - obdname, rc); + rc = class_process_config(lcfg); + OBD_FREE(lcfg, lustre_cfg_len(lcfg->lcfg_bufcount, + lcfg->lcfg_buflens)); + if (rc) + CDEBUG(D_INFO, "process config for %s error %d\n", + obdname, rc); - /* continue, even one with error */ - } + /* continue, even one with error */ + } - OBD_FREE(inst, PAGE_CACHE_SIZE); - RETURN(rc); + OBD_FREE(buf, PAGE_SIZE); + + RETURN(rc); } /** @@ -1573,6 +1617,7 @@ static int mgc_apply_recover_logs(struct obd_device *mgc, static int mgc_process_recover_nodemap_log(struct obd_device *obd, struct config_llog_data *cld) { + struct ptlrpc_connection *mgc_conn; struct ptlrpc_request *req = NULL; struct config_llog_instance *cfg = &cld->cld_cfg; struct mgs_config_body *body; @@ -1580,36 +1625,43 @@ static int mgc_process_recover_nodemap_log(struct obd_device *obd, struct nodemap_config *new_config = NULL; struct lu_nodemap *recent_nodemap = NULL; struct ptlrpc_bulk_desc *desc; - struct page **pages; + struct page **pages = NULL; __u64 config_read_offset = 0; - int nrpages; + __u8 nodemap_cur_pass = 0; + int nrpages = 0; bool eof = true; bool mne_swab = false; int i; int ealen; int rc; - ENTRY; - /* allocate buffer for bulk transfer. - * if this is the first time for this mgs to read logs, - * CONFIG_READ_NRPAGES_INIT will be used since it will read all logs - * once; otherwise, it only reads increment of logs, this should be - * small and CONFIG_READ_NRPAGES will be used. - */ - nrpages = CONFIG_READ_NRPAGES; + mgc_conn = class_exp2cliimp(cld->cld_mgcexp)->imp_connection; + + /* don't need to get local config */ + if (cld_is_nodemap(cld) && + LNetIsPeerLocal(&mgc_conn->c_peer.nid)) + GOTO(out, rc = 0); + + /* allocate buffer for bulk transfer. + * if this is the first time for this mgs to read logs, + * CONFIG_READ_NRPAGES_INIT will be used since it will read all logs + * once; otherwise, it only reads increment of logs, this should be + * small and CONFIG_READ_NRPAGES will be used. + */ + nrpages = CONFIG_READ_NRPAGES; if (cfg->cfg_last_idx == 0 || cld_is_nodemap(cld)) - nrpages = CONFIG_READ_NRPAGES_INIT; + nrpages = CONFIG_READ_NRPAGES_INIT; - OBD_ALLOC(pages, sizeof(*pages) * nrpages); - if (pages == NULL) - GOTO(out, rc = -ENOMEM); + OBD_ALLOC_PTR_ARRAY_LARGE(pages, nrpages); + if (pages == NULL) + GOTO(out, rc = -ENOMEM); - for (i = 0; i < nrpages; i++) { + for (i = 0; i < nrpages; i++) { pages[i] = alloc_page(GFP_KERNEL); - if (pages[i] == NULL) - GOTO(out, rc = -ENOMEM); - } + if (pages[i] == NULL) + GOTO(out, rc = -ENOMEM); + } again: #ifdef HAVE_SERVER_SUPPORT @@ -1645,12 +1697,13 @@ again: else body->mcb_offset = cfg->cfg_last_idx + 1; body->mcb_type = cld->cld_type; - body->mcb_bits = PAGE_CACHE_SHIFT; + body->mcb_bits = PAGE_SHIFT; body->mcb_units = nrpages; + body->mcb_nm_cur_pass = nodemap_cur_pass; /* allocate bulk transfer descriptor */ desc = ptlrpc_prep_bulk_imp(req, nrpages, 1, - PTLRPC_BULK_PUT_SINK | PTLRPC_BULK_BUF_KIOV, + PTLRPC_BULK_PUT_SINK, MGS_BULK_PORTAL, &ptlrpc_bulk_kiov_pin_ops); if (desc == NULL) @@ -1658,7 +1711,7 @@ again: for (i = 0; i < nrpages; i++) desc->bd_frag_ops->add_kiov_frag(desc, pages[i], 0, - PAGE_CACHE_SIZE); + PAGE_SIZE); ptlrpc_request_set_replen(req); rc = ptlrpc_queue_wait(req); @@ -1672,6 +1725,7 @@ again: if (cld_is_nodemap(cld)) { config_read_offset = res->mcr_offset; eof = config_read_offset == II_END_OFF; + nodemap_cur_pass = res->mcr_nm_cur_pass; } else { if (res->mcr_size < res->mcr_offset) GOTO(out, rc = -EINVAL); @@ -1682,7 +1736,7 @@ again: cfg->cfg_last_idx = res->mcr_offset; eof = res->mcr_offset == res->mcr_size; - CDEBUG(D_INFO, "Latest version "LPD64", more %d.\n", + CDEBUG(D_INFO, "Latest version %lld, more %d.\n", res->mcr_offset, eof == false); } @@ -1690,22 +1744,15 @@ again: if (ealen < 0) GOTO(out, rc = ealen); - if (ealen > nrpages << PAGE_CACHE_SHIFT) + if (ealen > nrpages << PAGE_SHIFT) GOTO(out, rc = -EINVAL); if (ealen == 0) { /* no logs transferred */ #ifdef HAVE_SERVER_SUPPORT /* config changed since first read RPC */ if (cld_is_nodemap(cld) && config_read_offset == 0) { - recent_nodemap = NULL; - nodemap_config_dealloc(new_config); - new_config = NULL; - CDEBUG(D_INFO, "nodemap config changed in transit, retrying\n"); - - /* setting eof to false, we request config again */ - eof = false; - GOTO(out, rc = 0); + GOTO(out, rc = -EAGAIN); } #endif if (!eof) @@ -1713,13 +1760,7 @@ again: GOTO(out, rc); } - mne_swab = !!ptlrpc_rep_need_swab(req); -#if LUSTRE_VERSION_CODE < OBD_OCD_VERSION(3, 0, 53, 0) - /* This import flag means the server did an extra swab of IR MNE - * records (fixed in LU-1252), reverse it here if needed. LU-1644 */ - if (unlikely(req->rq_import->imp_need_mne_swab)) - mne_swab = !mne_swab; -#endif + mne_swab = req_capsule_rep_need_swab(&req->rq_pill); /* When a nodemap config is received, we build a new nodemap config, * with new nodemap structs. We keep track of the most recently added @@ -1742,7 +1783,7 @@ again: rc2 = mgc_apply_recover_logs(obd, cld, res->mcr_offset, ptr, min_t(int, ealen, - PAGE_CACHE_SIZE), + PAGE_SIZE), mne_swab); kunmap(pages[i]); if (rc2 < 0) { @@ -1751,10 +1792,10 @@ again: cld_is_nodemap(cld) ? "nodemap" : "recovery", cld->cld_logname, rc2); - break; + GOTO(out, rc = rc2); } - ealen -= PAGE_CACHE_SIZE; + ealen -= PAGE_SIZE; } out: @@ -1782,11 +1823,27 @@ out: break; __free_page(pages[i]); } - OBD_FREE(pages, sizeof(*pages) * nrpages); + OBD_FREE_PTR_ARRAY_LARGE(pages, nrpages); } return rc; } +#ifdef HAVE_SERVER_SUPPORT +static int mgc_barrier_glimpse_ast(struct ldlm_lock *lock, void *data) +{ + struct config_llog_data *cld = lock->l_ast_data; + int rc; + ENTRY; + + if (cld->cld_stopping) + RETURN(-ENODEV); + + rc = barrier_handler(s2lsi(cld->cld_cfg.cfg_sb)->lsi_dt_dev, + (struct ptlrpc_request *)data); + + RETURN(rc); +} + /* Copy a remote log locally */ static int mgc_llog_local_copy(const struct lu_env *env, struct obd_device *obd, @@ -1830,44 +1887,14 @@ out: return rc; } -/* local_only means it cannot get remote llogs */ -static int mgc_process_cfg_log(struct obd_device *mgc, - struct config_llog_data *cld, int local_only) +static int +mgc_process_server_cfg_log(struct lu_env *env, struct llog_ctxt **ctxt, + struct lustre_sb_info *lsi, struct obd_device *mgc, + struct config_llog_data *cld, int local_only) { - struct llog_ctxt *ctxt, *lctxt = NULL; - struct client_obd *cli = &mgc->u.cli; - struct lustre_sb_info *lsi = NULL; - int rc = 0; - bool sptlrpc_started = false; - struct lu_env *env; - - ENTRY; - - LASSERT(cld); - LASSERT(mutex_is_locked(&cld->cld_lock)); - - /* - * local copy of sptlrpc log is controlled elsewhere, don't try to - * read it up here. - */ - if (cld_is_sptlrpc(cld) && local_only) - RETURN(0); - - if (cld->cld_cfg.cfg_sb) - lsi = s2lsi(cld->cld_cfg.cfg_sb); - - OBD_ALLOC_PTR(env); - if (env == NULL) - RETURN(-ENOMEM); - - rc = lu_env_init(env, LCT_MG_THREAD); - if (rc) - GOTO(out_free, rc); - - ctxt = llog_get_context(mgc, LLOG_CONFIG_REPL_CTXT); - LASSERT(ctxt); - - lctxt = llog_get_context(mgc, LLOG_CONFIG_ORIG_CTXT); + struct llog_ctxt *lctxt = llog_get_context(mgc, LLOG_CONFIG_ORIG_CTXT); + struct client_obd *cli = &mgc->u.cli; + int rc = 0; /* Copy the setup log locally if we can. Don't mess around if we're * running an MGS though (logs are already local). */ @@ -1875,65 +1902,118 @@ static int mgc_process_cfg_log(struct obd_device *mgc, cli->cl_mgc_configs_dir != NULL && lu2dt_dev(cli->cl_mgc_configs_dir->do_lu.lo_dev) == lsi->lsi_dt_dev) { - if (!local_only) + if (!local_only && !lsi->lsi_dt_dev->dd_rdonly) { /* Only try to copy log if we have the lock. */ - rc = mgc_llog_local_copy(env, mgc, ctxt, lctxt, + CDEBUG(D_INFO, "%s: copy local log %s\n", + mgc->obd_name, cld->cld_logname); + + rc = mgc_llog_local_copy(env, mgc, *ctxt, lctxt, cld->cld_logname); + if (!rc) + lsi->lsi_flags &= ~LDD_F_NO_LOCAL_LOGS; + } if (local_only || rc) { + if (unlikely(lsi->lsi_flags & LDD_F_NO_LOCAL_LOGS) || + rc) { + CWARN("%s: local log %s are not valid and/or remote logs are not accessbile rc = %d\n", + mgc->obd_name, cld->cld_logname, rc); + GOTO(out_pop, rc = -EIO); + } + if (strcmp(cld->cld_logname, PARAMS_FILENAME) != 0 && llog_is_empty(env, lctxt, cld->cld_logname)) { - LCONSOLE_ERROR_MSG(0x13a, "Failed to get MGS " - "log %s and no local copy." - "\n", cld->cld_logname); + LCONSOLE_ERROR_MSG(0x13a, "Failed to get MGS log %s and no local copy.\n", + cld->cld_logname); GOTO(out_pop, rc = -ENOENT); } - CDEBUG(D_MGC, "Failed to get MGS log %s, using local " - "copy for now, will try to update later.\n", - cld->cld_logname); + CDEBUG(D_MGC, "%s: Failed to get MGS log %s, using local copy for now, will try to update later.\n", + mgc->obd_name, cld->cld_logname); rc = 0; } /* Now, whether we copied or not, start using the local llog. * If we failed to copy, we'll start using whatever the old * log has. */ - llog_ctxt_put(ctxt); - ctxt = lctxt; + llog_ctxt_put(*ctxt); + *ctxt = lctxt; lctxt = NULL; - } else { - if (local_only) /* no local log at client side */ - GOTO(out_pop, rc = -EIO); + } else if (local_only) { /* no local log at client side */ + GOTO(out_pop, rc = -EIO); } - if (cld_is_sptlrpc(cld)) { - sptlrpc_conf_log_update_begin(cld->cld_logname); - sptlrpc_started = true; - } + rc = -EAGAIN; + if (lsi && IS_SERVER(lsi) && !IS_MGS(lsi) && + lsi->lsi_dt_dev->dd_rdonly) { + struct llog_ctxt *rctxt; - /* logname and instance info should be the same, so use our - * copy of the instance for the update. The cfg_last_idx will - * be updated here. */ - rc = class_config_parse_llog(env, ctxt, cld->cld_logname, - &cld->cld_cfg); - EXIT; + /* Under readonly mode, we may have no local copy or local + * copy is incomplete, so try to use remote llog firstly. */ + rctxt = llog_get_context(mgc, LLOG_CONFIG_REPL_CTXT); + LASSERT(rctxt); + rc = class_config_parse_llog(env, rctxt, cld->cld_logname, + &cld->cld_cfg); + llog_ctxt_put(rctxt); + } out_pop: - __llog_ctxt_put(env, ctxt); if (lctxt) __llog_ctxt_put(env, lctxt); + return rc; +} +#else /* !HAVE_SERVER_SUPPORT */ +#define mgc_barrier_glimpse_ast NULL +#endif /* HAVE_SERVER_SUPPORT */ + +/* local_only means it cannot get remote llogs */ +static int mgc_process_cfg_log(struct obd_device *mgc, + struct config_llog_data *cld, int local_only) +{ + struct llog_ctxt *ctxt; + struct lustre_sb_info *lsi = NULL; + int rc = 0; + struct lu_env *env; + + ENTRY; + LASSERT(cld); + LASSERT(mutex_is_locked(&cld->cld_lock)); + + if (cld->cld_cfg.cfg_sb) + lsi = s2lsi(cld->cld_cfg.cfg_sb); + + OBD_ALLOC_PTR(env); + if (!env) + RETURN(-ENOMEM); + + rc = lu_env_init(env, LCT_MG_THREAD); + if (rc) + GOTO(out_free, rc); + + ctxt = llog_get_context(mgc, LLOG_CONFIG_REPL_CTXT); + LASSERT(ctxt); +#ifdef HAVE_SERVER_SUPPORT + rc = mgc_process_server_cfg_log(env, &ctxt, lsi, mgc, cld, + local_only); + if (rc == -EIO && local_only) + GOTO(out_pop, rc); + if (rc && rc != -ENOENT) +#else + if (local_only) + GOTO(out_pop, rc); +#endif + rc = class_config_parse_llog(env, ctxt, cld->cld_logname, + &cld->cld_cfg); /* - * update settings on existing OBDs. doing it inside - * of llog_process_lock so no device is attaching/detaching - * in parallel. + * update settings on existing OBDs. * the logname must be -sptlrpc */ - if (sptlrpc_started) { - LASSERT(cld_is_sptlrpc(cld)); - sptlrpc_conf_log_update_end(cld->cld_logname); + if (rc == 0 && cld_is_sptlrpc(cld)) class_notify_sptlrpc_conf(cld->cld_logname, strlen(cld->cld_logname) - strlen("-sptlrpc")); - } + EXIT; +out_pop: + __llog_ctxt_put(env, ctxt); lu_env_fini(env); out_free: OBD_FREE_PTR(env); @@ -1999,31 +2079,34 @@ restart: mutex_lock(&cld->cld_lock); if (cld->cld_stopping) { mutex_unlock(&cld->cld_lock); - RETURN(0); - } + RETURN(0); + } - OBD_FAIL_TIMEOUT(OBD_FAIL_MGC_PAUSE_PROCESS_LOG, 20); + OBD_FAIL_TIMEOUT(OBD_FAIL_MGC_PAUSE_PROCESS_LOG, 20); - CDEBUG(D_MGC, "Process log %s:%p from %d\n", cld->cld_logname, + CDEBUG(D_MGC, "Process log %s-%016lx from %d\n", cld->cld_logname, cld->cld_cfg.cfg_instance, cld->cld_cfg.cfg_last_idx + 1); /* Get the cfg lock on the llog */ rcl = mgc_enqueue(mgc->u.cli.cl_mgc_mgsexp, LDLM_PLAIN, NULL, - LCK_CR, &flags, NULL, NULL, NULL, + LCK_CR, &flags, + cld_is_barrier(cld) ? mgc_barrier_glimpse_ast : NULL, cld, 0, NULL, &lockh); if (rcl == 0) { /* Get the cld, it will be released in mgc_blocking_ast. */ config_log_get(cld); rc = ldlm_lock_set_data(&lockh, (void *)cld); + LASSERT(!lustre_handle_is_used(&cld->cld_lockh)); LASSERT(rc == 0); + cld->cld_lockh = lockh; } else { CDEBUG(D_MGC, "Can't get cfg lock: %d\n", rcl); + cld->cld_lockh.cookie = 0; if (rcl == -ESHUTDOWN && atomic_read(&mgc->u.cli.cl_mgc_refcount) > 0 && !retry) { struct obd_import *imp; - struct l_wait_info lwi; - int secs = cfs_time_seconds(obd_timeout); + long timeout = cfs_time_seconds(obd_timeout); mutex_unlock(&cld->cld_lock); imp = class_exp2cliimp(mgc->u.cli.cl_mgc_mgsexp); @@ -2036,56 +2119,63 @@ restart: * FULL or closed */ ptlrpc_pinger_force(imp); - lwi = LWI_TIMEOUT(secs, NULL, NULL); - l_wait_event(imp->imp_recovery_waitq, - !mgc_import_in_recovery(imp), &lwi); + wait_event_idle_timeout(imp->imp_recovery_waitq, + !mgc_import_in_recovery(imp), + timeout); if (imp->imp_state == LUSTRE_IMP_FULL) { retry = true; goto restart; } else { mutex_lock(&cld->cld_lock); + /* unlock/lock mutex, so check stopping again */ + if (cld->cld_stopping) { + mutex_unlock(&cld->cld_lock); + RETURN(0); + } + spin_lock(&config_list_lock); cld->cld_lostlock = 1; + spin_unlock(&config_list_lock); } } else { /* mark cld_lostlock so that it will requeue * after MGC becomes available. */ + spin_lock(&config_list_lock); cld->cld_lostlock = 1; + spin_unlock(&config_list_lock); } - /* Get extra reference, it will be put in requeue thread */ - config_log_get(cld); } - if (cld_is_recover(cld) || cld_is_nodemap(cld)) { if (!rcl) rc = mgc_process_recover_nodemap_log(mgc, cld); else if (cld_is_nodemap(cld)) rc = rcl; - if (cld_is_recover(cld) && rc) { - if (!rcl) { - CERROR("%s: recover log %s failed, not fatal: rc = %d\n", - mgc->obd_name, cld->cld_logname, rc); - cld->cld_lostlock = 1; - } - rc = 0; /* this is not a fatal error for recover log */ - } - } else { + } else if (!cld_is_barrier(cld)) { rc = mgc_process_cfg_log(mgc, cld, rcl != 0); } CDEBUG(D_MGC, "%s: configuration from log '%s' %sed (%d).\n", mgc->obd_name, cld->cld_logname, rc ? "fail" : "succeed", rc); - mutex_unlock(&cld->cld_lock); - /* Now drop the lock so MGS can revoke it */ if (!rcl) { rcl = mgc_cancel(mgc->u.cli.cl_mgc_mgsexp, LCK_CR, &lockh); if (rcl) CERROR("Can't drop cfg lock: %d\n", rcl); } + mutex_unlock(&cld->cld_lock); + + /* requeue nodemap lock immediately if transfer was interrupted */ + if ((cld_is_nodemap(cld) && rc == -EAGAIN) || + (cld_is_recover(cld) && rc)) { + if (cld_is_recover(cld)) + CWARN("%s: IR log %s failed, not fatal: rc = %d\n", + mgc->obd_name, cld->cld_logname, rc); + mgc_requeue_add(cld); + rc = 0; + } RETURN(rc); } @@ -2104,25 +2194,27 @@ static int mgc_process_config(struct obd_device *obd, size_t len, void *buf) ENTRY; switch(lcfg->lcfg_command) { - case LCFG_LOV_ADD_OBD: { - /* Overloading this cfg command: register a new target */ - struct mgs_target_info *mti; - - if (LUSTRE_CFG_BUFLEN(lcfg, 1) != - sizeof(struct mgs_target_info)) - GOTO(out, rc = -EINVAL); - - mti = (struct mgs_target_info *)lustre_cfg_buf(lcfg, 1); - CDEBUG(D_MGC, "add_target %s %#x\n", - mti->mti_svname, mti->mti_flags); - rc = mgc_target_register(obd->u.cli.cl_mgc_mgsexp, mti); - break; - } - case LCFG_LOV_DEL_OBD: - /* Unregister has no meaning at the moment. */ - CERROR("lov_del_obd unimplemented\n"); - rc = -ENOSYS; - break; +#ifdef HAVE_SERVER_SUPPORT + case LCFG_LOV_ADD_OBD: { + /* Overloading this cfg command: register a new target */ + struct mgs_target_info *mti; + + if (LUSTRE_CFG_BUFLEN(lcfg, 1) != + sizeof(struct mgs_target_info)) + GOTO(out, rc = -EINVAL); + + mti = lustre_cfg_buf(lcfg, 1); + CDEBUG(D_MGC, "add_target %s %#x\n", + mti->mti_svname, mti->mti_flags); + rc = mgc_target_register(obd->u.cli.cl_mgc_mgsexp, mti); + break; + } + case LCFG_LOV_DEL_OBD: + /* Unregister has no meaning at the moment. */ + CERROR("lov_del_obd unimplemented\n"); + rc = -EINVAL; + break; +#endif case LCFG_SPTLRPC_CONF: { rc = sptlrpc_process_config(lcfg); break; @@ -2139,33 +2231,30 @@ static int mgc_process_config(struct obd_device *obd, size_t len, void *buf) cfg->cfg_last_idx); /* We're only called through here on the initial mount */ - rc = config_log_add(obd, logname, cfg, sb); - if (rc) - break; - cld = config_log_find(logname, cfg); - if (cld == NULL) { - rc = -ENOENT; - break; - } + cld = config_log_add(obd, logname, cfg, sb); + if (IS_ERR(cld)) { + rc = PTR_ERR(cld); + break; + } - /* COMPAT_146 */ - /* FIXME only set this for old logs! Right now this forces - us to always skip the "inside markers" check */ - cld->cld_cfg.cfg_flags |= CFG_F_COMPAT146; - - rc = mgc_process_log(obd, cld); - if (rc == 0 && cld->cld_recover != NULL) { - if (OCD_HAS_FLAG(&obd->u.cli.cl_import-> - imp_connect_data, IMP_RECOV)) { - rc = mgc_process_log(obd, cld->cld_recover); - } else { - struct config_llog_data *cir = cld->cld_recover; - cld->cld_recover = NULL; - config_log_put(cir); - } - if (rc) - CERROR("Cannot process recover llog %d\n", rc); - } + rc = mgc_process_log(obd, cld); + if (rc == 0 && cld->cld_recover != NULL) { + if (OCD_HAS_FLAG(&obd->u.cli.cl_import-> + imp_connect_data, IMP_RECOV)) { + rc = mgc_process_log(obd, cld->cld_recover); + } else { + struct config_llog_data *cir; + + mutex_lock(&cld->cld_lock); + cir = cld->cld_recover; + cld->cld_recover = NULL; + mutex_unlock(&cld->cld_lock); + config_log_put(cir); + } + + if (rc) + CERROR("Cannot process recover llog %d\n", rc); + } if (rc == 0 && cld->cld_params != NULL) { rc = mgc_process_log(obd, cld->cld_params); @@ -2179,7 +2268,6 @@ static int mgc_process_config(struct obd_device *obd, size_t len, void *buf) CERROR("%s: can't process params llog: rc = %d\n", obd->obd_name, rc); } - config_log_put(cld); break; } @@ -2202,7 +2290,7 @@ out: RETURN(rc); } -static struct obd_ops mgc_obd_ops = { +static const struct obd_ops mgc_obd_ops = { .o_owner = THIS_MODULE, .o_setup = mgc_setup, .o_precleanup = mgc_precleanup, @@ -2217,9 +2305,43 @@ static struct obd_ops mgc_obd_ops = { .o_process_config = mgc_process_config, }; +static int mgc_param_requeue_timeout_min_set(const char *val, + cfs_kernel_param_arg_t *kp) +{ + int rc; + unsigned int num; + + rc = kstrtouint(val, 0, &num); + if (rc < 0) + return rc; + if (num > 120) + return -EINVAL; + + mgc_requeue_timeout_min = num; + + return 0; +} + +static const struct kernel_param_ops param_ops_requeue_timeout_min = { + .set = mgc_param_requeue_timeout_min_set, + .get = param_get_uint, +}; + +#define param_check_requeue_timeout_min(name, p) \ + __param_check(name, p, unsigned int) + +unsigned int mgc_requeue_timeout_min = MGC_TIMEOUT_MIN_SECONDS; +#ifdef HAVE_KERNEL_PARAM_OPS +module_param(mgc_requeue_timeout_min, requeue_timeout_min, 0644); +#else +module_param_call(mgc_requeue_timeout_min, mgc_param_requeue_timeout_min_set, + param_get_uint, ¶m_ops_requeue_timeout_min, 0644); +#endif +MODULE_PARM_DESC(mgc_requeue_timeout_min, "Minimal requeue time to refresh logs"); + static int __init mgc_init(void) { - return class_register_type(&mgc_obd_ops, NULL, true, NULL, + return class_register_type(&mgc_obd_ops, NULL, false, LUSTRE_MGC_NAME, NULL); }