X-Git-Url: https://git.whamcloud.com/?p=fs%2Flustre-release.git;a=blobdiff_plain;f=lustre%2Fmgc%2Fmgc_request.c;h=4a982263d0c875fb4cc41e84df6ea18b538f0132;hp=943ee3a7065a1de1a5fbd7b2f3b2af388b245f70;hb=e34950d29f1168a8a85f26e862eaec888b94350e;hpb=a7ff5d050ee7db0e80baac5fb3848ffcfa04dea6 diff --git a/lustre/mgc/mgc_request.c b/lustre/mgc/mgc_request.c index 943ee3a..4a98226 100644 --- a/lustre/mgc/mgc_request.c +++ b/lustre/mgc/mgc_request.c @@ -27,7 +27,6 @@ */ /* * This file is part of Lustre, http://www.lustre.org/ - * Lustre is a trademark of Sun Microsystems, Inc. * * lustre/mgc/mgc_request.c * @@ -54,7 +53,7 @@ #include "mgc_internal.h" static int mgc_name2resid(char *name, int len, struct ldlm_res_id *res_id, - int type) + enum mgs_cfg_type type) { __u64 resname = 0; @@ -73,14 +72,14 @@ static int mgc_name2resid(char *name, int len, struct ldlm_res_id *res_id, res_id->name[0] = cpu_to_le64(resname); /* XXX: unfortunately, sptlprc and config llog share one lock */ switch(type) { - case CONFIG_T_CONFIG: - case CONFIG_T_SPTLRPC: + case MGS_CFG_T_CONFIG: + case MGS_CFG_T_SPTLRPC: resname = 0; break; - case CONFIG_T_RECOVER: - case CONFIG_T_PARAMS: - case CONFIG_T_NODEMAP: - case CONFIG_T_BARRIER: + case MGS_CFG_T_RECOVER: + case MGS_CFG_T_PARAMS: + case MGS_CFG_T_NODEMAP: + case MGS_CFG_T_BARRIER: resname = type; break; default: @@ -92,7 +91,8 @@ static int mgc_name2resid(char *name, int len, struct ldlm_res_id *res_id, return 0; } -int mgc_fsname2resid(char *fsname, struct ldlm_res_id *res_id, int type) +int mgc_fsname2resid(char *fsname, struct ldlm_res_id *res_id, + enum mgs_cfg_type type) { /* fsname is at most 8 chars long, maybe contain "-". * e.g. "lustre", "SUN-000" */ @@ -100,7 +100,8 @@ int mgc_fsname2resid(char *fsname, struct ldlm_res_id *res_id, int type) } EXPORT_SYMBOL(mgc_fsname2resid); -int mgc_logname2resid(char *logname, struct ldlm_res_id *res_id, int type) +int mgc_logname2resid(char *logname, struct ldlm_res_id *res_id, + enum mgs_cfg_type type) { char *name_end; int len; @@ -156,8 +157,10 @@ static void config_log_put(struct config_llog_data *cld) config_log_put(cld->cld_params); config_log_put(cld->cld_nodemap); config_log_put(cld->cld_sptlrpc); - if (cld_is_sptlrpc(cld)) + if (cld_is_sptlrpc(cld)) { + cld->cld_stopping = 1; sptlrpc_conf_log_stop(cld->cld_logname); + } class_export_put(cld->cld_mgcexp); OBD_FREE(cld, sizeof(*cld) + strlen(cld->cld_logname) + 1); @@ -199,7 +202,7 @@ struct config_llog_data *config_log_find(char *logname, static struct config_llog_data *do_config_log_add(struct obd_device *obd, char *logname, - int type, + enum mgs_cfg_type type, struct config_llog_instance *cfg, struct super_block *sb) { @@ -259,15 +262,14 @@ static struct config_llog_data *config_recover_log_add(struct obd_device *obd, struct super_block *sb) { struct config_llog_instance lcfg = *cfg; - struct lustre_sb_info *lsi = s2lsi(sb); struct config_llog_data *cld; char logname[32]; - if (IS_OST(lsi)) + if (IS_OST(s2lsi(sb))) return NULL; /* for osp-on-ost, see lustre_start_osp() */ - if (IS_MDT(lsi) && lcfg.cfg_instance) + if (IS_MDT(s2lsi(sb)) && lcfg.cfg_instance) return NULL; /* We have to use different llog for clients and MDTs for DNE, @@ -275,7 +277,7 @@ static struct config_llog_data *config_recover_log_add(struct obd_device *obd, */ LASSERT(strlen(fsname) < sizeof(logname) / 2); strncpy(logname, fsname, sizeof(logname)); - if (IS_SERVER(lsi)) { /* mdt */ + if (IS_SERVER(s2lsi(sb))) { /* mdt */ LASSERT(lcfg.cfg_instance == 0); lcfg.cfg_instance = ll_get_cfg_instance(sb); strncat(logname, "-mdtir", sizeof(logname)); @@ -284,13 +286,14 @@ static struct config_llog_data *config_recover_log_add(struct obd_device *obd, strncat(logname, "-cliir", sizeof(logname)); } - cld = do_config_log_add(obd, logname, CONFIG_T_RECOVER, &lcfg, sb); + cld = do_config_log_add(obd, logname, MGS_CFG_T_RECOVER, &lcfg, sb); return cld; } -static struct config_llog_data *config_log_find_or_add(struct obd_device *obd, - char *logname, struct super_block *sb, int type, - struct config_llog_instance *cfg) +static struct config_llog_data * +config_log_find_or_add(struct obd_device *obd, char *logname, + struct super_block *sb, enum mgs_cfg_type type, + struct config_llog_instance *cfg) { struct config_llog_instance lcfg = *cfg; struct config_llog_data *cld; @@ -344,7 +347,7 @@ config_log_add(struct obd_device *obd, char *logname, if (cfg->cfg_sub_clds & CONFIG_SUB_SPTLRPC) { sptlrpc_cld = config_log_find_or_add(obd, seclogname, NULL, - CONFIG_T_SPTLRPC, cfg); + MGS_CFG_T_SPTLRPC, cfg); if (IS_ERR(sptlrpc_cld)) { CERROR("%s: can't create sptlrpc log %s: rc = %ld\n", obd->obd_name, seclogname, PTR_ERR(sptlrpc_cld)); @@ -354,7 +357,7 @@ config_log_add(struct obd_device *obd, char *logname, if (!IS_MGS(lsi) && cfg->cfg_sub_clds & CONFIG_SUB_NODEMAP) { nodemap_cld = config_log_find_or_add(obd, LUSTRE_NODEMAP_NAME, - NULL, CONFIG_T_NODEMAP, + NULL, MGS_CFG_T_NODEMAP, cfg); if (IS_ERR(nodemap_cld)) { rc = PTR_ERR(nodemap_cld); @@ -366,7 +369,7 @@ config_log_add(struct obd_device *obd, char *logname, if (cfg->cfg_sub_clds & CONFIG_SUB_PARAMS) { params_cld = config_log_find_or_add(obd, PARAMS_FILENAME, sb, - CONFIG_T_PARAMS, cfg); + MGS_CFG_T_PARAMS, cfg); if (IS_ERR(params_cld)) { rc = PTR_ERR(params_cld); CERROR("%s: can't create params log: rc = %d\n", @@ -379,7 +382,7 @@ config_log_add(struct obd_device *obd, char *logname, snprintf(seclogname + (ptr - logname), sizeof(seclogname) - 1, "-%s", BARRIER_FILENAME); barrier_cld = config_log_find_or_add(obd, seclogname, sb, - CONFIG_T_BARRIER, cfg); + MGS_CFG_T_BARRIER, cfg); if (IS_ERR(barrier_cld)) { rc = PTR_ERR(barrier_cld); CERROR("%s: can't create barrier log: rc = %d\n", @@ -388,7 +391,7 @@ config_log_add(struct obd_device *obd, char *logname, } } - cld = do_config_log_add(obd, logname, CONFIG_T_CONFIG, cfg, sb); + cld = do_config_log_add(obd, logname, MGS_CFG_T_CONFIG, cfg, sb); if (IS_ERR(cld)) { rc = PTR_ERR(cld); CERROR("%s: can't create log: rc = %d\n", @@ -519,16 +522,17 @@ static int config_log_end(char *logname, struct config_llog_instance *cfg) mutex_unlock(&cld->cld_lock); config_mark_cld_stop(cld_recover); + config_log_put(cld_recover); config_mark_cld_stop(cld_params); - config_mark_cld_stop(cld_barrier); - config_mark_cld_stop(cld_sptlrpc); - config_log_put(cld_params); - config_log_put(cld_recover); - /* don't set cld_stopping on nm lock as other targets may be active */ - config_log_put(cld_nodemap); + config_mark_cld_stop(cld_barrier); config_log_put(cld_barrier); + /* don't explicitly set cld_stopping on sptlrpc lock here, as other + * targets may be active, it will be done in config_log_put if necessary + */ config_log_put(cld_sptlrpc); + /* don't set cld_stopping on nm lock as other targets may be active */ + config_log_put(cld_nodemap); /* drop the ref from the find */ config_log_put(cld); @@ -546,15 +550,20 @@ int lprocfs_mgc_rd_ir_state(struct seq_file *m, void *data) struct obd_import *imp; struct obd_connect_data *ocd; struct config_llog_data *cld; + int rc = 0; ENTRY; LASSERT(obd); - LPROCFS_CLIMP_CHECK(obd); - imp = obd->u.cli.cl_import; - ocd = &imp->imp_connect_data; + with_imp_locked(obd, imp, rc) { + ocd = &imp->imp_connect_data; + + seq_printf(m, "imperative_recovery: %s\n", + OCD_HAS_FLAG(ocd, IMP_RECOV) ? + "ENABLED" : "DISABLED"); + } + if (rc) + RETURN(rc); - seq_printf(m, "imperative_recovery: %s\n", - OCD_HAS_FLAG(ocd, IMP_RECOV) ? "ENABLED" : "DISABLED"); seq_printf(m, "client_state:\n"); spin_lock(&config_list_lock); @@ -567,7 +576,6 @@ int lprocfs_mgc_rd_ir_state(struct seq_file *m, void *data) } spin_unlock(&config_list_lock); - LPROCFS_CLIMP_EXIT(obd); RETURN(0); } @@ -610,13 +618,6 @@ static void do_requeue(struct config_llog_data *cld) EXIT; } -/* this timeout represents how many seconds MGC should wait before - * requeue config and recover lock to the MGS. We need to randomize this - * in order to not flood the MGS. - */ -#define MGC_TIMEOUT_MIN_SECONDS 5 -#define MGC_TIMEOUT_RAND_CENTISEC 500 - static int mgc_requeue_thread(void *data) { int rc = 0; @@ -630,7 +631,6 @@ static int mgc_requeue_thread(void *data) rq_state |= RQ_RUNNING; while (!(rq_state & RQ_STOP)) { struct config_llog_data *cld, *cld_prev; - int rand = prandom_u32_max(MGC_TIMEOUT_RAND_CENTISEC); int to; /* Any new or requeued lostlocks will change the state */ @@ -646,11 +646,11 @@ static int mgc_requeue_thread(void *data) * caused the lock revocation to finish its setup, plus some * random so everyone doesn't try to reconnect at once. */ - to = cfs_time_seconds(MGC_TIMEOUT_MIN_SECONDS * 100 + rand); - /* rand is centi-seconds */ + to = mgc_requeue_timeout_min == 0 ? 1 : mgc_requeue_timeout_min; + to = cfs_time_seconds(mgc_requeue_timeout_min) + + prandom_u32_max(cfs_time_seconds(to)); wait_event_idle_timeout(rq_waitq, - rq_state & (RQ_STOP | RQ_PRECLEANUP), - to/100); + rq_state & (RQ_STOP | RQ_PRECLEANUP), to); /* * iterate & processing through the list. for each cld, process @@ -737,6 +737,7 @@ static void mgc_requeue_add(struct config_llog_data *cld) } /********************** class fns **********************/ +#ifdef HAVE_SERVER_SUPPORT static int mgc_local_llog_init(const struct lu_env *env, struct obd_device *obd, struct obd_device *disk) @@ -796,7 +797,7 @@ static int mgc_fs_setup(const struct lu_env *env, struct obd_device *obd, rc = local_oid_storage_init(env, lsi->lsi_dt_dev, &fid, &cli->cl_mgc_los); if (rc) - RETURN(rc); + GOTO(out_mutex, rc); rc = dt_root_get(env, lsi->lsi_dt_dev, &rfid); if (rc) @@ -835,6 +836,7 @@ out_llog: out_los: if (rc < 0) { local_oid_storage_fini(env, cli->cl_mgc_los); +out_mutex: cli->cl_mgc_los = NULL; mutex_unlock(&cli->cl_mgc_mutex); } @@ -861,6 +863,7 @@ static int mgc_fs_cleanup(const struct lu_env *env, struct obd_device *obd) RETURN(0); } +#endif /* HAVE_SERVER_SUPPORT */ static int mgc_llog_init(const struct lu_env *env, struct obd_device *obd) { @@ -1073,45 +1076,6 @@ static int mgc_blocking_ast(struct ldlm_lock *lock, struct ldlm_lock_desc *desc, #define MGC_TARGET_REG_LIMIT_MAX RECONNECT_DELAY_MAX #define MGC_SEND_PARAM_LIMIT 10 -#if LUSTRE_VERSION_CODE < OBD_OCD_VERSION(2, 13, 53, 0) -/* Send parameter to MGS*/ -static int mgc_set_mgs_param(struct obd_export *exp, - struct mgs_send_param *msp) -{ - struct ptlrpc_request *req; - struct mgs_send_param *req_msp, *rep_msp; - int rc; - ENTRY; - - req = ptlrpc_request_alloc_pack(class_exp2cliimp(exp), - &RQF_MGS_SET_INFO, LUSTRE_MGS_VERSION, - MGS_SET_INFO); - if (!req) - RETURN(-ENOMEM); - - req_msp = req_capsule_client_get(&req->rq_pill, &RMF_MGS_SEND_PARAM); - if (!req_msp) { - ptlrpc_req_finished(req); - RETURN(-ENOMEM); - } - - memcpy(req_msp, msp, sizeof(*req_msp)); - ptlrpc_request_set_replen(req); - - /* Limit how long we will wait for the enqueue to complete */ - req->rq_delay_limit = MGC_SEND_PARAM_LIMIT; - rc = ptlrpc_queue_wait(req); - if (!rc) { - rep_msp = req_capsule_server_get(&req->rq_pill, &RMF_MGS_SEND_PARAM); - memcpy(msp, rep_msp, sizeof(*rep_msp)); - } - - ptlrpc_req_finished(req); - - RETURN(rc); -} -#endif - /* Take a config lock so we can get cancel notifications */ static int mgc_enqueue(struct obd_export *exp, enum ldlm_type type, union ldlm_policy_data *policy, enum ldlm_mode mode, @@ -1132,6 +1096,9 @@ static int mgc_enqueue(struct obd_export *exp, enum ldlm_type type, int rc; ENTRY; + if (!exp) + RETURN(-EBADR); + CDEBUG(D_MGC, "Enqueue for %s (res %#llx)\n", cld->cld_logname, cld->cld_resid.name[0]); @@ -1146,12 +1113,11 @@ static int mgc_enqueue(struct obd_export *exp, enum ldlm_type type, req_capsule_set_size(&req->rq_pill, &RMF_DLM_LVB, RCL_SERVER, 0); ptlrpc_request_set_replen(req); - /* check if this is server or client */ - if (cld->cld_cfg.cfg_sb) { - struct lustre_sb_info *lsi = s2lsi(cld->cld_cfg.cfg_sb); - if (lsi && IS_SERVER(lsi)) - short_limit = 1; - } + /* check if this is server or client */ + if (cld->cld_cfg.cfg_sb && + IS_SERVER(s2lsi(cld->cld_cfg.cfg_sb))) + short_limit = 1; + /* Limit how long we will wait for the enqueue to complete */ req->rq_delay_limit = short_limit ? 5 : MGC_ENQUEUE_LIMIT; rc = ldlm_cli_enqueue(exp, &req, &einfo, &cld->cld_resid, NULL, flags, @@ -1183,26 +1149,27 @@ static void mgc_notify_active(struct obd_device *unused) /* TODO: Help the MGS rebuild nidtbl. -jay */ } +#ifdef HAVE_SERVER_SUPPORT /* Send target_reg message to MGS */ static int mgc_target_register(struct obd_export *exp, - struct mgs_target_info *mti) + struct mgs_target_info *mti) { - struct ptlrpc_request *req; - struct mgs_target_info *req_mti, *rep_mti; - int rc; - ENTRY; + struct ptlrpc_request *req; + struct mgs_target_info *req_mti, *rep_mti; + int rc; + ENTRY; - req = ptlrpc_request_alloc_pack(class_exp2cliimp(exp), - &RQF_MGS_TARGET_REG, LUSTRE_MGS_VERSION, - MGS_TARGET_REG); - if (req == NULL) - RETURN(-ENOMEM); + req = ptlrpc_request_alloc_pack(class_exp2cliimp(exp), + &RQF_MGS_TARGET_REG, LUSTRE_MGS_VERSION, + MGS_TARGET_REG); + if (req == NULL) + RETURN(-ENOMEM); - req_mti = req_capsule_client_get(&req->rq_pill, &RMF_MGS_TARGET_INFO); - if (!req_mti) { - ptlrpc_req_finished(req); - RETURN(-ENOMEM); - } + req_mti = req_capsule_client_get(&req->rq_pill, &RMF_MGS_TARGET_INFO); + if (!req_mti) { + ptlrpc_req_finished(req); + RETURN(-ENOMEM); + } memcpy(req_mti, mti, sizeof(*req_mti)); ptlrpc_request_set_replen(req); @@ -1218,17 +1185,21 @@ static int mgc_target_register(struct obd_export *exp, req->rq_delay_limit = MGC_TARGET_REG_LIMIT_MAX; rc = ptlrpc_queue_wait(req); - if (!rc) { - rep_mti = req_capsule_server_get(&req->rq_pill, - &RMF_MGS_TARGET_INFO); - memcpy(mti, rep_mti, sizeof(*rep_mti)); - CDEBUG(D_MGC, "register %s got index = %d\n", - mti->mti_svname, mti->mti_stripe_index); + if (ptlrpc_client_replied(req)) { + rep_mti = req_capsule_server_get(&req->rq_pill, + &RMF_MGS_TARGET_INFO); + if (rep_mti) + memcpy(mti, rep_mti, sizeof(*rep_mti)); + } + if (!rc) { + CDEBUG(D_MGC, "register %s got index = %d\n", + mti->mti_svname, mti->mti_stripe_index); } ptlrpc_req_finished(req); RETURN(rc); } +#endif /* HAVE_SERVER_SUPPORT */ static int mgc_set_info_async(const struct lu_env *env, struct obd_export *exp, u32 keylen, void *key, @@ -1260,17 +1231,19 @@ static int mgc_set_info_async(const struct lu_env *env, struct obd_export *exp, RETURN(0); } - /* FIXME move this to mgc_process_config */ - if (KEY_IS(KEY_REGISTER_TARGET)) { - struct mgs_target_info *mti; - if (vallen != sizeof(struct mgs_target_info)) - RETURN(-EINVAL); - mti = (struct mgs_target_info *)val; - CDEBUG(D_MGC, "register_target %s %#x\n", - mti->mti_svname, mti->mti_flags); - rc = mgc_target_register(exp, mti); - RETURN(rc); - } +#ifdef HAVE_SERVER_SUPPORT + /* FIXME move this to mgc_process_config */ + if (KEY_IS(KEY_REGISTER_TARGET)) { + struct mgs_target_info *mti; + + if (vallen != sizeof(struct mgs_target_info)) + RETURN(-EINVAL); + mti = (struct mgs_target_info *)val; + CDEBUG(D_MGC, "register_target %s %#x\n", + mti->mti_svname, mti->mti_flags); + rc = mgc_target_register(exp, mti); + RETURN(rc); + } if (KEY_IS(KEY_SET_FS)) { struct super_block *sb = (struct super_block *)val; @@ -1286,14 +1259,6 @@ static int mgc_set_info_async(const struct lu_env *env, struct obd_export *exp, rc = mgc_fs_cleanup(env, exp->exp_obd); RETURN(rc); } -#if LUSTRE_VERSION_CODE < OBD_OCD_VERSION(2, 13, 53, 0) - if (KEY_IS(KEY_SET_INFO)) { - struct mgs_send_param *msp; - - msp = (struct mgs_send_param *)val; - rc = mgc_set_mgs_param(exp, msp); - RETURN(rc); - } #endif if (KEY_IS(KEY_MGSSEC)) { struct client_obd *cli = &exp->exp_obd->u.cli; @@ -1413,75 +1378,74 @@ static int mgc_apply_recover_logs(struct obd_device *mgc, void *data, int datalen, bool mne_swab) { struct config_llog_instance *cfg = &cld->cld_cfg; - struct lustre_sb_info *lsi = s2lsi(cfg->cfg_sb); struct mgs_nidtbl_entry *entry; struct lustre_cfg *lcfg; struct lustre_cfg_bufs bufs; u64 prev_version = 0; - char *inst; + char inst[MTI_NAME_MAXLEN + 1]; char *buf; int bufsz; int pos = 0; int rc = 0; int off = 0; + unsigned long dynamic_nids; ENTRY; LASSERT(cfg->cfg_instance != 0); LASSERT(ll_get_cfg_instance(cfg->cfg_sb) == cfg->cfg_instance); - OBD_ALLOC(inst, PAGE_SIZE); - if (inst == NULL) - RETURN(-ENOMEM); + /* get dynamic nids setting */ + dynamic_nids = mgc->obd_dynamic_nids; - if (!IS_SERVER(lsi)) { - pos = snprintf(inst, PAGE_SIZE, "%016lx", cfg->cfg_instance); - if (pos >= PAGE_SIZE) { - OBD_FREE(inst, PAGE_SIZE); + if (!IS_SERVER(s2lsi(cfg->cfg_sb))) { + pos = snprintf(inst, sizeof(inst), "%016lx", cfg->cfg_instance); + if (pos >= PAGE_SIZE) return -E2BIG; - } #ifdef HAVE_SERVER_SUPPORT } else { + struct lustre_sb_info *lsi = s2lsi(cfg->cfg_sb); + LASSERT(IS_MDT(lsi)); rc = server_name2svname(lsi->lsi_svname, inst, NULL, - PAGE_SIZE); - if (rc) { - OBD_FREE(inst, PAGE_SIZE); + sizeof(inst)); + if (rc) RETURN(-EINVAL); - } - pos = strlen(inst); #endif /* HAVE_SERVER_SUPPORT */ } - ++pos; - buf = inst + pos; - bufsz = PAGE_SIZE - pos; - - while (datalen > 0) { - int entry_len = sizeof(*entry); - int is_ost, i; - struct obd_device *obd; - char *obdname; - char *cname; - char *params; - char *uuid; - - rc = -EINVAL; - if (datalen < sizeof(*entry)) - break; - - entry = (typeof(entry))(data + off); - - /* sanity check */ - if (entry->mne_nid_type != 0) /* only support type 0 for ipv4 */ - break; - if (entry->mne_nid_count == 0) /* at least one nid entry */ - break; - if (entry->mne_nid_size != sizeof(lnet_nid_t)) - break; - - entry_len += entry->mne_nid_count * entry->mne_nid_size; - if (datalen < entry_len) /* must have entry_len at least */ - break; + OBD_ALLOC(buf, PAGE_SIZE); + if (!buf) + return -ENOMEM; + bufsz = PAGE_SIZE; + pos = 0; + + while (datalen > 0) { + int entry_len = sizeof(*entry); + int is_ost; + struct obd_device *obd; + struct obd_import *imp; + char *obdname; + char *cname; + char *params; + char *uuid; + + rc = -EINVAL; + if (datalen < sizeof(*entry)) + break; + + entry = (typeof(entry))(data + off); + + /* sanity check */ + if (entry->mne_nid_type != 0) /* only support type 0 for ipv4 */ + break; + if (entry->mne_nid_count == 0) /* at least one nid entry */ + break; + if (entry->mne_nid_size != sizeof(lnet_nid_t)) + break; + + entry_len += entry->mne_nid_count * entry->mne_nid_size; + if (datalen < entry_len) /* must have entry_len at least */ + break; /* Keep this swab for normal mixed endian handling. LU-1644 */ if (mne_swab) @@ -1494,102 +1458,130 @@ static int mgc_apply_recover_logs(struct obd_device *mgc, if (entry->mne_length < entry_len) break; - off += entry->mne_length; - datalen -= entry->mne_length; - if (datalen < 0) - break; + off += entry->mne_length; + datalen -= entry->mne_length; + if (datalen < 0) + break; - if (entry->mne_version > max_version) { - CERROR("entry index(%lld) is over max_index(%lld)\n", - entry->mne_version, max_version); - break; - } + if (entry->mne_version > max_version) { + CERROR("entry index(%lld) is over max_index(%lld)\n", + entry->mne_version, max_version); + break; + } - if (prev_version >= entry->mne_version) { - CERROR("index unsorted, prev %lld, now %lld\n", - prev_version, entry->mne_version); - break; - } - prev_version = entry->mne_version; + if (prev_version >= entry->mne_version) { + CERROR("index unsorted, prev %lld, now %lld\n", + prev_version, entry->mne_version); + break; + } + prev_version = entry->mne_version; - /* - * Write a string with format "nid::instance" to - * lustre//--/import. - */ + /* + * Write a string with format "nid::instance" to + * lustre//--/import. + */ - is_ost = entry->mne_type == LDD_F_SV_TYPE_OST; - memset(buf, 0, bufsz); - obdname = buf; - pos = 0; - - /* lustre-OST0001-osc- */ - strcpy(obdname, cld->cld_logname); - cname = strrchr(obdname, '-'); - if (cname == NULL) { - CERROR("mgc %s: invalid logname %s\n", - mgc->obd_name, obdname); - break; - } + is_ost = entry->mne_type == LDD_F_SV_TYPE_OST; + memset(buf, 0, bufsz); + obdname = buf; + pos = 0; + + /* lustre-OST0001-osc- */ + strcpy(obdname, cld->cld_logname); + cname = strrchr(obdname, '-'); + if (cname == NULL) { + CERROR("mgc %s: invalid logname %s\n", + mgc->obd_name, obdname); + break; + } - pos = cname - obdname; - obdname[pos] = 0; - pos += sprintf(obdname + pos, "-%s%04x", - is_ost ? "OST" : "MDT", entry->mne_index); + pos = cname - obdname; + obdname[pos] = 0; + pos += sprintf(obdname + pos, "-%s%04x", + is_ost ? "OST" : "MDT", entry->mne_index); - cname = is_ost ? "osc" : "mdc", - pos += sprintf(obdname + pos, "-%s-%s", cname, inst); - lustre_cfg_bufs_reset(&bufs, obdname); + cname = is_ost ? "osc" : "mdc", + pos += snprintf(obdname + pos, bufsz, "-%s-%s", cname, inst); + lustre_cfg_bufs_reset(&bufs, obdname); - /* find the obd by obdname */ - obd = class_name2obd(obdname); - if (obd == NULL) { - CDEBUG(D_INFO, "mgc %s: cannot find obdname %s\n", - mgc->obd_name, obdname); + /* find the obd by obdname */ + obd = class_name2obd(obdname); + if (obd == NULL) { + CDEBUG(D_INFO, "mgc %s: cannot find obdname %s\n", + mgc->obd_name, obdname); rc = 0; - /* this is a safe race, when the ost is starting up...*/ - continue; - } - - /* osc.import = "connection=::" */ - ++pos; - params = buf + pos; - pos += sprintf(params, "%s.import=%s", cname, "connection="); - uuid = buf + pos; + /* this is a safe race, when the ost is starting up...*/ + continue; + } - down_read(&obd->u.cli.cl_sem); - if (obd->u.cli.cl_import == NULL) { + /* osc.import = "connection=::" */ + ++pos; + params = buf + pos; + pos += sprintf(params, "%s.import=%s", cname, "connection="); + uuid = buf + pos; + + with_imp_locked(obd, imp, rc) { + /* iterate all nids to find one */ + /* find uuid by nid */ + /* create import entries if they don't exist */ + rc = client_import_add_nids_to_conn( + imp, entry->u.nids, entry->mne_nid_count, + (struct obd_uuid *)uuid); + + if (rc == -ENOENT && dynamic_nids) { + /* create a new connection for this import */ + char *primary_nid = + libcfs_nid2str(entry->u.nids[0]); + int prim_nid_len = strlen(primary_nid) + 1; + struct obd_uuid server_uuid; + + if (prim_nid_len > UUID_MAX) + goto fail; + strncpy(server_uuid.uuid, primary_nid, + prim_nid_len); + + CDEBUG(D_INFO, "Adding a connection for %s\n", + primary_nid); + + rc = client_import_dyn_add_conn( + imp, &server_uuid, entry->u.nids[0], 1); + if (rc < 0) { + CERROR("%s: Failed to add new connection with NID '%s' to import: rc = %d\n", + obd->obd_name, primary_nid, rc); + goto fail; + } + rc = client_import_add_nids_to_conn( + imp, entry->u.nids, + entry->mne_nid_count, + (struct obd_uuid *)uuid); + if (rc < 0) { + CERROR("%s: failed to lookup UUID: rc = %d\n", + obd->obd_name, rc); + goto fail; + } + } +fail:; + } + if (rc == -ENODEV) { /* client does not connect to the OST yet */ - up_read(&obd->u.cli.cl_sem); rc = 0; continue; } - /* iterate all nids to find one */ - /* find uuid by nid */ - rc = -ENOENT; - for (i = 0; i < entry->mne_nid_count; i++) { - rc = client_import_find_conn(obd->u.cli.cl_import, - entry->u.nids[i], - (struct obd_uuid *)uuid); - if (rc == 0) - break; + if (rc < 0 && rc != -ENOSPC) { + CERROR("mgc: cannot find UUID by nid '%s': rc = %d\n", + libcfs_nid2str(entry->u.nids[0]), rc); + break; } - up_read(&obd->u.cli.cl_sem); - if (rc < 0) { - CERROR("mgc: cannot find uuid by nid %s\n", - libcfs_nid2str(entry->u.nids[0])); - break; - } - - CDEBUG(D_INFO, "Find uuid %s by nid %s\n", - uuid, libcfs_nid2str(entry->u.nids[0])); + CDEBUG(D_INFO, "Found UUID '%s' by NID '%s'\n", + uuid, libcfs_nid2str(entry->u.nids[0])); - pos += strlen(uuid); - pos += sprintf(buf + pos, "::%u", entry->mne_instance); - LASSERT(pos < bufsz); + pos += strlen(uuid); + pos += sprintf(buf + pos, "::%u", entry->mne_instance); + LASSERT(pos < bufsz); - lustre_cfg_bufs_set_string(&bufs, 1, params); + lustre_cfg_bufs_set_string(&bufs, 1, params); OBD_ALLOC(lcfg, lustre_cfg_len(bufs.lcfg_bufcount, bufs.lcfg_buflen)); @@ -1600,20 +1592,21 @@ static int mgc_apply_recover_logs(struct obd_device *mgc, lustre_cfg_init(lcfg, LCFG_PARAM, &bufs); CDEBUG(D_INFO, "ir apply logs %lld/%lld for %s -> %s\n", - prev_version, max_version, obdname, params); + prev_version, max_version, obdname, params); - rc = class_process_config(lcfg); + rc = class_process_config(lcfg); OBD_FREE(lcfg, lustre_cfg_len(lcfg->lcfg_bufcount, lcfg->lcfg_buflens)); - if (rc) - CDEBUG(D_INFO, "process config for %s error %d\n", - obdname, rc); + if (rc) + CDEBUG(D_INFO, "process config for %s error %d\n", + obdname, rc); - /* continue, even one with error */ - } + /* continue, even one with error */ + } - OBD_FREE(inst, PAGE_SIZE); - RETURN(rc); + OBD_FREE(buf, PAGE_SIZE); + + RETURN(rc); } /** @@ -1647,28 +1640,28 @@ static int mgc_process_recover_nodemap_log(struct obd_device *obd, /* don't need to get local config */ if (cld_is_nodemap(cld) && - (LNET_NETTYP(LNET_NIDNET(mgc_conn->c_peer.nid)) == LOLND)) + LNetIsPeerLocal(&mgc_conn->c_peer.nid)) GOTO(out, rc = 0); - /* allocate buffer for bulk transfer. - * if this is the first time for this mgs to read logs, - * CONFIG_READ_NRPAGES_INIT will be used since it will read all logs - * once; otherwise, it only reads increment of logs, this should be - * small and CONFIG_READ_NRPAGES will be used. - */ - nrpages = CONFIG_READ_NRPAGES; + /* allocate buffer for bulk transfer. + * if this is the first time for this mgs to read logs, + * CONFIG_READ_NRPAGES_INIT will be used since it will read all logs + * once; otherwise, it only reads increment of logs, this should be + * small and CONFIG_READ_NRPAGES will be used. + */ + nrpages = CONFIG_READ_NRPAGES; if (cfg->cfg_last_idx == 0 || cld_is_nodemap(cld)) - nrpages = CONFIG_READ_NRPAGES_INIT; + nrpages = CONFIG_READ_NRPAGES_INIT; - OBD_ALLOC(pages, sizeof(*pages) * nrpages); - if (pages == NULL) - GOTO(out, rc = -ENOMEM); + OBD_ALLOC_PTR_ARRAY_LARGE(pages, nrpages); + if (pages == NULL) + GOTO(out, rc = -ENOMEM); - for (i = 0; i < nrpages; i++) { + for (i = 0; i < nrpages; i++) { pages[i] = alloc_page(GFP_KERNEL); - if (pages[i] == NULL) - GOTO(out, rc = -ENOMEM); - } + if (pages[i] == NULL) + GOTO(out, rc = -ENOMEM); + } again: #ifdef HAVE_SERVER_SUPPORT @@ -1710,7 +1703,7 @@ again: /* allocate bulk transfer descriptor */ desc = ptlrpc_prep_bulk_imp(req, nrpages, 1, - PTLRPC_BULK_PUT_SINK | PTLRPC_BULK_BUF_KIOV, + PTLRPC_BULK_PUT_SINK, MGS_BULK_PORTAL, &ptlrpc_bulk_kiov_pin_ops); if (desc == NULL) @@ -1767,7 +1760,7 @@ again: GOTO(out, rc); } - mne_swab = ptlrpc_rep_need_swab(req); + mne_swab = req_capsule_rep_need_swab(&req->rq_pill); /* When a nodemap config is received, we build a new nodemap config, * with new nodemap structs. We keep track of the most recently added @@ -1830,11 +1823,12 @@ out: break; __free_page(pages[i]); } - OBD_FREE(pages, sizeof(*pages) * nrpages); + OBD_FREE_PTR_ARRAY_LARGE(pages, nrpages); } return rc; } +#ifdef HAVE_SERVER_SUPPORT static int mgc_barrier_glimpse_ast(struct ldlm_lock *lock, void *data) { struct config_llog_data *cld = lock->l_ast_data; @@ -1893,36 +1887,14 @@ out: return rc; } -/* local_only means it cannot get remote llogs */ -static int mgc_process_cfg_log(struct obd_device *mgc, - struct config_llog_data *cld, int local_only) +static int +mgc_process_server_cfg_log(struct lu_env *env, struct llog_ctxt **ctxt, + struct lustre_sb_info *lsi, struct obd_device *mgc, + struct config_llog_data *cld, int local_only) { - struct llog_ctxt *ctxt, *lctxt = NULL; - struct client_obd *cli = &mgc->u.cli; - struct lustre_sb_info *lsi = NULL; - int rc = 0; - struct lu_env *env; - - ENTRY; - - LASSERT(cld); - LASSERT(mutex_is_locked(&cld->cld_lock)); - - if (cld->cld_cfg.cfg_sb) - lsi = s2lsi(cld->cld_cfg.cfg_sb); - - OBD_ALLOC_PTR(env); - if (env == NULL) - RETURN(-ENOMEM); - - rc = lu_env_init(env, LCT_MG_THREAD); - if (rc) - GOTO(out_free, rc); - - ctxt = llog_get_context(mgc, LLOG_CONFIG_REPL_CTXT); - LASSERT(ctxt); - - lctxt = llog_get_context(mgc, LLOG_CONFIG_ORIG_CTXT); + struct llog_ctxt *lctxt = llog_get_context(mgc, LLOG_CONFIG_ORIG_CTXT); + struct client_obd *cli = &mgc->u.cli; + int rc = 0; /* Copy the setup log locally if we can. Don't mess around if we're * running an MGS though (logs are already local). */ @@ -1930,32 +1902,42 @@ static int mgc_process_cfg_log(struct obd_device *mgc, cli->cl_mgc_configs_dir != NULL && lu2dt_dev(cli->cl_mgc_configs_dir->do_lu.lo_dev) == lsi->lsi_dt_dev) { - if (!local_only && !lsi->lsi_dt_dev->dd_rdonly) + if (!local_only && !lsi->lsi_dt_dev->dd_rdonly) { /* Only try to copy log if we have the lock. */ - rc = mgc_llog_local_copy(env, mgc, ctxt, lctxt, + CDEBUG(D_INFO, "%s: copy local log %s\n", + mgc->obd_name, cld->cld_logname); + + rc = mgc_llog_local_copy(env, mgc, *ctxt, lctxt, cld->cld_logname); + if (!rc) + lsi->lsi_flags &= ~LDD_F_NO_LOCAL_LOGS; + } if (local_only || rc) { + if (unlikely(lsi->lsi_flags & LDD_F_NO_LOCAL_LOGS) || + rc) { + CWARN("%s: local log %s are not valid and/or remote logs are not accessbile rc = %d\n", + mgc->obd_name, cld->cld_logname, rc); + GOTO(out_pop, rc = -EIO); + } + if (strcmp(cld->cld_logname, PARAMS_FILENAME) != 0 && llog_is_empty(env, lctxt, cld->cld_logname)) { - LCONSOLE_ERROR_MSG(0x13a, "Failed to get MGS " - "log %s and no local copy." - "\n", cld->cld_logname); + LCONSOLE_ERROR_MSG(0x13a, "Failed to get MGS log %s and no local copy.\n", + cld->cld_logname); GOTO(out_pop, rc = -ENOENT); } - CDEBUG(D_MGC, "Failed to get MGS log %s, using local " - "copy for now, will try to update later.\n", - cld->cld_logname); + CDEBUG(D_MGC, "%s: Failed to get MGS log %s, using local copy for now, will try to update later.\n", + mgc->obd_name, cld->cld_logname); rc = 0; } /* Now, whether we copied or not, start using the local llog. * If we failed to copy, we'll start using whatever the old * log has. */ - llog_ctxt_put(ctxt); - ctxt = lctxt; + llog_ctxt_put(*ctxt); + *ctxt = lctxt; lctxt = NULL; - } else { - if (local_only) /* no local log at client side */ - GOTO(out_pop, rc = -EIO); + } else if (local_only) { /* no local log at client side */ + GOTO(out_pop, rc = -EIO); } rc = -EAGAIN; @@ -1972,8 +1954,51 @@ static int mgc_process_cfg_log(struct obd_device *mgc, &cld->cld_cfg); llog_ctxt_put(rctxt); } +out_pop: + if (lctxt) + __llog_ctxt_put(env, lctxt); + return rc; +} +#else /* !HAVE_SERVER_SUPPORT */ +#define mgc_barrier_glimpse_ast NULL +#endif /* HAVE_SERVER_SUPPORT */ +/* local_only means it cannot get remote llogs */ +static int mgc_process_cfg_log(struct obd_device *mgc, + struct config_llog_data *cld, int local_only) +{ + struct llog_ctxt *ctxt; + struct lustre_sb_info *lsi = NULL; + int rc = 0; + struct lu_env *env; + + ENTRY; + LASSERT(cld); + LASSERT(mutex_is_locked(&cld->cld_lock)); + + if (cld->cld_cfg.cfg_sb) + lsi = s2lsi(cld->cld_cfg.cfg_sb); + + OBD_ALLOC_PTR(env); + if (!env) + RETURN(-ENOMEM); + + rc = lu_env_init(env, LCT_MG_THREAD); + if (rc) + GOTO(out_free, rc); + + ctxt = llog_get_context(mgc, LLOG_CONFIG_REPL_CTXT); + LASSERT(ctxt); +#ifdef HAVE_SERVER_SUPPORT + rc = mgc_process_server_cfg_log(env, &ctxt, lsi, mgc, cld, + local_only); + if (rc == -EIO && local_only) + GOTO(out_pop, rc); if (rc && rc != -ENOENT) +#else + if (local_only) + GOTO(out_pop, rc); +#endif rc = class_config_parse_llog(env, ctxt, cld->cld_logname, &cld->cld_cfg); @@ -1989,9 +2014,6 @@ static int mgc_process_cfg_log(struct obd_device *mgc, out_pop: __llog_ctxt_put(env, ctxt); - if (lctxt) - __llog_ctxt_put(env, lctxt); - lu_env_fini(env); out_free: OBD_FREE_PTR(env); @@ -2172,25 +2194,27 @@ static int mgc_process_config(struct obd_device *obd, size_t len, void *buf) ENTRY; switch(lcfg->lcfg_command) { - case LCFG_LOV_ADD_OBD: { - /* Overloading this cfg command: register a new target */ - struct mgs_target_info *mti; - - if (LUSTRE_CFG_BUFLEN(lcfg, 1) != - sizeof(struct mgs_target_info)) - GOTO(out, rc = -EINVAL); - - mti = (struct mgs_target_info *)lustre_cfg_buf(lcfg, 1); - CDEBUG(D_MGC, "add_target %s %#x\n", - mti->mti_svname, mti->mti_flags); - rc = mgc_target_register(obd->u.cli.cl_mgc_mgsexp, mti); - break; - } - case LCFG_LOV_DEL_OBD: - /* Unregister has no meaning at the moment. */ - CERROR("lov_del_obd unimplemented\n"); - rc = -ENOSYS; - break; +#ifdef HAVE_SERVER_SUPPORT + case LCFG_LOV_ADD_OBD: { + /* Overloading this cfg command: register a new target */ + struct mgs_target_info *mti; + + if (LUSTRE_CFG_BUFLEN(lcfg, 1) != + sizeof(struct mgs_target_info)) + GOTO(out, rc = -EINVAL); + + mti = lustre_cfg_buf(lcfg, 1); + CDEBUG(D_MGC, "add_target %s %#x\n", + mti->mti_svname, mti->mti_flags); + rc = mgc_target_register(obd->u.cli.cl_mgc_mgsexp, mti); + break; + } + case LCFG_LOV_DEL_OBD: + /* Unregister has no meaning at the moment. */ + CERROR("lov_del_obd unimplemented\n"); + rc = -EINVAL; + break; +#endif case LCFG_SPTLRPC_CONF: { rc = sptlrpc_process_config(lcfg); break; @@ -2281,9 +2305,43 @@ static const struct obd_ops mgc_obd_ops = { .o_process_config = mgc_process_config, }; +static int mgc_param_requeue_timeout_min_set(const char *val, + cfs_kernel_param_arg_t *kp) +{ + int rc; + unsigned int num; + + rc = kstrtouint(val, 0, &num); + if (rc < 0) + return rc; + if (num > 120) + return -EINVAL; + + mgc_requeue_timeout_min = num; + + return 0; +} + +static const struct kernel_param_ops param_ops_requeue_timeout_min = { + .set = mgc_param_requeue_timeout_min_set, + .get = param_get_uint, +}; + +#define param_check_requeue_timeout_min(name, p) \ + __param_check(name, p, unsigned int) + +unsigned int mgc_requeue_timeout_min = MGC_TIMEOUT_MIN_SECONDS; +#ifdef HAVE_KERNEL_PARAM_OPS +module_param(mgc_requeue_timeout_min, requeue_timeout_min, 0644); +#else +module_param_call(mgc_requeue_timeout_min, mgc_param_requeue_timeout_min_set, + param_get_uint, ¶m_ops_requeue_timeout_min, 0644); +#endif +MODULE_PARM_DESC(mgc_requeue_timeout_min, "Minimal requeue time to refresh logs"); + static int __init mgc_init(void) { - return class_register_type(&mgc_obd_ops, NULL, false, NULL, + return class_register_type(&mgc_obd_ops, NULL, false, LUSTRE_MGC_NAME, NULL); }