*
* You should have received a copy of the GNU General Public License
* version 2 along with this program; If not, see
- * http://www.sun.com/software/products/lustre/docs/GPLv2.pdf
- *
- * Please contact Sun Microsystems, Inc., 4150 Network Circle, Santa Clara,
- * CA 95054 USA or visit www.sun.com if you need additional information or
- * have any questions.
+ * http://www.gnu.org/licenses/gpl-2.0.html
*
* GPL HEADER END
*/
* Copyright (c) 2007, 2010, Oracle and/or its affiliates. All rights reserved.
* Use is subject to license terms.
*
- * Copyright (c) 2011, 2015, Intel Corporation.
+ * Copyright (c) 2011, 2017, Intel Corporation.
*/
/*
* This file is part of Lustre, http://www.lustre.org/
- * Lustre is a trademark of Sun Microsystems, Inc.
*
* lustre/mgc/mgc_request.c
*
#include <linux/module.h>
#include <linux/kthread.h>
+#include <linux/random.h>
#include <dt_object.h>
#include <lprocfs_status.h>
#include <lustre_nodemap.h>
#include <lustre_swab.h>
#include <obd_class.h>
+#include <lustre_barrier.h>
#include "mgc_internal.h"
static int mgc_name2resid(char *name, int len, struct ldlm_res_id *res_id,
- int type)
+ enum mgs_cfg_type type)
{
__u64 resname = 0;
res_id->name[0] = cpu_to_le64(resname);
/* XXX: unfortunately, sptlprc and config llog share one lock */
switch(type) {
- case CONFIG_T_CONFIG:
- case CONFIG_T_SPTLRPC:
+ case MGS_CFG_T_CONFIG:
+ case MGS_CFG_T_SPTLRPC:
resname = 0;
break;
- case CONFIG_T_RECOVER:
- case CONFIG_T_PARAMS:
- case CONFIG_T_NODEMAP:
+ case MGS_CFG_T_RECOVER:
+ case MGS_CFG_T_PARAMS:
+ case MGS_CFG_T_NODEMAP:
+ case MGS_CFG_T_BARRIER:
resname = type;
break;
default:
return 0;
}
-int mgc_fsname2resid(char *fsname, struct ldlm_res_id *res_id, int type)
+int mgc_fsname2resid(char *fsname, struct ldlm_res_id *res_id,
+ enum mgs_cfg_type type)
{
/* fsname is at most 8 chars long, maybe contain "-".
* e.g. "lustre", "SUN-000" */
}
EXPORT_SYMBOL(mgc_fsname2resid);
-static int mgc_logname2resid(char *logname, struct ldlm_res_id *res_id,
- int type)
+int mgc_logname2resid(char *logname, struct ldlm_res_id *res_id,
+ enum mgs_cfg_type type)
{
char *name_end;
int len;
len = name_end - logname;
return mgc_name2resid(logname, len, res_id, type);
}
+EXPORT_SYMBOL(mgc_logname2resid);
/********************** config llog list **********************/
-static struct list_head config_llog_list = LIST_HEAD_INIT(config_llog_list);
-static DEFINE_SPINLOCK(config_list_lock);
+static LIST_HEAD(config_llog_list);
+static DEFINE_SPINLOCK(config_list_lock); /* protects config_llog_list */
/* Take a reference to a config log */
static int config_log_get(struct config_llog_data *cld)
{
ENTRY;
atomic_inc(&cld->cld_refcount);
- CDEBUG(D_INFO, "log %s refs %d\n", cld->cld_logname,
+ CDEBUG(D_INFO, "log %s (%p) refs %d\n", cld->cld_logname, cld,
atomic_read(&cld->cld_refcount));
RETURN(0);
}
{
ENTRY;
- CDEBUG(D_INFO, "log %s refs %d\n", cld->cld_logname,
+ if (unlikely(!cld))
+ RETURN_EXIT;
+
+ CDEBUG(D_INFO, "log %s(%p) refs %d\n", cld->cld_logname, cld,
atomic_read(&cld->cld_refcount));
LASSERT(atomic_read(&cld->cld_refcount) > 0);
CDEBUG(D_MGC, "dropping config log %s\n", cld->cld_logname);
- if (cld->cld_recover)
- config_log_put(cld->cld_recover);
- if (cld->cld_params)
- config_log_put(cld->cld_params);
- if (cld->cld_nodemap)
- config_log_put(cld->cld_nodemap);
- if (cld->cld_sptlrpc)
- config_log_put(cld->cld_sptlrpc);
- if (cld_is_sptlrpc(cld))
+ config_log_put(cld->cld_barrier);
+ config_log_put(cld->cld_recover);
+ config_log_put(cld->cld_params);
+ config_log_put(cld->cld_nodemap);
+ config_log_put(cld->cld_sptlrpc);
+ if (cld_is_sptlrpc(cld)) {
+ cld->cld_stopping = 1;
sptlrpc_conf_log_stop(cld->cld_logname);
+ }
class_export_put(cld->cld_mgcexp);
OBD_FREE(cld, sizeof(*cld) + strlen(cld->cld_logname) + 1);
struct config_llog_data *config_log_find(char *logname,
struct config_llog_instance *cfg)
{
- struct config_llog_data *cld;
- struct config_llog_data *found = NULL;
- void * instance;
- ENTRY;
+ struct config_llog_data *cld;
+ struct config_llog_data *found = NULL;
+ unsigned long cfg_instance;
- LASSERT(logname != NULL);
+ ENTRY;
+ LASSERT(logname != NULL);
- instance = cfg ? cfg->cfg_instance : NULL;
+ cfg_instance = cfg ? cfg->cfg_instance : 0;
spin_lock(&config_list_lock);
list_for_each_entry(cld, &config_llog_list, cld_list_chain) {
- /* check if instance equals */
- if (instance != cld->cld_cfg.cfg_instance)
+ /* check if cfg_instance is the one we want */
+ if (cfg_instance != cld->cld_cfg.cfg_instance)
continue;
/* instance may be NULL, should check name */
static
struct config_llog_data *do_config_log_add(struct obd_device *obd,
char *logname,
- int type,
+ enum mgs_cfg_type type,
struct config_llog_instance *cfg,
struct super_block *sb)
{
ENTRY;
- CDEBUG(D_MGC, "do adding config log %s:%p\n", logname,
- cfg ? cfg->cfg_instance : NULL);
+ CDEBUG(D_MGC, "do adding config log %s-%016lx\n", logname,
+ cfg ? cfg->cfg_instance : 0);
OBD_ALLOC(cld, sizeof(*cld) + strlen(logname) + 1);
if (!cld)
/* Keep the mgc around until we are done */
cld->cld_mgcexp = class_export_get(obd->obd_self_export);
- if (cld_is_sptlrpc(cld)) {
+ if (cld_is_sptlrpc(cld))
sptlrpc_conf_log_start(logname);
- cld->cld_cfg.cfg_obdname = obd->obd_name;
- }
spin_lock(&config_list_lock);
list_add(&cld->cld_list_chain, &config_llog_list);
spin_unlock(&config_list_lock);
- if (cld_is_sptlrpc(cld) || cld_is_nodemap(cld)) {
+ if (cld_is_sptlrpc(cld) || cld_is_nodemap(cld) || cld_is_barrier(cld)) {
rc = mgc_process_log(obd, cld);
if (rc && rc != -ENOENT)
CERROR("%s: failed processing log, type %d: rc = %d\n",
}
static struct config_llog_data *config_recover_log_add(struct obd_device *obd,
- char *fsname,
- struct config_llog_instance *cfg,
- struct super_block *sb)
+ char *fsname,
+ struct config_llog_instance *cfg,
+ struct super_block *sb)
{
- struct config_llog_instance lcfg = *cfg;
- struct lustre_sb_info *lsi = s2lsi(sb);
- struct config_llog_data *cld;
- char logname[32];
+ struct config_llog_instance lcfg = *cfg;
+ struct config_llog_data *cld;
+ char logname[32];
- if (IS_OST(lsi))
- return NULL;
+ if (IS_OST(s2lsi(sb)))
+ return NULL;
/* for osp-on-ost, see lustre_start_osp() */
- if (IS_MDT(lsi) && lcfg.cfg_instance)
+ if (IS_MDT(s2lsi(sb)) && lcfg.cfg_instance)
return NULL;
- /* we have to use different llog for clients and mdts for cmd
- * where only clients are notified if one of cmd server restarts */
- LASSERT(strlen(fsname) < sizeof(logname) / 2);
- strcpy(logname, fsname);
- if (IS_SERVER(lsi)) { /* mdt */
- LASSERT(lcfg.cfg_instance == NULL);
- lcfg.cfg_instance = sb;
- strcat(logname, "-mdtir");
- } else {
- LASSERT(lcfg.cfg_instance != NULL);
- strcat(logname, "-cliir");
- }
+ /* We have to use different llog for clients and MDTs for DNE,
+ * where only clients are notified if one of DNE server restarts.
+ */
+ LASSERT(strlen(fsname) < sizeof(logname) / 2);
+ strncpy(logname, fsname, sizeof(logname));
+ if (IS_SERVER(s2lsi(sb))) { /* mdt */
+ LASSERT(lcfg.cfg_instance == 0);
+ lcfg.cfg_instance = ll_get_cfg_instance(sb);
+ strncat(logname, "-mdtir", sizeof(logname));
+ } else {
+ LASSERT(lcfg.cfg_instance != 0);
+ strncat(logname, "-cliir", sizeof(logname));
+ }
- cld = do_config_log_add(obd, logname, CONFIG_T_RECOVER, &lcfg, sb);
- return cld;
+ cld = do_config_log_add(obd, logname, MGS_CFG_T_RECOVER, &lcfg, sb);
+ return cld;
}
-static struct config_llog_data *config_params_log_add(struct obd_device *obd,
- struct config_llog_instance *cfg, struct super_block *sb)
+static struct config_llog_data *
+config_log_find_or_add(struct obd_device *obd, char *logname,
+ struct super_block *sb, enum mgs_cfg_type type,
+ struct config_llog_instance *cfg)
{
- struct config_llog_instance lcfg = *cfg;
- struct config_llog_data *cld;
+ struct config_llog_instance lcfg = *cfg;
+ struct config_llog_data *cld;
- lcfg.cfg_instance = sb;
+ /* Note class_config_llog_handler() depends on getting "obd" back */
+ lcfg.cfg_instance = sb ? ll_get_cfg_instance(sb) : (unsigned long)obd;
- cld = do_config_log_add(obd, PARAMS_FILENAME, CONFIG_T_PARAMS,
- &lcfg, sb);
+ cld = config_log_find(logname, &lcfg);
+ if (unlikely(cld != NULL))
+ return cld;
- return cld;
+ return do_config_log_add(obd, logname, type, &lcfg, sb);
}
/** Add this log to the list of active logs watched by an MGC.
struct config_llog_instance *cfg, struct super_block *sb)
{
struct lustre_sb_info *lsi = s2lsi(sb);
- struct config_llog_data *cld;
- struct config_llog_data *sptlrpc_cld;
- struct config_llog_data *params_cld;
- struct config_llog_data *nodemap_cld;
+ struct config_llog_data *cld = NULL;
+ struct config_llog_data *sptlrpc_cld = NULL;
+ struct config_llog_data *params_cld = NULL;
+ struct config_llog_data *nodemap_cld = NULL;
+ struct config_llog_data *barrier_cld = NULL;
char seclogname[32];
char *ptr;
int rc;
bool locked = false;
ENTRY;
- CDEBUG(D_MGC, "adding config log %s:%p\n", logname, cfg->cfg_instance);
+ CDEBUG(D_MGC, "add config log %s-%016lx\n", logname,
+ cfg->cfg_instance);
/*
* for each regular log, the depended sptlrpc log name is
memcpy(seclogname, logname, ptr - logname);
strcpy(seclogname + (ptr - logname), "-sptlrpc");
- sptlrpc_cld = config_log_find(seclogname, NULL);
- if (sptlrpc_cld == NULL) {
- sptlrpc_cld = do_config_log_add(obd, seclogname,
- CONFIG_T_SPTLRPC, NULL, NULL);
+ if (cfg->cfg_sub_clds & CONFIG_SUB_SPTLRPC) {
+ sptlrpc_cld = config_log_find_or_add(obd, seclogname, NULL,
+ MGS_CFG_T_SPTLRPC, cfg);
if (IS_ERR(sptlrpc_cld)) {
- CERROR("can't create sptlrpc log: %s\n", seclogname);
- GOTO(out, rc = PTR_ERR(sptlrpc_cld));
+ CERROR("%s: can't create sptlrpc log %s: rc = %ld\n",
+ obd->obd_name, seclogname, PTR_ERR(sptlrpc_cld));
+ RETURN(sptlrpc_cld);
}
}
- nodemap_cld = config_log_find(LUSTRE_NODEMAP_NAME, NULL);
- if (!nodemap_cld && IS_SERVER(lsi) && !IS_MGS(lsi)) {
- nodemap_cld = do_config_log_add(obd, LUSTRE_NODEMAP_NAME,
- CONFIG_T_NODEMAP, NULL, NULL);
+ if (!IS_MGS(lsi) && cfg->cfg_sub_clds & CONFIG_SUB_NODEMAP) {
+ nodemap_cld = config_log_find_or_add(obd, LUSTRE_NODEMAP_NAME,
+ NULL, MGS_CFG_T_NODEMAP,
+ cfg);
if (IS_ERR(nodemap_cld)) {
rc = PTR_ERR(nodemap_cld);
CERROR("%s: cannot create nodemap log: rc = %d\n",
}
}
- params_cld = config_params_log_add(obd, cfg, sb);
- if (IS_ERR(params_cld)) {
- rc = PTR_ERR(params_cld);
- CERROR("%s: can't create params log: rc = %d\n",
- obd->obd_name, rc);
- GOTO(out_nodemap, rc);
+ if (cfg->cfg_sub_clds & CONFIG_SUB_PARAMS) {
+ params_cld = config_log_find_or_add(obd, PARAMS_FILENAME, sb,
+ MGS_CFG_T_PARAMS, cfg);
+ if (IS_ERR(params_cld)) {
+ rc = PTR_ERR(params_cld);
+ CERROR("%s: can't create params log: rc = %d\n",
+ obd->obd_name, rc);
+ GOTO(out_nodemap, rc);
+ }
+ }
+
+ if (IS_MDT(s2lsi(sb)) && cfg->cfg_sub_clds & CONFIG_SUB_BARRIER) {
+ snprintf(seclogname + (ptr - logname), sizeof(seclogname) - 1,
+ "-%s", BARRIER_FILENAME);
+ barrier_cld = config_log_find_or_add(obd, seclogname, sb,
+ MGS_CFG_T_BARRIER, cfg);
+ if (IS_ERR(barrier_cld)) {
+ rc = PTR_ERR(barrier_cld);
+ CERROR("%s: can't create barrier log: rc = %d\n",
+ obd->obd_name, rc);
+ GOTO(out_params, rc);
+ }
}
- cld = do_config_log_add(obd, logname, CONFIG_T_CONFIG, cfg, sb);
+ cld = do_config_log_add(obd, logname, MGS_CFG_T_CONFIG, cfg, sb);
if (IS_ERR(cld)) {
- CERROR("can't create log: %s\n", logname);
- GOTO(out_params, rc = PTR_ERR(cld));
+ rc = PTR_ERR(cld);
+ CERROR("%s: can't create log: rc = %d\n",
+ obd->obd_name, rc);
+ GOTO(out_barrier, rc = PTR_ERR(cld));
}
LASSERT(lsi->lsi_lmd);
- if (!(lsi->lsi_lmd->lmd_flags & LMD_FLG_NOIR)) {
+ if (!(lsi->lsi_lmd->lmd_flags & LMD_FLG_NOIR) &&
+ cfg->cfg_sub_clds & CONFIG_SUB_RECOVER) {
struct config_llog_data *recover_cld;
ptr = strrchr(seclogname, '-');
}
recover_cld = config_recover_log_add(obd, seclogname, cfg, sb);
- if (IS_ERR(recover_cld))
- GOTO(out_cld, rc = PTR_ERR(recover_cld));
+ if (IS_ERR(recover_cld)) {
+ rc = PTR_ERR(recover_cld);
+ CERROR("%s: can't create recover log: rc = %d\n",
+ obd->obd_name, rc);
+ GOTO(out_cld, rc);
+ }
mutex_lock(&cld->cld_lock);
locked = true;
if (!locked)
mutex_lock(&cld->cld_lock);
cld->cld_params = params_cld;
+ cld->cld_barrier = barrier_cld;
cld->cld_nodemap = nodemap_cld;
cld->cld_sptlrpc = sptlrpc_cld;
mutex_unlock(&cld->cld_lock);
out_cld:
config_log_put(cld);
-
+out_barrier:
+ config_log_put(barrier_cld);
out_params:
config_log_put(params_cld);
-
out_nodemap:
config_log_put(nodemap_cld);
-
out_sptlrpc:
config_log_put(sptlrpc_cld);
-out:
return ERR_PTR(rc);
}
DEFINE_MUTEX(llog_process_lock);
-static inline void config_mark_cld_stop(struct config_llog_data *cld)
+static inline void config_mark_cld_stop_nolock(struct config_llog_data *cld)
{
- mutex_lock(&cld->cld_lock);
+ ENTRY;
+
spin_lock(&config_list_lock);
cld->cld_stopping = 1;
spin_unlock(&config_list_lock);
- mutex_unlock(&cld->cld_lock);
+
+ CDEBUG(D_INFO, "lockh %#llx\n", cld->cld_lockh.cookie);
+ if (!ldlm_lock_addref_try(&cld->cld_lockh, LCK_CR))
+ ldlm_lock_decref_and_cancel(&cld->cld_lockh, LCK_CR);
+}
+
+static inline void config_mark_cld_stop(struct config_llog_data *cld)
+{
+ if (cld) {
+ mutex_lock(&cld->cld_lock);
+ config_mark_cld_stop_nolock(cld);
+ mutex_unlock(&cld->cld_lock);
+ }
}
/** Stop watching for updates on this log.
struct config_llog_data *cld_params = NULL;
struct config_llog_data *cld_recover = NULL;
struct config_llog_data *cld_nodemap = NULL;
+ struct config_llog_data *cld_barrier = NULL;
int rc = 0;
ENTRY;
RETURN(rc);
}
- spin_lock(&config_list_lock);
- cld->cld_stopping = 1;
- spin_unlock(&config_list_lock);
-
cld_recover = cld->cld_recover;
cld->cld_recover = NULL;
cld_params = cld->cld_params;
cld->cld_params = NULL;
cld_nodemap = cld->cld_nodemap;
cld->cld_nodemap = NULL;
+ cld_barrier = cld->cld_barrier;
+ cld->cld_barrier = NULL;
cld_sptlrpc = cld->cld_sptlrpc;
cld->cld_sptlrpc = NULL;
- mutex_unlock(&cld->cld_lock);
- if (cld_recover) {
- config_mark_cld_stop(cld_recover);
- config_log_put(cld_recover);
- }
-
- if (cld_params) {
- config_mark_cld_stop(cld_params);
- config_log_put(cld_params);
- }
+ config_mark_cld_stop_nolock(cld);
+ mutex_unlock(&cld->cld_lock);
+ config_mark_cld_stop(cld_recover);
+ config_log_put(cld_recover);
+ config_mark_cld_stop(cld_params);
+ config_log_put(cld_params);
+ config_mark_cld_stop(cld_barrier);
+ config_log_put(cld_barrier);
+ /* don't explicitly set cld_stopping on sptlrpc lock here, as other
+ * targets may be active, it will be done in config_log_put if necessary
+ */
+ config_log_put(cld_sptlrpc);
/* don't set cld_stopping on nm lock as other targets may be active */
- if (cld_nodemap)
- config_log_put(cld_nodemap);
-
- if (cld_sptlrpc)
- config_log_put(cld_sptlrpc);
+ config_log_put(cld_nodemap);
/* drop the ref from the find */
config_log_put(cld);
RETURN(rc);
}
-#ifdef CONFIG_PROC_FS
int lprocfs_mgc_rd_ir_state(struct seq_file *m, void *data)
{
struct obd_device *obd = data;
struct obd_import *imp;
struct obd_connect_data *ocd;
struct config_llog_data *cld;
+ int rc = 0;
+
ENTRY;
+ LASSERT(obd);
+ with_imp_locked(obd, imp, rc) {
+ ocd = &imp->imp_connect_data;
- LASSERT(obd != NULL);
- LPROCFS_CLIMP_CHECK(obd);
- imp = obd->u.cli.cl_import;
- ocd = &imp->imp_connect_data;
+ seq_printf(m, "imperative_recovery: %s\n",
+ OCD_HAS_FLAG(ocd, IMP_RECOV) ?
+ "ENABLED" : "DISABLED");
+ }
+ if (rc)
+ RETURN(rc);
- seq_printf(m, "imperative_recovery: %s\n",
- OCD_HAS_FLAG(ocd, IMP_RECOV) ? "ENABLED" : "DISABLED");
seq_printf(m, "client_state:\n");
spin_lock(&config_list_lock);
}
spin_unlock(&config_list_lock);
- LPROCFS_CLIMP_EXIT(obd);
RETURN(0);
}
-#endif
/* reenqueue any lost locks */
#define RQ_RUNNING 0x1
EXIT;
}
-/* this timeout represents how many seconds MGC should wait before
- * requeue config and recover lock to the MGS. We need to randomize this
- * in order to not flood the MGS.
- */
-#define MGC_TIMEOUT_MIN_SECONDS 5
-#define MGC_TIMEOUT_RAND_CENTISEC 0x1ff /* ~500 */
-
static int mgc_requeue_thread(void *data)
{
int rc = 0;
spin_lock(&config_list_lock);
rq_state |= RQ_RUNNING;
while (!(rq_state & RQ_STOP)) {
- struct l_wait_info lwi;
struct config_llog_data *cld, *cld_prev;
- int rand = cfs_rand() & MGC_TIMEOUT_RAND_CENTISEC;
int to;
/* Any new or requeued lostlocks will change the state */
}
/* Always wait a few seconds to allow the server who
- caused the lock revocation to finish its setup, plus some
- random so everyone doesn't try to reconnect at once. */
- to = msecs_to_jiffies(MGC_TIMEOUT_MIN_SECONDS * MSEC_PER_SEC);
- /* rand is centi-seconds */
- to += msecs_to_jiffies(rand * MSEC_PER_SEC / 100);
- lwi = LWI_TIMEOUT(to, NULL, NULL);
- l_wait_event(rq_waitq, rq_state & (RQ_STOP | RQ_PRECLEANUP),
- &lwi);
-
- /*
- * iterate & processing through the list. for each cld, process
- * its depending sptlrpc cld firstly (if any) and then itself.
- *
- * it's guaranteed any item in the list must have
- * reference > 0; and if cld_lostlock is set, at
- * least one reference is taken by the previous enqueue.
- */
- cld_prev = NULL;
+ * caused the lock revocation to finish its setup, plus some
+ * random so everyone doesn't try to reconnect at once.
+ */
+ to = mgc_requeue_timeout_min == 0 ? 1 : mgc_requeue_timeout_min;
+ to = cfs_time_seconds(mgc_requeue_timeout_min) +
+ prandom_u32_max(cfs_time_seconds(to));
+ wait_event_idle_timeout(rq_waitq,
+ rq_state & (RQ_STOP | RQ_PRECLEANUP), to);
+
+ /*
+ * iterate & processing through the list. for each cld, process
+ * its depending sptlrpc cld firstly (if any) and then itself.
+ *
+ * it's guaranteed any item in the list must have
+ * reference > 0; and if cld_lostlock is set, at
+ * least one reference is taken by the previous enqueue.
+ */
+ cld_prev = NULL;
spin_lock(&config_list_lock);
rq_state &= ~RQ_PRECLEANUP;
cld->cld_lostlock = 0;
spin_unlock(&config_list_lock);
- if (cld_prev)
- config_log_put(cld_prev);
+ config_log_put(cld_prev);
cld_prev = cld;
if (likely(!(rq_state & RQ_STOP))) {
}
}
spin_unlock(&config_list_lock);
- if (cld_prev)
- config_log_put(cld_prev);
+ config_log_put(cld_prev);
/* Wait a bit to see if anyone else needs a requeue */
- lwi = (struct l_wait_info) { 0 };
- l_wait_event(rq_waitq, rq_state & (RQ_NOW | RQ_STOP),
- &lwi);
+ wait_event_idle(rq_waitq, rq_state & (RQ_NOW | RQ_STOP));
spin_lock(&config_list_lock);
}
cld->cld_stopping, rq_state);
LASSERT(atomic_read(&cld->cld_refcount) > 0);
+ /* lets cancel an existent lock to mark cld as "lostlock" */
+ CDEBUG(D_INFO, "lockh %#llx\n", cld->cld_lockh.cookie);
+ if (!ldlm_lock_addref_try(&cld->cld_lockh, LCK_CR))
+ ldlm_lock_decref_and_cancel(&cld->cld_lockh, LCK_CR);
+
mutex_lock(&cld->cld_lock);
spin_lock(&config_list_lock);
- if (!(rq_state & RQ_STOP) && !cld->cld_stopping && !cld->cld_lostlock) {
+ if (!(rq_state & RQ_STOP) && !cld->cld_stopping) {
cld->cld_lostlock = 1;
rq_state |= RQ_NOW;
wakeup = true;
}
/********************** class fns **********************/
+#ifdef HAVE_SERVER_SUPPORT
static int mgc_local_llog_init(const struct lu_env *env,
struct obd_device *obd,
struct obd_device *disk)
rc = local_oid_storage_init(env, lsi->lsi_dt_dev, &fid,
&cli->cl_mgc_los);
if (rc)
- RETURN(rc);
+ GOTO(out_mutex, rc);
rc = dt_root_get(env, lsi->lsi_dt_dev, &rfid);
if (rc)
dto = local_file_find_or_create(env, cli->cl_mgc_los, root,
MOUNT_CONFIGS_DIR,
S_IFDIR | S_IRUGO | S_IWUSR | S_IXUGO);
- lu_object_put_nocache(env, &root->do_lu);
+ dt_object_put_nocache(env, root);
if (IS_ERR(dto))
GOTO(out_los, rc = PTR_ERR(dto));
EXIT;
out_llog:
if (rc) {
- lu_object_put(env, &cli->cl_mgc_configs_dir->do_lu);
+ dt_object_put(env, cli->cl_mgc_configs_dir);
cli->cl_mgc_configs_dir = NULL;
}
out_los:
if (rc < 0) {
local_oid_storage_fini(env, cli->cl_mgc_los);
+out_mutex:
cli->cl_mgc_los = NULL;
mutex_unlock(&cli->cl_mgc_mutex);
}
mgc_local_llog_fini(env, obd);
- lu_object_put_nocache(env, &cli->cl_mgc_configs_dir->do_lu);
+ dt_object_put_nocache(env, cli->cl_mgc_configs_dir);
cli->cl_mgc_configs_dir = NULL;
local_oid_storage_fini(env, cli->cl_mgc_los);
RETURN(0);
}
+#endif /* HAVE_SERVER_SUPPORT */
static int mgc_llog_init(const struct lu_env *env, struct obd_device *obd)
{
/* COMPAT_146 - old config logs may have added profiles we don't
know about */
- if (obd->obd_type->typ_refcnt <= 1)
+ if (atomic_read(&obd->obd_type->typ_refcnt) <= 1)
/* Only for the last mgc */
class_del_profiles();
GOTO(err_cleanup, rc);
}
-#ifdef CONFIG_PROC_FS
- obd->obd_vars = lprocfs_mgc_obd_vars;
- lprocfs_obd_setup(obd);
-#endif
- sptlrpc_lprocfs_cliobd_attach(obd);
+ rc = mgc_tunables_init(obd);
+ if (rc)
+ GOTO(err_sysfs, rc);
if (atomic_inc_return(&mgc_count) == 1) {
rq_state = 0;
CERROR("%s: cannot start requeue thread: rc = %d; "
"no more log updates\n",
obd->obd_name, rc);
- GOTO(err_cleanup, rc);
+ GOTO(err_sysfs, rc);
}
/* rc is the task_struct pointer of mgc_requeue_thread. */
rc = 0;
RETURN(rc);
+err_sysfs:
+ lprocfs_obd_cleanup(obd);
err_cleanup:
client_obd_cleanup(obd);
err_decref:
LASSERT(atomic_read(&cld->cld_refcount) > 0);
lock->l_ast_data = NULL;
+ cld->cld_lockh.cookie = 0;
/* Are we done with this log? */
if (cld->cld_stopping) {
CDEBUG(D_MGC, "log %s: stopping, won't requeue\n",
#define MGC_ENQUEUE_LIMIT (INITIAL_CONNECT_TIMEOUT + (AT_OFF ? 0 : at_min) \
+ PING_INTERVAL)
#define MGC_TARGET_REG_LIMIT 10
+#define MGC_TARGET_REG_LIMIT_MAX RECONNECT_DELAY_MAX
#define MGC_SEND_PARAM_LIMIT 10
-#if LUSTRE_VERSION_CODE < OBD_OCD_VERSION(2, 13, 53, 0)
-/* Send parameter to MGS*/
-static int mgc_set_mgs_param(struct obd_export *exp,
- struct mgs_send_param *msp)
-{
- struct ptlrpc_request *req;
- struct mgs_send_param *req_msp, *rep_msp;
- int rc;
- ENTRY;
-
- req = ptlrpc_request_alloc_pack(class_exp2cliimp(exp),
- &RQF_MGS_SET_INFO, LUSTRE_MGS_VERSION,
- MGS_SET_INFO);
- if (!req)
- RETURN(-ENOMEM);
-
- req_msp = req_capsule_client_get(&req->rq_pill, &RMF_MGS_SEND_PARAM);
- if (!req_msp) {
- ptlrpc_req_finished(req);
- RETURN(-ENOMEM);
- }
-
- memcpy(req_msp, msp, sizeof(*req_msp));
- ptlrpc_request_set_replen(req);
-
- /* Limit how long we will wait for the enqueue to complete */
- req->rq_delay_limit = MGC_SEND_PARAM_LIMIT;
- rc = ptlrpc_queue_wait(req);
- if (!rc) {
- rep_msp = req_capsule_server_get(&req->rq_pill, &RMF_MGS_SEND_PARAM);
- memcpy(msp, rep_msp, sizeof(*rep_msp));
- }
-
- ptlrpc_req_finished(req);
-
- RETURN(rc);
-}
-#endif
-
/* Take a config lock so we can get cancel notifications */
static int mgc_enqueue(struct obd_export *exp, enum ldlm_type type,
union ldlm_policy_data *policy, enum ldlm_mode mode,
- __u64 *flags, void *bl_cb, void *cp_cb, void *gl_cb,
+ __u64 *flags, ldlm_glimpse_callback glimpse_callback,
void *data, __u32 lvb_len, void *lvb_swabber,
struct lustre_handle *lockh)
{
.ei_mode = mode,
.ei_cb_bl = mgc_blocking_ast,
.ei_cb_cp = ldlm_completion_ast,
+ .ei_cb_gl = glimpse_callback,
};
struct ptlrpc_request *req;
int short_limit = cld_is_sptlrpc(cld);
int rc;
ENTRY;
+ if (!exp)
+ RETURN(-EBADR);
+
CDEBUG(D_MGC, "Enqueue for %s (res %#llx)\n", cld->cld_logname,
cld->cld_resid.name[0]);
req_capsule_set_size(&req->rq_pill, &RMF_DLM_LVB, RCL_SERVER, 0);
ptlrpc_request_set_replen(req);
- /* check if this is server or client */
- if (cld->cld_cfg.cfg_sb) {
- struct lustre_sb_info *lsi = s2lsi(cld->cld_cfg.cfg_sb);
- if (lsi && IS_SERVER(lsi))
- short_limit = 1;
- }
+ /* check if this is server or client */
+ if (cld->cld_cfg.cfg_sb &&
+ IS_SERVER(s2lsi(cld->cld_cfg.cfg_sb)))
+ short_limit = 1;
+
/* Limit how long we will wait for the enqueue to complete */
req->rq_delay_limit = short_limit ? 5 : MGC_ENQUEUE_LIMIT;
rc = ldlm_cli_enqueue(exp, &req, &einfo, &cld->cld_resid, NULL, flags,
/* TODO: Help the MGS rebuild nidtbl. -jay */
}
+#ifdef HAVE_SERVER_SUPPORT
/* Send target_reg message to MGS */
static int mgc_target_register(struct obd_export *exp,
- struct mgs_target_info *mti)
+ struct mgs_target_info *mti)
{
- struct ptlrpc_request *req;
- struct mgs_target_info *req_mti, *rep_mti;
- int rc;
- ENTRY;
+ struct ptlrpc_request *req;
+ struct mgs_target_info *req_mti, *rep_mti;
+ int rc;
+ ENTRY;
- req = ptlrpc_request_alloc_pack(class_exp2cliimp(exp),
- &RQF_MGS_TARGET_REG, LUSTRE_MGS_VERSION,
- MGS_TARGET_REG);
- if (req == NULL)
- RETURN(-ENOMEM);
+ req = ptlrpc_request_alloc_pack(class_exp2cliimp(exp),
+ &RQF_MGS_TARGET_REG, LUSTRE_MGS_VERSION,
+ MGS_TARGET_REG);
+ if (req == NULL)
+ RETURN(-ENOMEM);
- req_mti = req_capsule_client_get(&req->rq_pill, &RMF_MGS_TARGET_INFO);
- if (!req_mti) {
- ptlrpc_req_finished(req);
- RETURN(-ENOMEM);
- }
+ req_mti = req_capsule_client_get(&req->rq_pill, &RMF_MGS_TARGET_INFO);
+ if (!req_mti) {
+ ptlrpc_req_finished(req);
+ RETURN(-ENOMEM);
+ }
- memcpy(req_mti, mti, sizeof(*req_mti));
- ptlrpc_request_set_replen(req);
- CDEBUG(D_MGC, "register %s\n", mti->mti_svname);
- /* Limit how long we will wait for the enqueue to complete */
- req->rq_delay_limit = MGC_TARGET_REG_LIMIT;
+ memcpy(req_mti, mti, sizeof(*req_mti));
+ ptlrpc_request_set_replen(req);
+ CDEBUG(D_MGC, "register %s\n", mti->mti_svname);
+ /* Limit how long we will wait for the enqueue to complete */
+ req->rq_delay_limit = MGC_TARGET_REG_LIMIT;
+
+ /* if the target needs to regenerate the config log in MGS, it's better
+ * to use some longer limit to let MGC have time to change connection to
+ * another MGS (or try again with the same MGS) for the target (server)
+ * will fail and exit if the request expired due to delay limit. */
+ if (mti->mti_flags & (LDD_F_UPDATE | LDD_F_NEED_INDEX))
+ req->rq_delay_limit = MGC_TARGET_REG_LIMIT_MAX;
rc = ptlrpc_queue_wait(req);
- if (!rc) {
- rep_mti = req_capsule_server_get(&req->rq_pill,
- &RMF_MGS_TARGET_INFO);
- memcpy(mti, rep_mti, sizeof(*rep_mti));
- CDEBUG(D_MGC, "register %s got index = %d\n",
- mti->mti_svname, mti->mti_stripe_index);
+ if (ptlrpc_client_replied(req)) {
+ rep_mti = req_capsule_server_get(&req->rq_pill,
+ &RMF_MGS_TARGET_INFO);
+ if (rep_mti)
+ memcpy(mti, rep_mti, sizeof(*rep_mti));
+ }
+ if (!rc) {
+ CDEBUG(D_MGC, "register %s got index = %d\n",
+ mti->mti_svname, mti->mti_stripe_index);
}
ptlrpc_req_finished(req);
RETURN(rc);
}
+#endif /* HAVE_SERVER_SUPPORT */
static int mgc_set_info_async(const struct lu_env *env, struct obd_export *exp,
u32 keylen, void *key,
int rc = -EINVAL;
ENTRY;
- /* Turn off initial_recov after we try all backup servers once */
- if (KEY_IS(KEY_INIT_RECOV_BACKUP)) {
- struct obd_import *imp = class_exp2cliimp(exp);
- int value;
- if (vallen != sizeof(int))
- RETURN(-EINVAL);
- value = *(int *)val;
- CDEBUG(D_MGC, "InitRecov %s %d/d%d:i%d:r%d:or%d:%s\n",
- imp->imp_obd->obd_name, value,
- imp->imp_deactive, imp->imp_invalid,
- imp->imp_replayable, imp->imp_obd->obd_replayable,
- ptlrpc_import_state_name(imp->imp_state));
- /* Resurrect if we previously died */
- if ((imp->imp_state != LUSTRE_IMP_FULL &&
- imp->imp_state != LUSTRE_IMP_NEW) || value > 1)
- ptlrpc_reconnect_import(imp);
- RETURN(0);
- }
- /* FIXME move this to mgc_process_config */
- if (KEY_IS(KEY_REGISTER_TARGET)) {
- struct mgs_target_info *mti;
- if (vallen != sizeof(struct mgs_target_info))
- RETURN(-EINVAL);
- mti = (struct mgs_target_info *)val;
- CDEBUG(D_MGC, "register_target %s %#x\n",
- mti->mti_svname, mti->mti_flags);
- rc = mgc_target_register(exp, mti);
- RETURN(rc);
- }
+ /* Turn off initial_recov after we try all backup servers once */
+ if (KEY_IS(KEY_INIT_RECOV_BACKUP)) {
+ struct obd_import *imp = class_exp2cliimp(exp);
+ int value;
+ if (vallen != sizeof(int))
+ RETURN(-EINVAL);
+ value = *(int *)val;
+ CDEBUG(D_MGC, "InitRecov %s %d/d%d:i%d:r%d:or%d:%s\n",
+ imp->imp_obd->obd_name, value,
+ imp->imp_deactive, imp->imp_invalid,
+ imp->imp_replayable, imp->imp_obd->obd_replayable,
+ ptlrpc_import_state_name(imp->imp_state));
+ /* Resurrect the import immediately if
+ * 1. we previously got disconnected,
+ * 2. value > 1 (at the same node with MGS)
+ * */
+ if (imp->imp_state == LUSTRE_IMP_DISCON || value > 1)
+ ptlrpc_reconnect_import(imp);
+
+ RETURN(0);
+ }
+
+#ifdef HAVE_SERVER_SUPPORT
+ /* FIXME move this to mgc_process_config */
+ if (KEY_IS(KEY_REGISTER_TARGET)) {
+ struct mgs_target_info *mti;
+
+ if (vallen != sizeof(struct mgs_target_info))
+ RETURN(-EINVAL);
+ mti = (struct mgs_target_info *)val;
+ CDEBUG(D_MGC, "register_target %s %#x\n",
+ mti->mti_svname, mti->mti_flags);
+ rc = mgc_target_register(exp, mti);
+ RETURN(rc);
+ }
if (KEY_IS(KEY_SET_FS)) {
struct super_block *sb = (struct super_block *)val;
rc = mgc_fs_cleanup(env, exp->exp_obd);
RETURN(rc);
}
-#if LUSTRE_VERSION_CODE < OBD_OCD_VERSION(2, 13, 53, 0)
- if (KEY_IS(KEY_SET_INFO)) {
- struct mgs_send_param *msp;
-
- msp = (struct mgs_send_param *)val;
- rc = mgc_set_mgs_param(exp, msp);
- RETURN(rc);
- }
#endif
if (KEY_IS(KEY_MGSSEC)) {
struct client_obd *cli = &exp->exp_obd->u.cli;
__u64 max_version,
void *data, int datalen, bool mne_swab)
{
- struct config_llog_instance *cfg = &cld->cld_cfg;
- struct lustre_sb_info *lsi = s2lsi(cfg->cfg_sb);
- struct mgs_nidtbl_entry *entry;
- struct lustre_cfg *lcfg;
- struct lustre_cfg_bufs bufs;
- u64 prev_version = 0;
- char *inst;
- char *buf;
- int bufsz;
- int pos;
- int rc = 0;
- int off = 0;
- ENTRY;
+ struct config_llog_instance *cfg = &cld->cld_cfg;
+ struct mgs_nidtbl_entry *entry;
+ struct lustre_cfg *lcfg;
+ struct lustre_cfg_bufs bufs;
+ u64 prev_version = 0;
+ char inst[MTI_NAME_MAXLEN + 1];
+ char *buf;
+ int bufsz;
+ int pos = 0;
+ int rc = 0;
+ int off = 0;
+ unsigned long dynamic_nids;
- LASSERT(cfg->cfg_instance != NULL);
- LASSERT(cfg->cfg_sb == cfg->cfg_instance);
+ ENTRY;
+ LASSERT(cfg->cfg_instance != 0);
+ LASSERT(ll_get_cfg_instance(cfg->cfg_sb) == cfg->cfg_instance);
- OBD_ALLOC(inst, PAGE_SIZE);
- if (inst == NULL)
- RETURN(-ENOMEM);
+ /* get dynamic nids setting */
+ dynamic_nids = mgc->obd_dynamic_nids;
- if (!IS_SERVER(lsi)) {
- pos = snprintf(inst, PAGE_SIZE, "%p", cfg->cfg_instance);
- if (pos >= PAGE_SIZE) {
- OBD_FREE(inst, PAGE_SIZE);
+ if (!IS_SERVER(s2lsi(cfg->cfg_sb))) {
+ pos = snprintf(inst, sizeof(inst), "%016lx", cfg->cfg_instance);
+ if (pos >= PAGE_SIZE)
return -E2BIG;
- }
- } else {
+#ifdef HAVE_SERVER_SUPPORT
+ } else {
+ struct lustre_sb_info *lsi = s2lsi(cfg->cfg_sb);
+
LASSERT(IS_MDT(lsi));
rc = server_name2svname(lsi->lsi_svname, inst, NULL,
- PAGE_SIZE);
- if (rc) {
- OBD_FREE(inst, PAGE_SIZE);
+ sizeof(inst));
+ if (rc)
RETURN(-EINVAL);
- }
- pos = strlen(inst);
- }
+#endif /* HAVE_SERVER_SUPPORT */
+ }
+
+ OBD_ALLOC(buf, PAGE_SIZE);
+ if (!buf)
+ return -ENOMEM;
+ bufsz = PAGE_SIZE;
+ pos = 0;
+
+ while (datalen > 0) {
+ int entry_len = sizeof(*entry);
+ int is_ost;
+ struct obd_device *obd;
+ struct obd_import *imp;
+ char *obdname;
+ char *cname;
+ char *params;
+ char *uuid;
+
+ rc = -EINVAL;
+ if (datalen < sizeof(*entry))
+ break;
- ++pos;
- buf = inst + pos;
- bufsz = PAGE_SIZE - pos;
-
- while (datalen > 0) {
- int entry_len = sizeof(*entry);
- int is_ost, i;
- struct obd_device *obd;
- char *obdname;
- char *cname;
- char *params;
- char *uuid;
-
- rc = -EINVAL;
- if (datalen < sizeof(*entry))
- break;
-
- entry = (typeof(entry))(data + off);
-
- /* sanity check */
- if (entry->mne_nid_type != 0) /* only support type 0 for ipv4 */
- break;
- if (entry->mne_nid_count == 0) /* at least one nid entry */
- break;
- if (entry->mne_nid_size != sizeof(lnet_nid_t))
- break;
-
- entry_len += entry->mne_nid_count * entry->mne_nid_size;
- if (datalen < entry_len) /* must have entry_len at least */
- break;
+ entry = (typeof(entry))(data + off);
+
+ /* sanity check */
+ if (entry->mne_nid_type != 0) /* only support type 0 for ipv4 */
+ break;
+ if (entry->mne_nid_count == 0) /* at least one nid entry */
+ break;
+ if (entry->mne_nid_size != sizeof(lnet_nid_t))
+ break;
+
+ entry_len += entry->mne_nid_count * entry->mne_nid_size;
+ if (datalen < entry_len) /* must have entry_len at least */
+ break;
/* Keep this swab for normal mixed endian handling. LU-1644 */
if (mne_swab)
if (entry->mne_length < entry_len)
break;
- off += entry->mne_length;
- datalen -= entry->mne_length;
- if (datalen < 0)
- break;
+ off += entry->mne_length;
+ datalen -= entry->mne_length;
+ if (datalen < 0)
+ break;
- if (entry->mne_version > max_version) {
- CERROR("entry index(%lld) is over max_index(%lld)\n",
- entry->mne_version, max_version);
- break;
- }
+ if (entry->mne_version > max_version) {
+ CERROR("entry index(%lld) is over max_index(%lld)\n",
+ entry->mne_version, max_version);
+ break;
+ }
- if (prev_version >= entry->mne_version) {
- CERROR("index unsorted, prev %lld, now %lld\n",
- prev_version, entry->mne_version);
- break;
- }
- prev_version = entry->mne_version;
+ if (prev_version >= entry->mne_version) {
+ CERROR("index unsorted, prev %lld, now %lld\n",
+ prev_version, entry->mne_version);
+ break;
+ }
+ prev_version = entry->mne_version;
- /*
- * Write a string with format "nid::instance" to
- * lustre/<osc|mdc>/<target>-<osc|mdc>-<instance>/import.
- */
+ /*
+ * Write a string with format "nid::instance" to
+ * lustre/<osc|mdc>/<target>-<osc|mdc>-<instance>/import.
+ */
- is_ost = entry->mne_type == LDD_F_SV_TYPE_OST;
- memset(buf, 0, bufsz);
- obdname = buf;
- pos = 0;
-
- /* lustre-OST0001-osc-<instance #> */
- strcpy(obdname, cld->cld_logname);
- cname = strrchr(obdname, '-');
- if (cname == NULL) {
- CERROR("mgc %s: invalid logname %s\n",
- mgc->obd_name, obdname);
- break;
- }
+ is_ost = entry->mne_type == LDD_F_SV_TYPE_OST;
+ memset(buf, 0, bufsz);
+ obdname = buf;
+ pos = 0;
+
+ /* lustre-OST0001-osc-<instance #> */
+ strcpy(obdname, cld->cld_logname);
+ cname = strrchr(obdname, '-');
+ if (cname == NULL) {
+ CERROR("mgc %s: invalid logname %s\n",
+ mgc->obd_name, obdname);
+ break;
+ }
- pos = cname - obdname;
- obdname[pos] = 0;
- pos += sprintf(obdname + pos, "-%s%04x",
- is_ost ? "OST" : "MDT", entry->mne_index);
+ pos = cname - obdname;
+ obdname[pos] = 0;
+ pos += sprintf(obdname + pos, "-%s%04x",
+ is_ost ? "OST" : "MDT", entry->mne_index);
- cname = is_ost ? "osc" : "mdc",
- pos += sprintf(obdname + pos, "-%s-%s", cname, inst);
- lustre_cfg_bufs_reset(&bufs, obdname);
+ cname = is_ost ? "osc" : "mdc",
+ pos += snprintf(obdname + pos, bufsz, "-%s-%s", cname, inst);
+ lustre_cfg_bufs_reset(&bufs, obdname);
- /* find the obd by obdname */
- obd = class_name2obd(obdname);
- if (obd == NULL) {
- CDEBUG(D_INFO, "mgc %s: cannot find obdname %s\n",
- mgc->obd_name, obdname);
+ /* find the obd by obdname */
+ obd = class_name2obd(obdname);
+ if (obd == NULL) {
+ CDEBUG(D_INFO, "mgc %s: cannot find obdname %s\n",
+ mgc->obd_name, obdname);
rc = 0;
- /* this is a safe race, when the ost is starting up...*/
- continue;
- }
-
- /* osc.import = "connection=<Conn UUID>::<target instance>" */
- ++pos;
- params = buf + pos;
- pos += sprintf(params, "%s.import=%s", cname, "connection=");
- uuid = buf + pos;
+ /* this is a safe race, when the ost is starting up...*/
+ continue;
+ }
- down_read(&obd->u.cli.cl_sem);
- if (obd->u.cli.cl_import == NULL) {
+ /* osc.import = "connection=<Conn UUID>::<target instance>" */
+ ++pos;
+ params = buf + pos;
+ pos += sprintf(params, "%s.import=%s", cname, "connection=");
+ uuid = buf + pos;
+
+ with_imp_locked(obd, imp, rc) {
+ /* iterate all nids to find one */
+ /* find uuid by nid */
+ /* create import entries if they don't exist */
+ rc = client_import_add_nids_to_conn(
+ imp, entry->u.nids, entry->mne_nid_count,
+ (struct obd_uuid *)uuid);
+
+ if (rc == -ENOENT && dynamic_nids) {
+ /* create a new connection for this import */
+ char *primary_nid =
+ libcfs_nid2str(entry->u.nids[0]);
+ int prim_nid_len = strlen(primary_nid) + 1;
+ struct obd_uuid server_uuid;
+
+ if (prim_nid_len > UUID_MAX)
+ goto fail;
+ strncpy(server_uuid.uuid, primary_nid,
+ prim_nid_len);
+
+ CDEBUG(D_INFO, "Adding a connection for %s\n",
+ primary_nid);
+
+ rc = client_import_dyn_add_conn(
+ imp, &server_uuid, entry->u.nids[0], 1);
+ if (rc < 0) {
+ CERROR("%s: Failed to add new connection with NID '%s' to import: rc = %d\n",
+ obd->obd_name, primary_nid, rc);
+ goto fail;
+ }
+ rc = client_import_add_nids_to_conn(
+ imp, entry->u.nids,
+ entry->mne_nid_count,
+ (struct obd_uuid *)uuid);
+ if (rc < 0) {
+ CERROR("%s: failed to lookup UUID: rc = %d\n",
+ obd->obd_name, rc);
+ goto fail;
+ }
+ }
+fail:;
+ }
+ if (rc == -ENODEV) {
/* client does not connect to the OST yet */
- up_read(&obd->u.cli.cl_sem);
rc = 0;
continue;
}
- /* iterate all nids to find one */
- /* find uuid by nid */
- rc = -ENOENT;
- for (i = 0; i < entry->mne_nid_count; i++) {
- rc = client_import_find_conn(obd->u.cli.cl_import,
- entry->u.nids[i],
- (struct obd_uuid *)uuid);
- if (rc == 0)
- break;
+ if (rc < 0 && rc != -ENOSPC) {
+ CERROR("mgc: cannot find UUID by nid '%s': rc = %d\n",
+ libcfs_nid2str(entry->u.nids[0]), rc);
+ break;
}
- up_read(&obd->u.cli.cl_sem);
- if (rc < 0) {
- CERROR("mgc: cannot find uuid by nid %s\n",
- libcfs_nid2str(entry->u.nids[0]));
- break;
- }
-
- CDEBUG(D_INFO, "Find uuid %s by nid %s\n",
- uuid, libcfs_nid2str(entry->u.nids[0]));
+ CDEBUG(D_INFO, "Found UUID '%s' by NID '%s'\n",
+ uuid, libcfs_nid2str(entry->u.nids[0]));
- pos += strlen(uuid);
- pos += sprintf(buf + pos, "::%u", entry->mne_instance);
- LASSERT(pos < bufsz);
+ pos += strlen(uuid);
+ pos += sprintf(buf + pos, "::%u", entry->mne_instance);
+ LASSERT(pos < bufsz);
- lustre_cfg_bufs_set_string(&bufs, 1, params);
+ lustre_cfg_bufs_set_string(&bufs, 1, params);
- lcfg = lustre_cfg_new(LCFG_PARAM, &bufs);
- if (lcfg == NULL) {
+ OBD_ALLOC(lcfg, lustre_cfg_len(bufs.lcfg_bufcount,
+ bufs.lcfg_buflen));
+ if (!lcfg) {
rc = -ENOMEM;
break;
}
+ lustre_cfg_init(lcfg, LCFG_PARAM, &bufs);
CDEBUG(D_INFO, "ir apply logs %lld/%lld for %s -> %s\n",
- prev_version, max_version, obdname, params);
+ prev_version, max_version, obdname, params);
- rc = class_process_config(lcfg);
- lustre_cfg_free(lcfg);
- if (rc)
- CDEBUG(D_INFO, "process config for %s error %d\n",
- obdname, rc);
+ rc = class_process_config(lcfg);
+ OBD_FREE(lcfg, lustre_cfg_len(lcfg->lcfg_bufcount,
+ lcfg->lcfg_buflens));
+ if (rc)
+ CDEBUG(D_INFO, "process config for %s error %d\n",
+ obdname, rc);
- /* continue, even one with error */
- }
+ /* continue, even one with error */
+ }
- OBD_FREE(inst, PAGE_SIZE);
- RETURN(rc);
+ OBD_FREE(buf, PAGE_SIZE);
+
+ RETURN(rc);
}
/**
struct ptlrpc_bulk_desc *desc;
struct page **pages = NULL;
__u64 config_read_offset = 0;
+ __u8 nodemap_cur_pass = 0;
int nrpages = 0;
bool eof = true;
bool mne_swab = false;
/* don't need to get local config */
if (cld_is_nodemap(cld) &&
- (LNET_NETTYP(LNET_NIDNET(mgc_conn->c_peer.nid)) == LOLND))
+ LNetIsPeerLocal(&mgc_conn->c_peer.nid))
GOTO(out, rc = 0);
- /* allocate buffer for bulk transfer.
- * if this is the first time for this mgs to read logs,
- * CONFIG_READ_NRPAGES_INIT will be used since it will read all logs
- * once; otherwise, it only reads increment of logs, this should be
- * small and CONFIG_READ_NRPAGES will be used.
- */
- nrpages = CONFIG_READ_NRPAGES;
+ /* allocate buffer for bulk transfer.
+ * if this is the first time for this mgs to read logs,
+ * CONFIG_READ_NRPAGES_INIT will be used since it will read all logs
+ * once; otherwise, it only reads increment of logs, this should be
+ * small and CONFIG_READ_NRPAGES will be used.
+ */
+ nrpages = CONFIG_READ_NRPAGES;
if (cfg->cfg_last_idx == 0 || cld_is_nodemap(cld))
- nrpages = CONFIG_READ_NRPAGES_INIT;
+ nrpages = CONFIG_READ_NRPAGES_INIT;
- OBD_ALLOC(pages, sizeof(*pages) * nrpages);
- if (pages == NULL)
- GOTO(out, rc = -ENOMEM);
+ OBD_ALLOC_PTR_ARRAY_LARGE(pages, nrpages);
+ if (pages == NULL)
+ GOTO(out, rc = -ENOMEM);
- for (i = 0; i < nrpages; i++) {
+ for (i = 0; i < nrpages; i++) {
pages[i] = alloc_page(GFP_KERNEL);
- if (pages[i] == NULL)
- GOTO(out, rc = -ENOMEM);
- }
+ if (pages[i] == NULL)
+ GOTO(out, rc = -ENOMEM);
+ }
again:
#ifdef HAVE_SERVER_SUPPORT
body->mcb_type = cld->cld_type;
body->mcb_bits = PAGE_SHIFT;
body->mcb_units = nrpages;
+ body->mcb_nm_cur_pass = nodemap_cur_pass;
/* allocate bulk transfer descriptor */
desc = ptlrpc_prep_bulk_imp(req, nrpages, 1,
- PTLRPC_BULK_PUT_SINK | PTLRPC_BULK_BUF_KIOV,
+ PTLRPC_BULK_PUT_SINK,
MGS_BULK_PORTAL,
&ptlrpc_bulk_kiov_pin_ops);
if (desc == NULL)
if (cld_is_nodemap(cld)) {
config_read_offset = res->mcr_offset;
eof = config_read_offset == II_END_OFF;
+ nodemap_cur_pass = res->mcr_nm_cur_pass;
} else {
if (res->mcr_size < res->mcr_offset)
GOTO(out, rc = -EINVAL);
#ifdef HAVE_SERVER_SUPPORT
/* config changed since first read RPC */
if (cld_is_nodemap(cld) && config_read_offset == 0) {
- recent_nodemap = NULL;
- nodemap_config_dealloc(new_config);
- new_config = NULL;
-
CDEBUG(D_INFO, "nodemap config changed in transit, retrying\n");
-
- /* setting eof to false, we request config again */
- eof = false;
- GOTO(out, rc = 0);
+ GOTO(out, rc = -EAGAIN);
}
#endif
if (!eof)
GOTO(out, rc);
}
- mne_swab = !!ptlrpc_rep_need_swab(req);
-#if LUSTRE_VERSION_CODE < OBD_OCD_VERSION(3, 0, 53, 0)
- /* This import flag means the server did an extra swab of IR MNE
- * records (fixed in LU-1252), reverse it here if needed. LU-1644 */
- if (unlikely(req->rq_import->imp_need_mne_swab))
- mne_swab = !mne_swab;
-#endif
+ mne_swab = req_capsule_rep_need_swab(&req->rq_pill);
/* When a nodemap config is received, we build a new nodemap config,
* with new nodemap structs. We keep track of the most recently added
cld_is_nodemap(cld) ? "nodemap" : "recovery",
cld->cld_logname,
rc2);
- break;
+ GOTO(out, rc = rc2);
}
ealen -= PAGE_SIZE;
break;
__free_page(pages[i]);
}
- OBD_FREE(pages, sizeof(*pages) * nrpages);
+ OBD_FREE_PTR_ARRAY_LARGE(pages, nrpages);
}
return rc;
}
+#ifdef HAVE_SERVER_SUPPORT
+static int mgc_barrier_glimpse_ast(struct ldlm_lock *lock, void *data)
+{
+ struct config_llog_data *cld = lock->l_ast_data;
+ int rc;
+ ENTRY;
+
+ if (cld->cld_stopping)
+ RETURN(-ENODEV);
+
+ rc = barrier_handler(s2lsi(cld->cld_cfg.cfg_sb)->lsi_dt_dev,
+ (struct ptlrpc_request *)data);
+
+ RETURN(rc);
+}
+
/* Copy a remote log locally */
static int mgc_llog_local_copy(const struct lu_env *env,
struct obd_device *obd,
return rc;
}
-/* local_only means it cannot get remote llogs */
-static int mgc_process_cfg_log(struct obd_device *mgc,
- struct config_llog_data *cld, int local_only)
+static int
+mgc_process_server_cfg_log(struct lu_env *env, struct llog_ctxt **ctxt,
+ struct lustre_sb_info *lsi, struct obd_device *mgc,
+ struct config_llog_data *cld, int local_only)
{
- struct llog_ctxt *ctxt, *lctxt = NULL;
- struct client_obd *cli = &mgc->u.cli;
- struct lustre_sb_info *lsi = NULL;
- int rc = 0;
- bool sptlrpc_started = false;
- struct lu_env *env;
-
- ENTRY;
-
- LASSERT(cld);
- LASSERT(mutex_is_locked(&cld->cld_lock));
-
- /*
- * local copy of sptlrpc log is controlled elsewhere, don't try to
- * read it up here.
- */
- if (cld_is_sptlrpc(cld) && local_only)
- RETURN(0);
-
- if (cld->cld_cfg.cfg_sb)
- lsi = s2lsi(cld->cld_cfg.cfg_sb);
-
- OBD_ALLOC_PTR(env);
- if (env == NULL)
- RETURN(-ENOMEM);
-
- rc = lu_env_init(env, LCT_MG_THREAD);
- if (rc)
- GOTO(out_free, rc);
-
- ctxt = llog_get_context(mgc, LLOG_CONFIG_REPL_CTXT);
- LASSERT(ctxt);
-
- lctxt = llog_get_context(mgc, LLOG_CONFIG_ORIG_CTXT);
+ struct llog_ctxt *lctxt = llog_get_context(mgc, LLOG_CONFIG_ORIG_CTXT);
+ struct client_obd *cli = &mgc->u.cli;
+ int rc = 0;
/* Copy the setup log locally if we can. Don't mess around if we're
* running an MGS though (logs are already local). */
cli->cl_mgc_configs_dir != NULL &&
lu2dt_dev(cli->cl_mgc_configs_dir->do_lu.lo_dev) ==
lsi->lsi_dt_dev) {
- if (!local_only)
+ if (!local_only && !lsi->lsi_dt_dev->dd_rdonly) {
/* Only try to copy log if we have the lock. */
- rc = mgc_llog_local_copy(env, mgc, ctxt, lctxt,
+ CDEBUG(D_INFO, "%s: copy local log %s\n",
+ mgc->obd_name, cld->cld_logname);
+
+ rc = mgc_llog_local_copy(env, mgc, *ctxt, lctxt,
cld->cld_logname);
+ if (!rc)
+ lsi->lsi_flags &= ~LDD_F_NO_LOCAL_LOGS;
+ }
if (local_only || rc) {
+ if (unlikely(lsi->lsi_flags & LDD_F_NO_LOCAL_LOGS) ||
+ rc) {
+ CWARN("%s: local log %s are not valid and/or remote logs are not accessbile rc = %d\n",
+ mgc->obd_name, cld->cld_logname, rc);
+ GOTO(out_pop, rc = -EIO);
+ }
+
if (strcmp(cld->cld_logname, PARAMS_FILENAME) != 0 &&
llog_is_empty(env, lctxt, cld->cld_logname)) {
- LCONSOLE_ERROR_MSG(0x13a, "Failed to get MGS "
- "log %s and no local copy."
- "\n", cld->cld_logname);
+ LCONSOLE_ERROR_MSG(0x13a, "Failed to get MGS log %s and no local copy.\n",
+ cld->cld_logname);
GOTO(out_pop, rc = -ENOENT);
}
- CDEBUG(D_MGC, "Failed to get MGS log %s, using local "
- "copy for now, will try to update later.\n",
- cld->cld_logname);
+ CDEBUG(D_MGC, "%s: Failed to get MGS log %s, using local copy for now, will try to update later.\n",
+ mgc->obd_name, cld->cld_logname);
rc = 0;
}
/* Now, whether we copied or not, start using the local llog.
* If we failed to copy, we'll start using whatever the old
* log has. */
- llog_ctxt_put(ctxt);
- ctxt = lctxt;
+ llog_ctxt_put(*ctxt);
+ *ctxt = lctxt;
lctxt = NULL;
- } else {
- if (local_only) /* no local log at client side */
- GOTO(out_pop, rc = -EIO);
+ } else if (local_only) { /* no local log at client side */
+ GOTO(out_pop, rc = -EIO);
}
- if (cld_is_sptlrpc(cld)) {
- sptlrpc_conf_log_update_begin(cld->cld_logname);
- sptlrpc_started = true;
- }
+ rc = -EAGAIN;
+ if (lsi && IS_SERVER(lsi) && !IS_MGS(lsi) &&
+ lsi->lsi_dt_dev->dd_rdonly) {
+ struct llog_ctxt *rctxt;
- /* logname and instance info should be the same, so use our
- * copy of the instance for the update. The cfg_last_idx will
- * be updated here. */
- rc = class_config_parse_llog(env, ctxt, cld->cld_logname,
- &cld->cld_cfg);
- EXIT;
+ /* Under readonly mode, we may have no local copy or local
+ * copy is incomplete, so try to use remote llog firstly. */
+ rctxt = llog_get_context(mgc, LLOG_CONFIG_REPL_CTXT);
+ LASSERT(rctxt);
+ rc = class_config_parse_llog(env, rctxt, cld->cld_logname,
+ &cld->cld_cfg);
+ llog_ctxt_put(rctxt);
+ }
out_pop:
- __llog_ctxt_put(env, ctxt);
if (lctxt)
__llog_ctxt_put(env, lctxt);
+ return rc;
+}
+#else /* !HAVE_SERVER_SUPPORT */
+#define mgc_barrier_glimpse_ast NULL
+#endif /* HAVE_SERVER_SUPPORT */
+
+/* local_only means it cannot get remote llogs */
+static int mgc_process_cfg_log(struct obd_device *mgc,
+ struct config_llog_data *cld, int local_only)
+{
+ struct llog_ctxt *ctxt;
+ struct lustre_sb_info *lsi = NULL;
+ int rc = 0;
+ struct lu_env *env;
+
+ ENTRY;
+ LASSERT(cld);
+ LASSERT(mutex_is_locked(&cld->cld_lock));
+
+ if (cld->cld_cfg.cfg_sb)
+ lsi = s2lsi(cld->cld_cfg.cfg_sb);
+
+ OBD_ALLOC_PTR(env);
+ if (!env)
+ RETURN(-ENOMEM);
+
+ rc = lu_env_init(env, LCT_MG_THREAD);
+ if (rc)
+ GOTO(out_free, rc);
+
+ ctxt = llog_get_context(mgc, LLOG_CONFIG_REPL_CTXT);
+ LASSERT(ctxt);
+#ifdef HAVE_SERVER_SUPPORT
+ rc = mgc_process_server_cfg_log(env, &ctxt, lsi, mgc, cld,
+ local_only);
+ if (rc == -EIO && local_only)
+ GOTO(out_pop, rc);
+ if (rc && rc != -ENOENT)
+#else
+ if (local_only)
+ GOTO(out_pop, rc);
+#endif
+ rc = class_config_parse_llog(env, ctxt, cld->cld_logname,
+ &cld->cld_cfg);
/*
- * update settings on existing OBDs. doing it inside
- * of llog_process_lock so no device is attaching/detaching
- * in parallel.
+ * update settings on existing OBDs.
* the logname must be <fsname>-sptlrpc
*/
- if (sptlrpc_started) {
- LASSERT(cld_is_sptlrpc(cld));
- sptlrpc_conf_log_update_end(cld->cld_logname);
+ if (rc == 0 && cld_is_sptlrpc(cld))
class_notify_sptlrpc_conf(cld->cld_logname,
strlen(cld->cld_logname) -
strlen("-sptlrpc"));
- }
+ EXIT;
+out_pop:
+ __llog_ctxt_put(env, ctxt);
lu_env_fini(env);
out_free:
OBD_FREE_PTR(env);
mutex_lock(&cld->cld_lock);
if (cld->cld_stopping) {
mutex_unlock(&cld->cld_lock);
- RETURN(0);
- }
+ RETURN(0);
+ }
- OBD_FAIL_TIMEOUT(OBD_FAIL_MGC_PAUSE_PROCESS_LOG, 20);
+ OBD_FAIL_TIMEOUT(OBD_FAIL_MGC_PAUSE_PROCESS_LOG, 20);
- CDEBUG(D_MGC, "Process log %s:%p from %d\n", cld->cld_logname,
+ CDEBUG(D_MGC, "Process log %s-%016lx from %d\n", cld->cld_logname,
cld->cld_cfg.cfg_instance, cld->cld_cfg.cfg_last_idx + 1);
/* Get the cfg lock on the llog */
rcl = mgc_enqueue(mgc->u.cli.cl_mgc_mgsexp, LDLM_PLAIN, NULL,
- LCK_CR, &flags, NULL, NULL, NULL,
+ LCK_CR, &flags,
+ cld_is_barrier(cld) ? mgc_barrier_glimpse_ast : NULL,
cld, 0, NULL, &lockh);
if (rcl == 0) {
/* Get the cld, it will be released in mgc_blocking_ast. */
config_log_get(cld);
rc = ldlm_lock_set_data(&lockh, (void *)cld);
+ LASSERT(!lustre_handle_is_used(&cld->cld_lockh));
LASSERT(rc == 0);
+ cld->cld_lockh = lockh;
} else {
CDEBUG(D_MGC, "Can't get cfg lock: %d\n", rcl);
+ cld->cld_lockh.cookie = 0;
if (rcl == -ESHUTDOWN &&
atomic_read(&mgc->u.cli.cl_mgc_refcount) > 0 && !retry) {
struct obd_import *imp;
- struct l_wait_info lwi;
- int secs = cfs_time_seconds(obd_timeout);
+ long timeout = cfs_time_seconds(obd_timeout);
mutex_unlock(&cld->cld_lock);
imp = class_exp2cliimp(mgc->u.cli.cl_mgc_mgsexp);
* FULL or closed */
ptlrpc_pinger_force(imp);
- lwi = LWI_TIMEOUT(secs, NULL, NULL);
- l_wait_event(imp->imp_recovery_waitq,
- !mgc_import_in_recovery(imp), &lwi);
+ wait_event_idle_timeout(imp->imp_recovery_waitq,
+ !mgc_import_in_recovery(imp),
+ timeout);
if (imp->imp_state == LUSTRE_IMP_FULL) {
retry = true;
goto restart;
} else {
mutex_lock(&cld->cld_lock);
+ /* unlock/lock mutex, so check stopping again */
+ if (cld->cld_stopping) {
+ mutex_unlock(&cld->cld_lock);
+ RETURN(0);
+ }
spin_lock(&config_list_lock);
cld->cld_lostlock = 1;
spin_unlock(&config_list_lock);
else if (cld_is_nodemap(cld))
rc = rcl;
- if (cld_is_recover(cld) && rc) {
- if (!rcl) {
- CERROR("%s: recover log %s failed, not fatal: rc = %d\n",
- mgc->obd_name, cld->cld_logname, rc);
- spin_lock(&config_list_lock);
- cld->cld_lostlock = 1;
- spin_unlock(&config_list_lock);
- }
- rc = 0; /* this is not a fatal error for recover log */
- }
- } else {
+ } else if (!cld_is_barrier(cld)) {
rc = mgc_process_cfg_log(mgc, cld, rcl != 0);
}
CDEBUG(D_MGC, "%s: configuration from log '%s' %sed (%d).\n",
mgc->obd_name, cld->cld_logname, rc ? "fail" : "succeed", rc);
- mutex_unlock(&cld->cld_lock);
-
/* Now drop the lock so MGS can revoke it */
if (!rcl) {
rcl = mgc_cancel(mgc->u.cli.cl_mgc_mgsexp, LCK_CR, &lockh);
if (rcl)
CERROR("Can't drop cfg lock: %d\n", rcl);
}
+ mutex_unlock(&cld->cld_lock);
+
+ /* requeue nodemap lock immediately if transfer was interrupted */
+ if ((cld_is_nodemap(cld) && rc == -EAGAIN) ||
+ (cld_is_recover(cld) && rc)) {
+ if (cld_is_recover(cld))
+ CWARN("%s: IR log %s failed, not fatal: rc = %d\n",
+ mgc->obd_name, cld->cld_logname, rc);
+ mgc_requeue_add(cld);
+ rc = 0;
+ }
RETURN(rc);
}
ENTRY;
switch(lcfg->lcfg_command) {
- case LCFG_LOV_ADD_OBD: {
- /* Overloading this cfg command: register a new target */
- struct mgs_target_info *mti;
-
- if (LUSTRE_CFG_BUFLEN(lcfg, 1) !=
- sizeof(struct mgs_target_info))
- GOTO(out, rc = -EINVAL);
-
- mti = (struct mgs_target_info *)lustre_cfg_buf(lcfg, 1);
- CDEBUG(D_MGC, "add_target %s %#x\n",
- mti->mti_svname, mti->mti_flags);
- rc = mgc_target_register(obd->u.cli.cl_mgc_mgsexp, mti);
- break;
- }
- case LCFG_LOV_DEL_OBD:
- /* Unregister has no meaning at the moment. */
- CERROR("lov_del_obd unimplemented\n");
- rc = -ENOSYS;
- break;
+#ifdef HAVE_SERVER_SUPPORT
+ case LCFG_LOV_ADD_OBD: {
+ /* Overloading this cfg command: register a new target */
+ struct mgs_target_info *mti;
+
+ if (LUSTRE_CFG_BUFLEN(lcfg, 1) !=
+ sizeof(struct mgs_target_info))
+ GOTO(out, rc = -EINVAL);
+
+ mti = lustre_cfg_buf(lcfg, 1);
+ CDEBUG(D_MGC, "add_target %s %#x\n",
+ mti->mti_svname, mti->mti_flags);
+ rc = mgc_target_register(obd->u.cli.cl_mgc_mgsexp, mti);
+ break;
+ }
+ case LCFG_LOV_DEL_OBD:
+ /* Unregister has no meaning at the moment. */
+ CERROR("lov_del_obd unimplemented\n");
+ rc = -EINVAL;
+ break;
+#endif
case LCFG_SPTLRPC_CONF: {
rc = sptlrpc_process_config(lcfg);
break;
break;
}
- /* COMPAT_146 */
- /* FIXME only set this for old logs! Right now this forces
- us to always skip the "inside markers" check */
- cld->cld_cfg.cfg_flags |= CFG_F_COMPAT146;
-
rc = mgc_process_log(obd, cld);
if (rc == 0 && cld->cld_recover != NULL) {
if (OCD_HAS_FLAG(&obd->u.cli.cl_import->
RETURN(rc);
}
-static struct obd_ops mgc_obd_ops = {
+static const struct obd_ops mgc_obd_ops = {
.o_owner = THIS_MODULE,
.o_setup = mgc_setup,
.o_precleanup = mgc_precleanup,
.o_process_config = mgc_process_config,
};
+static int mgc_param_requeue_timeout_min_set(const char *val,
+ cfs_kernel_param_arg_t *kp)
+{
+ int rc;
+ unsigned int num;
+
+ rc = kstrtouint(val, 0, &num);
+ if (rc < 0)
+ return rc;
+ if (num > 120)
+ return -EINVAL;
+
+ mgc_requeue_timeout_min = num;
+
+ return 0;
+}
+
+static const struct kernel_param_ops param_ops_requeue_timeout_min = {
+ .set = mgc_param_requeue_timeout_min_set,
+ .get = param_get_uint,
+};
+
+#define param_check_requeue_timeout_min(name, p) \
+ __param_check(name, p, unsigned int)
+
+unsigned int mgc_requeue_timeout_min = MGC_TIMEOUT_MIN_SECONDS;
+#ifdef HAVE_KERNEL_PARAM_OPS
+module_param(mgc_requeue_timeout_min, requeue_timeout_min, 0644);
+#else
+module_param_call(mgc_requeue_timeout_min, mgc_param_requeue_timeout_min_set,
+ param_get_uint, ¶m_ops_requeue_timeout_min, 0644);
+#endif
+MODULE_PARM_DESC(mgc_requeue_timeout_min, "Minimal requeue time to refresh logs");
+
static int __init mgc_init(void)
{
- return class_register_type(&mgc_obd_ops, NULL, true, NULL,
+ return class_register_type(&mgc_obd_ops, NULL, false,
LUSTRE_MGC_NAME, NULL);
}