From 85a5da01636b97081088b94dd8c2e0320f5ac68c Mon Sep 17 00:00:00 2001 From: wangdi Date: Wed, 2 Oct 2013 04:40:56 -0700 Subject: [PATCH] LU-1187 lod: move seq client init from MDT to LOD Move seq client init from MDT to LOD, so to unify config log for adding OSP and MD OSP. Change-Id: I93f91c0dd6f3568196a99eecf422e0b409abda31 Signed-off-by: Wang Di Reviewed-on: http://review.whamcloud.com/4337 Tested-by: Hudson Tested-by: Maloo Reviewed-by: Alex Zhuravlev Reviewed-by: Mike Pershin Reviewed-by: Andreas Dilger --- lustre/lod/lod_dev.c | 120 +++++++++++++++++++++++++++++++++++++++++++ lustre/mdt/mdt_handler.c | 130 +++++------------------------------------------ 2 files changed, 133 insertions(+), 117 deletions(-) diff --git a/lustre/lod/lod_dev.c b/lustre/lod/lod_dev.c index 4d1d854..5ea6737 100644 --- a/lustre/lod/lod_dev.c +++ b/lustre/lod/lod_dev.c @@ -44,6 +44,7 @@ #define DEBUG_SUBSYSTEM S_MDS #include +#include #include #include "lod_internal.h" @@ -213,6 +214,120 @@ static int lodname2mdt_index(char *lodname, int *index) return 0; } +/* + * Init client sequence manager which is used by local MDS to talk to sequence + * controller on remote node. + */ +static int lod_seq_init_cli(const struct lu_env *env, + struct lod_device *lod, + char *tgtuuid, int index) +{ + struct seq_server_site *ss; + struct obd_device *osp; + int rc; + char *prefix; + struct obd_uuid obd_uuid; + ENTRY; + + ss = lu_site2seq(lod2lu_dev(lod)->ld_site); + LASSERT(ss != NULL); + + /* check if this is adding the first MDC and controller is not yet + * initialized. */ + if (index != 0 || ss->ss_client_seq) + RETURN(0); + + obd_str2uuid(&obd_uuid, tgtuuid); + osp = class_find_client_obd(&obd_uuid, LUSTRE_OSP_NAME, + &lod->lod_dt_dev.dd_lu_dev.ld_obd->obd_uuid); + if (osp == NULL) { + CERROR("%s: can't find %s device\n", + lod->lod_dt_dev.dd_lu_dev.ld_obd->obd_name, + tgtuuid); + RETURN(-EINVAL); + } + + if (!osp->obd_set_up) { + CERROR("target %s not set up\n", osp->obd_name); + rc = -EINVAL; + } + + LASSERT(ss->ss_control_exp); + OBD_ALLOC_PTR(ss->ss_client_seq); + if (ss->ss_client_seq == NULL) + RETURN(-ENOMEM); + + OBD_ALLOC(prefix, MAX_OBD_NAME + 5); + if (!prefix) { + OBD_FREE_PTR(ss->ss_client_seq); + ss->ss_client_seq = NULL; + RETURN(-ENOMEM); + } + + snprintf(prefix, MAX_OBD_NAME + 5, "ctl-%s", osp->obd_name); + rc = seq_client_init(ss->ss_client_seq, ss->ss_control_exp, + LUSTRE_SEQ_METADATA, prefix, NULL); + OBD_FREE(prefix, MAX_OBD_NAME + 5); + if (rc) { + OBD_FREE_PTR(ss->ss_client_seq); + ss->ss_client_seq = NULL; + RETURN(rc); + } + + LASSERT(ss->ss_server_seq != NULL); + rc = seq_server_set_cli(ss->ss_server_seq, ss->ss_client_seq, + env); + + RETURN(rc); +} + +static void lod_seq_fini_cli(struct lod_device *lod) +{ + struct seq_server_site *ss; + + ENTRY; + + ss = lu_site2seq(lod2lu_dev(lod)->ld_site); + if (ss == NULL) { + EXIT; + return; + } + + if (ss->ss_server_seq) + seq_server_set_cli(ss->ss_server_seq, + NULL, NULL); + + if (ss->ss_control_exp) { + class_export_put(ss->ss_control_exp); + ss->ss_control_exp = NULL; + } + + EXIT; + return; +} + +/** + * Procss config log on LOD + * \param env environment info + * \param dev lod device + * \param lcfg config log + * + * Add osc config log, + * marker 20 (flags=0x01, v2.2.49.56) lustre-OST0001 'add osc' + * add_uuid nid=192.168.122.162@tcp(0x20000c0a87aa2) 0: 1:nidxxx + * attach 0:lustre-OST0001-osc-MDT0001 1:osc 2:lustre-MDT0001-mdtlov_UUID + * setup 0:lustre-OST0001-osc-MDT0001 1:lustre-OST0001_UUID 2:nid + * lov_modify_tgts add 0:lustre-MDT0001-mdtlov 1:lustre-OST0001_UUID 2:1 3:1 + * marker 20 (flags=0x02, v2.2.49.56) lustre-OST0001 'add osc' + * + * Add mdc config log + * marker 10 (flags=0x01, v2.2.49.56) lustre-MDT0000 'add osp' + * add_uuid nid=192.168.122.162@tcp(0x20000c0a87aa2) 0: 1:nid + * attach 0:lustre-MDT0000-osp-MDT0001 1:osp 2:lustre-MDT0001-mdtlov_UUID + * setup 0:lustre-MDT0000-osp-MDT0001 1:lustre-MDT0000_UUID 2:nid + * modify_mdc_tgts add 0:lustre-MDT0001 1:lustre-MDT0000_UUID 2:0 3:1 + * marker 10 (flags=0x02, v2.2.49.56) lustre-MDT0000_UUID 'add osp' + **/ static int lod_process_config(const struct lu_env *env, struct lu_device *dev, struct lustre_cfg *lcfg) @@ -260,6 +375,9 @@ static int lod_process_config(const struct lu_env *env, mdt_index = index; rc = lod_add_device(env, lod, arg1, index, gen, mdt_index, LUSTRE_MDC_NAME, 1); + if (rc == 0) + rc = lod_seq_init_cli(env, lod, arg1, + mdt_index); } else if (lcfg->lcfg_command == LCFG_LOV_ADD_INA) { /*FIXME: Add mdt_index for LCFG_LOV_ADD_INA*/ mdt_index = 0; @@ -289,6 +407,8 @@ static int lod_process_config(const struct lu_env *env, lu_dev_del_linkage(dev->ld_site, dev); lod_cleanup_desc_tgts(env, lod, &lod->lod_mdt_descs, lcfg); lod_cleanup_desc_tgts(env, lod, &lod->lod_ost_descs, lcfg); + + lod_seq_fini_cli(lod); /* * do cleanup on underlying storage only when * all OSPs are cleaned up, as they use that OSD as well diff --git a/lustre/mdt/mdt_handler.c b/lustre/mdt/mdt_handler.c index 8132a6a..41cc425 100644 --- a/lustre/mdt/mdt_handler.c +++ b/lustre/mdt/mdt_handler.c @@ -3914,100 +3914,6 @@ out_seq_fini: } /* - * Init client sequence manager which is used by local MDS to talk to sequence - * controller on remote node. - */ -static int mdt_seq_init_cli(const struct lu_env *env, - struct mdt_device *m, - struct lustre_cfg *cfg) -{ - struct seq_server_site *ss = mdt_seq_site(m); - struct obd_device *mdc; - int rc; - int index; - struct mdt_thread_info *info; - char *p, *index_string = lustre_cfg_string(cfg, 2); - ENTRY; - - info = lu_context_key_get(&env->le_ctx, &mdt_thread_key); - - LASSERT(index_string); - index = simple_strtol(index_string, &p, 10); - if (*p) { - CERROR("%s: Invalid index in lustre_cgf, offset 2\n", - mdt2obd_dev(m)->obd_name); - RETURN(-EINVAL); - } - - /* check if this is adding the first MDC and controller is not yet - * initialized. */ - if (index != 0 || ss->ss_client_seq) - RETURN(0); - - mdc = class_name2obd(lustre_cfg_string(cfg, 1)); - if (!mdc) { - CERROR("%s: can't find %s device\n", - mdt2obd_dev(m)->obd_name, lustre_cfg_string(cfg, 1)); - rc = -ENOENT; - } else if (!mdc->obd_set_up) { - CERROR("%s: target %s not set up\n", - mdt2obd_dev(m)->obd_name, mdc->obd_name); - rc = -EINVAL; - } else { - LASSERT(ss->ss_control_exp); - OBD_ALLOC_PTR(ss->ss_client_seq); - if (ss->ss_client_seq != NULL) { - char *prefix; - - OBD_ALLOC(prefix, MAX_OBD_NAME + 5); - if (!prefix) - RETURN(-ENOMEM); - - snprintf(prefix, MAX_OBD_NAME + 5, "ctl-%s", - mdc->obd_name); - - rc = seq_client_init(ss->ss_client_seq, - ss->ss_control_exp, - LUSTRE_SEQ_METADATA, - prefix, NULL); - OBD_FREE(prefix, MAX_OBD_NAME + 5); - } else - rc = -ENOMEM; - - if (rc) - RETURN(rc); - - LASSERT(ss->ss_server_seq != NULL); - rc = seq_server_set_cli(ss->ss_server_seq, ss->ss_client_seq, - env); - } - - RETURN(rc); -} - -static void mdt_seq_fini_cli(struct mdt_device *m) -{ - struct seq_server_site *ss; - - ENTRY; - - ss = mdt_seq_site(m); - - if (ss == NULL) - RETURN_EXIT; - - if (ss->ss_server_seq) - seq_server_set_cli(ss->ss_server_seq, NULL, NULL); - - if (ss->ss_control_exp) { - class_export_put(ss->ss_control_exp); - ss->ss_control_exp = NULL; - } - - EXIT; -} - -/* * FLD wrappers */ static int mdt_fld_fini(const struct lu_env *env, @@ -4461,7 +4367,6 @@ static void mdt_fini(const struct lu_env *env, struct mdt_device *m) } mdt_seq_fini(env, m); - mdt_seq_fini_cli(m); mdt_fld_fini(env, m); sptlrpc_rule_set_free(&m->mdt_sptlrpc_rset); @@ -4620,6 +4525,14 @@ static int mdt_init0(const struct lu_env *env, struct mdt_device *m, if (rc) GOTO(err_fini_stack, rc); + rc = mdt_fld_init(env, obd->obd_name, m); + if (rc) + GOTO(err_lut, rc); + + rc = mdt_seq_init(env, obd->obd_name, m); + if (rc) + GOTO(err_fini_fld, rc); + snprintf(info->mti_u.ns_name, sizeof info->mti_u.ns_name, LUSTRE_MDT_NAME"-%p", m); m->mdt_namespace = ldlm_namespace_new(obd, info->mti_u.ns_name, @@ -4723,9 +4636,11 @@ err_free_ns: ldlm_namespace_free(m->mdt_namespace, NULL, 0); obd->obd_namespace = m->mdt_namespace = NULL; err_fini_seq: - mdt_seq_fini(env, m); - mdt_fld_fini(env, m); - tgt_fini(env, &m->mdt_lut); + mdt_seq_fini(env, m); +err_fini_fld: + mdt_fld_fini(env, m); +err_lut: + tgt_fini(env, &m->mdt_lut); err_fini_stack: mdt_stack_fini(env, m, md2lu_dev(m->mdt_child)); err_lmi: @@ -4805,17 +4720,6 @@ static int mdt_process_config(const struct lu_env *env, break; } - case LCFG_ADD_MDC: - /* - * Add mdc hook to get first MDT uuid and connect it to - * ls->controller to use for seq manager. - */ - rc = next->ld_ops->ldo_process_config(env, next, cfg); - if (rc) - CERROR("Can't add mdc, rc %d\n", rc); - else - rc = mdt_seq_init_cli(env, mdt_dev(d), cfg); - break; default: /* others are passed further */ rc = next->ld_ops->ldo_process_config(env, next, cfg); @@ -4920,14 +4824,6 @@ static int mdt_prepare(const struct lu_env *env, if (rc) RETURN(rc); - rc = mdt_fld_init(env, obd->obd_name, mdt); - if (rc) - RETURN(rc); - - rc = mdt_seq_init(env, obd->obd_name, mdt); - if (rc) - RETURN(rc); - rc = mdt->mdt_child->md_ops->mdo_root_get(env, mdt->mdt_child, &mdt->mdt_md_root_fid); if (rc) -- 1.8.3.1