From 74ec68346e14851ad8a1912185e1dccd3e6d12cd Mon Sep 17 00:00:00 2001 From: wangdi Date: Fri, 8 Nov 2013 03:43:32 -0800 Subject: [PATCH] LU-1187 lod: Fix config log and setup process for DNE OSP will be used for local proxy client for remote MDT, and LOD will be used to manage these OSPs. 1. Add DNE config log in mgs. 2. Lod_add_device will reorganize MD OSP and add them. 3. In osp_init0 and remove some unnecessary init stuff for MD OSP. 4. Check whether osp is on OST or on MDT during client setup. 5. Only stop MDT0--osp--MDT0 in lustre_disconnect_osp, and other OSPs on MDT will be stoped in mdt stack cleanup. Signed-off-by: wang di Change-Id: I1c8e24edbb95d638500dacd2f9a4e423f1007170 Reviewed-on: http://review.whamcloud.com/4922 Tested-by: Hudson Reviewed-by: Alex Zhuravlev Reviewed-by: Andreas Dilger Tested-by: Maloo --- lustre/include/lustre/lustre_idl.h | 7 -- lustre/include/obd.h | 70 ++++++++++- lustre/ldlm/ldlm_lib.c | 25 ++-- lustre/lod/lod_dev.c | 75 ++++++++++-- lustre/lod/lod_internal.h | 5 +- lustre/lod/lod_lov.c | 93 ++++++++++---- lustre/mdt/mdt_handler.c | 61 ++++------ lustre/mgs/mgs_llog.c | 244 ++++++++++++++++++++++--------------- lustre/obdclass/obd_mount.c | 28 ++++- lustre/osd-ldiskfs/osd_handler.c | 3 + lustre/osp/lproc_osp.c | 34 +++--- lustre/osp/osp_dev.c | 233 +++++++++++++++++++++++------------ lustre/osp/osp_internal.h | 8 +- lustre/osp/osp_ost.c | 3 +- 14 files changed, 606 insertions(+), 283 deletions(-) diff --git a/lustre/include/lustre/lustre_idl.h b/lustre/include/lustre/lustre_idl.h index fa0e8b8..50151d5 100644 --- a/lustre/include/lustre/lustre_idl.h +++ b/lustre/include/lustre/lustre_idl.h @@ -354,13 +354,6 @@ struct hsm_attrs { }; extern void lustre_hsm_swab(struct hsm_attrs *attrs); -/* This is the maximum number of MDTs allowed in CMD testing until such - * a time that FID-on-OST is implemented. This is due to the limitations - * of packing non-0-MDT numbers into the FID SEQ namespace. Once FID-on-OST - * is implemented this limit will be virtually unlimited. */ -#define MAX_MDT_COUNT 8 - - /** * fid constants */ diff --git a/lustre/include/obd.h b/lustre/include/obd.h index 2b21509..3c28ff0 100644 --- a/lustre/include/obd.h +++ b/lustre/include/obd.h @@ -797,12 +797,46 @@ struct niobuf_local { #define LUSTRE_MGS_OBDNAME "MGS" #define LUSTRE_MGC_OBDNAME "MGC" +static inline int is_osp_on_mdt(char *name) +{ + char *ptr; + + ptr = strrchr(name, '-'); + if (ptr == NULL) { + CERROR("%s is not a obdname\n", name); + return 0; + } + + /* 1.8 OSC/OSP name on MDT is fsname-OSTxxxx-osc */ + if (strncmp(ptr + 1, "osc", 3) == 0) + return 1; + + if (strncmp(ptr + 1, "MDT", 3) != 0) + return 0; + + while (*(--ptr) != '-' && ptr != name); + + if (ptr == name) + return 0; + + if (strncmp(ptr + 1, LUSTRE_OSP_NAME, strlen(LUSTRE_OSP_NAME)) != 0 && + strncmp(ptr + 1, LUSTRE_OSC_NAME, strlen(LUSTRE_OSC_NAME)) != 0) + return 0; + + return 1; +} + /* Don't conflict with on-wire flags OBD_BRW_WRITE, etc */ #define N_LOCAL_TEMP_PAGE 0x10000000 -static inline int is_osp_on_ost(char *name) +/* Currently the connection osp is only for connecting MDT0, the + * name would either be + * fsname-MDT0000-osp-OSTxxxx or fsname-MDT0000-osp-MDT0000 */ +static inline int is_osp_for_connection(char *name) { - char *ptr; + char *ptr; + int osp_on_mdt0 = 0; + char *endptr; ptr = strrchr(name, '-'); if (ptr == NULL) { @@ -813,6 +847,19 @@ static inline int is_osp_on_ost(char *name) if (strncmp(ptr + 1, "OST", 3) != 0 && strncmp(ptr + 1, "MDT", 3) != 0) return 0; + if (strncmp(ptr + 1, "MDT", 3) == 0) { + int index; + +#ifdef __KERNEL__ + index = simple_strtoul(ptr + 4, &endptr, 16); +#else + index = strtoul(ptr + 4, &endptr, 16); +#endif + if (index != 0) + return 0; + osp_on_mdt0 = 1; + } + /* match the "-osp" */ if (ptr - name < strlen(LUSTRE_OSP_NAME) + 1) return 0; @@ -824,6 +871,25 @@ static inline int is_osp_on_ost(char *name) if (strncmp(ptr + 1, LUSTRE_OSP_NAME, strlen(LUSTRE_OSP_NAME)) != 0) return 0; + if (osp_on_mdt0) { + int index = 0; + while (*(--ptr) != '-' && ptr != name); + + if (ptr == name) { + CERROR("%s is not a valid osp name\n", name); + return 0; + } + + if (strncmp(ptr + 1, "MDT", 3) != 0) + return 0; +#ifdef __KERNEL__ + index = simple_strtoul(ptr + 4, &endptr, 16); +#else + index = strtoul(ptr + 4, &endptr, 16); +#endif + if (index != 0) + return 0; + } return 1; } diff --git a/lustre/ldlm/ldlm_lib.c b/lustre/ldlm/ldlm_lib.c index 540c6bb..b4f7a6c 100644 --- a/lustre/ldlm/ldlm_lib.c +++ b/lustre/ldlm/ldlm_lib.c @@ -270,30 +270,34 @@ int client_obd_setup(struct obd_device *obddev, struct lustre_cfg *lcfg) char *name = obddev->obd_type->typ_name; ldlm_ns_type_t ns_type = LDLM_NS_TYPE_UNKNOWN; int rc; + char *cli_name = lustre_cfg_buf(lcfg, 0); ENTRY; /* In a more perfect world, we would hang a ptlrpc_client off of * obd_type and just use the values from there. */ if (!strcmp(name, LUSTRE_OSC_NAME) || - (!strcmp(name, LUSTRE_OSP_NAME) && - !is_osp_on_ost(lustre_cfg_buf(lcfg, 0)))) { + (!(strcmp(name, LUSTRE_OSP_NAME)) && + (is_osp_on_mdt(cli_name) && + strstr(lustre_cfg_buf(lcfg, 1), "OST") != NULL))) { + /* OSC or OSP_on_MDT for OSTs */ rq_portal = OST_REQUEST_PORTAL; rp_portal = OSC_REPLY_PORTAL; connect_op = OST_CONNECT; cli->cl_sp_me = LUSTRE_SP_CLI; cli->cl_sp_to = LUSTRE_SP_OST; ns_type = LDLM_NS_TYPE_OSC; - } else if (!strcmp(name, LUSTRE_MDC_NAME) || (!strcmp(name, LUSTRE_OSP_NAME) && - is_osp_on_ost(lustre_cfg_buf(lcfg, 0)))) { + (is_osp_for_connection(cli_name) || + (is_osp_on_mdt(cli_name) && + strstr(lustre_cfg_buf(lcfg, 1), "OST") == NULL)))) { + /* MDC or OSP_on_MDT for other MDTs */ rq_portal = MDS_REQUEST_PORTAL; rp_portal = MDC_REPLY_PORTAL; connect_op = MDS_CONNECT; cli->cl_sp_me = LUSTRE_SP_CLI; cli->cl_sp_to = LUSTRE_SP_MDT; ns_type = LDLM_NS_TYPE_MDC; - } else if (!strcmp(name, LUSTRE_MGC_NAME)) { rq_portal = MGS_REQUEST_PORTAL; rp_portal = MGC_REPLY_PORTAL; @@ -302,16 +306,7 @@ int client_obd_setup(struct obd_device *obddev, struct lustre_cfg *lcfg) cli->cl_sp_to = LUSTRE_SP_MGS; cli->cl_flvr_mgc.sf_rpc = SPTLRPC_FLVR_INVALID; ns_type = LDLM_NS_TYPE_MGC; - - } else if (!strcmp(name, LUSTRE_OSP_NAME)) { - rq_portal = OST_REQUEST_PORTAL; - rp_portal = OSC_REPLY_PORTAL; - connect_op = OST_CONNECT; - cli->cl_sp_me = LUSTRE_SP_CLI; - cli->cl_sp_to = LUSTRE_SP_OST; - ns_type = LDLM_NS_TYPE_OSC; - - } else { + } else { CERROR("unknown client OBD type \"%s\", can't setup\n", name); RETURN(-EINVAL); diff --git a/lustre/lod/lod_dev.c b/lustre/lod/lod_dev.c index 194e496..608f754 100644 --- a/lustre/lod/lod_dev.c +++ b/lustre/lod/lod_dev.c @@ -165,6 +165,40 @@ static int lod_cleanup_desc_tgts(const struct lu_env *env, return rc; } +static int lodname2mdt_index(char *lodname, int *index) +{ + char *ptr, *tmp; + + /* The lodname suppose to be fsname-MDTxxxx-mdtlov */ + ptr = strrchr(lodname, '-'); + if (ptr == NULL) { + CERROR("invalid MDT index in '%s'\n", lodname); + return -EINVAL; + } + + if (strncmp(ptr, "-mdtlov", 7) != 0) { + CERROR("invalid MDT index in '%s'\n", lodname); + return -EINVAL; + } + + if ((unsigned long)ptr - (unsigned long)lodname <= 8) { + CERROR("invalid MDT index in '%s'\n", lodname); + return -EINVAL; + } + + if (strncmp(ptr - 8, "-MDT", 4) != 0) { + CERROR("invalid MDT index in '%s'\n", lodname); + return -EINVAL; + } + + *index = simple_strtol(ptr - 4, &tmp, 16); + if (*tmp != '-' || *index > INT_MAX || *index < 0) { + CERROR("invalid MDT index in '%s'\n", lodname); + return -EINVAL; + } + return 0; +} + static int lod_process_config(const struct lu_env *env, struct lu_device *dev, struct lustre_cfg *lcfg) @@ -172,16 +206,21 @@ static int lod_process_config(const struct lu_env *env, struct lod_device *lod = lu2lod_dev(dev); struct lu_device *next = &lod->lod_child->dd_lu_dev; char *arg1; - int rc; + int rc = 0; ENTRY; switch(lcfg->lcfg_command) { case LCFG_LOV_DEL_OBD: case LCFG_LOV_ADD_INA: - case LCFG_LOV_ADD_OBD: { + case LCFG_LOV_ADD_OBD: + case LCFG_ADD_MDC: { __u32 index; + __u32 mdt_index; int gen; - /* lov_modify_tgts add 0:lov_mdsA 1:osp 2:0 3:1 */ + /* lov_modify_tgts add 0:lov_mdsA 1:osp 2:0 3:1 + * modify_mdc_tgts add 0:lustre-MDT0001 + * 1:lustre-MDT0001-mdc0002 + * 2:2 3:1*/ arg1 = lustre_cfg_string(lcfg, 1); if (sscanf(lustre_cfg_buf(lcfg, 2), "%d", &index) != 1) @@ -189,14 +228,34 @@ static int lod_process_config(const struct lu_env *env, if (sscanf(lustre_cfg_buf(lcfg, 3), "%d", &gen) != 1) GOTO(out, rc = -EINVAL); - if (lcfg->lcfg_command == LCFG_LOV_ADD_OBD) - rc = lod_add_device(env, lod, arg1, index, gen, 1); - else if (lcfg->lcfg_command == LCFG_LOV_ADD_INA) - rc = lod_add_device(env, lod, arg1, index, gen, 0); - else + if (lcfg->lcfg_command == LCFG_LOV_ADD_OBD) { + char *mdt; + mdt = strstr(lustre_cfg_string(lcfg, 0), "-MDT"); + /* 1.8 configs don't have "-MDT0000" at the end */ + if (mdt == NULL) { + mdt_index = 0; + } else { + rc = lodname2mdt_index( + lustre_cfg_string(lcfg, 0), &mdt_index); + if (rc != 0) + GOTO(out, rc); + } + rc = lod_add_device(env, lod, arg1, index, gen, + mdt_index, LUSTRE_OSC_NAME, 1); + } else if (lcfg->lcfg_command == LCFG_ADD_MDC) { + mdt_index = index; + rc = lod_add_device(env, lod, arg1, index, gen, + mdt_index, LUSTRE_MDC_NAME, 1); + } else if (lcfg->lcfg_command == LCFG_LOV_ADD_INA) { + /*FIXME: Add mdt_index for LCFG_LOV_ADD_INA*/ + mdt_index = 0; + rc = lod_add_device(env, lod, arg1, index, gen, + mdt_index, LUSTRE_OSC_NAME, 0); + } else { rc = lod_del_device(env, lod, &lod->lod_ost_descs, arg1, index, gen); + } break; } diff --git a/lustre/lod/lod_internal.h b/lustre/lod/lod_internal.h index a07201f..719887f 100644 --- a/lustre/lod/lod_internal.h +++ b/lustre/lod/lod_internal.h @@ -272,8 +272,9 @@ int lod_fld_lookup(const struct lu_env *env, struct lod_device *lod, /* lod_lov.c */ void lod_getref(struct lod_tgt_descs *ltd); void lod_putref(struct lod_device *lod, struct lod_tgt_descs *ltd); -int lod_add_device(const struct lu_env *env, struct lod_device *m, - char *osp, unsigned index, unsigned gen, int active); +int lod_add_device(const struct lu_env *env, struct lod_device *lod, + char *osp, unsigned index, unsigned gen, int mdt_index, + char *type, int active); int lod_del_device(const struct lu_env *env, struct lod_device *lod, struct lod_tgt_descs *ltd, char *osp, unsigned idx, unsigned gen); diff --git a/lustre/lod/lod_lov.c b/lustre/lod/lod_lov.c index 81a5ed1..0800e53 100644 --- a/lustre/lod/lod_lov.c +++ b/lustre/lod/lod_lov.c @@ -171,9 +171,12 @@ out: * \param osp - is the name of OSP device name about to be added * \param index - is the OSP index * \param gen - is the generation number + * \param tgt_index - is the group of the OSP. + * \param type - is the type of device (mdc or osc) */ int lod_add_device(const struct lu_env *env, struct lod_device *lod, - char *osp, unsigned index, unsigned gen, int active) + char *osp, unsigned index, unsigned gen, int tgt_index, + char *type, int active) { struct obd_connect_data *data = NULL; struct obd_export *exp = NULL; @@ -182,7 +185,7 @@ int lod_add_device(const struct lu_env *env, struct lod_device *lod, struct dt_device *d; int rc; struct lod_tgt_desc *tgt_desc; - struct lod_tgt_descs *ltd = &lod->lod_ost_descs; + struct lod_tgt_descs *ltd; struct obd_uuid obd_uuid; ENTRY; @@ -211,6 +214,47 @@ int lod_add_device(const struct lu_env *env, struct lod_device *lod, data->ocd_version = LUSTRE_VERSION_CODE; data->ocd_index = index; + if (strcmp(LUSTRE_OSC_NAME, type) == 0) { + data->ocd_connect_flags |= OBD_CONNECT_AT | + OBD_CONNECT_FULL20 | + OBD_CONNECT_INDEX | +#ifdef HAVE_LRU_RESIZE_SUPPORT + OBD_CONNECT_LRU_RESIZE | +#endif + OBD_CONNECT_MDS | + OBD_CONNECT_OSS_CAPA | + OBD_CONNECT_REQPORTAL | + OBD_CONNECT_SKIP_ORPHAN | + OBD_CONNECT_FID | + OBD_CONNECT_LVB_TYPE | + OBD_CONNECT_VERSION; + + data->ocd_group = tgt_index; + ltd = &lod->lod_ost_descs; + } else { + struct obd_import *imp = obd->u.cli.cl_import; + + data->ocd_ibits_known = MDS_INODELOCK_UPDATE; + data->ocd_connect_flags |= OBD_CONNECT_ACL | + OBD_CONNECT_MDS_CAPA | + OBD_CONNECT_OSS_CAPA | + OBD_CONNECT_IBITS | + OBD_CONNECT_MDS_MDS | + OBD_CONNECT_FID | + OBD_CONNECT_AT | + OBD_CONNECT_FULL20; + /* XXX set MDS-MDS flags, remove this when running this + * on client*/ + data->ocd_connect_flags |= OBD_CONNECT_MDS_MDS; + spin_lock(&imp->imp_lock); + imp->imp_server_timeout = 1; + spin_unlock(&imp->imp_lock); + imp->imp_client->cli_request_portal = MDS_MDS_PORTAL; + CDEBUG(D_OTHER, "%s: Set 'mds' portal and timeout\n", + obd->obd_name); + ltd = &lod->lod_mdt_descs; + } + rc = obd_connect(env, &exp, obd, &obd->obd_uuid, data, NULL); OBD_FREE_PTR(data); if (rc) { @@ -273,27 +317,29 @@ int lod_add_device(const struct lu_env *env, struct lod_device *lod, } } - /* pool and qos are not supported for MDS stack yet */ - rc = lod_ost_pool_add(&lod->lod_pool_info, index, - lod->lod_osts_size); - if (rc) { - CERROR("%s: can't set up pool, failed with %d\n", - obd->obd_name, rc); - GOTO(out_mutex, rc); - } + if (!strcmp(LUSTRE_OSC_NAME, type)) { + /* pool and qos are not supported for MDS stack yet */ + rc = lod_ost_pool_add(&lod->lod_pool_info, index, + lod->lod_osts_size); + if (rc) { + CERROR("%s: can't set up pool, failed with %d\n", + obd->obd_name, rc); + GOTO(out_mutex, rc); + } - rc = qos_add_tgt(lod, tgt_desc); - if (rc) { - CERROR("%s: qos_add_tgt failed with %d\n", - obd->obd_name, rc); - GOTO(out_pool, rc); - } + rc = qos_add_tgt(lod, tgt_desc); + if (rc) { + CERROR("%s: qos_add_tgt failed with %d\n", + obd->obd_name, rc); + GOTO(out_pool, rc); + } - /* The new OST is now a full citizen */ - if (index >= lod->lod_desc.ld_tgt_count) - lod->lod_desc.ld_tgt_count = index + 1; - if (active) - lod->lod_desc.ld_active_tgt_count++; + /* The new OST is now a full citizen */ + if (index >= lod->lod_desc.ld_tgt_count) + lod->lod_desc.ld_tgt_count = index + 1; + if (active) + lod->lod_desc.ld_active_tgt_count++; + } LTD_TGT(ltd, index) = tgt_desc; cfs_bitmap_set(ltd->ltd_tgt_bitmap, index); @@ -376,7 +422,10 @@ int lod_del_device(const struct lu_env *env, struct lod_device *lod, CDEBUG(D_CONFIG, "osp:%s idx:%d gen:%d\n", osp, idx, gen); - obd = class_name2obd(osp); + obd_str2uuid(&uuid, osp); + + obd = class_find_client_obd(&uuid, LUSTRE_OSP_NAME, + &lod->lod_dt_dev.dd_lu_dev.ld_obd->obd_uuid); if (obd == NULL) { CERROR("can't find %s device\n", osp); RETURN(-EINVAL); diff --git a/lustre/mdt/mdt_handler.c b/lustre/mdt/mdt_handler.c index 4e266be..27ebdbb 100644 --- a/lustre/mdt/mdt_handler.c +++ b/lustre/mdt/mdt_handler.c @@ -3908,6 +3908,7 @@ out_seq_fini: return rc; } + /* * Init client sequence manager which is used by local MDS to talk to sequence * controller on remote node. @@ -3917,46 +3918,38 @@ static int mdt_seq_init_cli(const struct lu_env *env, struct lustre_cfg *cfg) { struct seq_server_site *ss = mdt_seq_site(m); - struct obd_device *mdc; - struct obd_uuid *uuidp, *mdcuuidp; - char *uuid_str, *mdc_uuid_str; - int rc; - int index; - struct mdt_thread_info *info; - char *p, *index_string = lustre_cfg_string(cfg, 2); - ENTRY; - - info = lu_context_key_get(&env->le_ctx, &mdt_thread_key); - uuidp = &info->mti_u.uuid[0]; - mdcuuidp = &info->mti_u.uuid[1]; + struct obd_device *mdc; + int rc; + int index; + struct mdt_thread_info *info; + char *p, *index_string = lustre_cfg_string(cfg, 2); + ENTRY; - LASSERT(index_string); + info = lu_context_key_get(&env->le_ctx, &mdt_thread_key); - index = simple_strtol(index_string, &p, 10); - if (*p) { - CERROR("Invalid index in lustre_cgf, offset 2\n"); - RETURN(-EINVAL); - } + LASSERT(index_string); + index = simple_strtol(index_string, &p, 10); + if (*p) { + CERROR("%s: Invalid index in lustre_cgf, offset 2\n", + mdt2obd_dev(m)->obd_name); + RETURN(-EINVAL); + } /* check if this is adding the first MDC and controller is not yet * initialized. */ if (index != 0 || ss->ss_client_seq) RETURN(0); - uuid_str = lustre_cfg_string(cfg, 1); - mdc_uuid_str = lustre_cfg_string(cfg, 4); - obd_str2uuid(uuidp, uuid_str); - obd_str2uuid(mdcuuidp, mdc_uuid_str); - - mdc = class_find_client_obd(uuidp, LUSTRE_MDC_NAME, mdcuuidp); - if (!mdc) { - CERROR("can't find controller MDC by uuid %s\n", - uuid_str); - rc = -ENOENT; - } else if (!mdc->obd_set_up) { - CERROR("target %s not set up\n", mdc->obd_name); - rc = -EINVAL; - } else { + mdc = class_name2obd(lustre_cfg_string(cfg, 1)); + if (!mdc) { + CERROR("%s: can't find %s device\n", + mdt2obd_dev(m)->obd_name, lustre_cfg_string(cfg, 1)); + rc = -ENOENT; + } else if (!mdc->obd_set_up) { + CERROR("%s: target %s not set up\n", + mdt2obd_dev(m)->obd_name, mdc->obd_name); + rc = -EINVAL; + } else { LASSERT(ss->ss_control_exp); OBD_ALLOC_PTR(ss->ss_client_seq); if (ss->ss_client_seq != NULL) { @@ -3983,9 +3976,9 @@ static int mdt_seq_init_cli(const struct lu_env *env, LASSERT(ss->ss_server_seq != NULL); rc = seq_server_set_cli(ss->ss_server_seq, ss->ss_client_seq, env); - } + } - RETURN(rc); + RETURN(rc); } static void mdt_seq_fini_cli(struct mdt_device *m) diff --git a/lustre/mgs/mgs_llog.c b/lustre/mgs/mgs_llog.c index 49a12d7..cf43a4b 100644 --- a/lustre/mgs/mgs_llog.c +++ b/lustre/mgs/mgs_llog.c @@ -548,11 +548,6 @@ static int mgs_set_index(const struct lu_env *env, imap = fsdb->fsdb_ost_index_map; } else if (mti->mti_flags & LDD_F_SV_TYPE_MDT) { imap = fsdb->fsdb_mdt_index_map; - if (fsdb->fsdb_mdt_count >= MAX_MDT_COUNT) { - LCONSOLE_ERROR_MSG(0x13f, "The max mdt count" - "is %d\n", (int)MAX_MDT_COUNT); - GOTO(out_up, rc = -ERANGE); - } } else { GOTO(out_up, rc = -EINVAL); } @@ -1383,7 +1378,7 @@ static inline int record_lov_add(const struct lu_env *env, char *index, char *gen) { return record_base(env,llh,lov_name,0,LCFG_LOV_ADD_OBD, - ost_uuid,index,gen,0); + ost_uuid, index, gen, 0); } static inline int record_mount_opt(const struct lu_env *env, @@ -1607,16 +1602,16 @@ next: RETURN(rc); } -static int mgs_write_log_mdc_to_mdt(const struct lu_env *env, +static int mgs_write_log_osp_to_mdt(const struct lu_env *env, struct mgs_device *mgs, struct fs_db *fsdb, struct mgs_target_info *mti, - char *logname); + int index, char *logname); static int mgs_write_log_osc_to_lov(const struct lu_env *env, struct mgs_device *mgs, struct fs_db *fsdb, struct mgs_target_info *mti, - char *logname, char *suffix, char *lovname, + char *logname, char *suffix, char *lovname, enum lustre_sec_part sec_part, int flags); static int name_create_mdt_and_lov(char **logname, char **lovname, struct fs_db *fsdb, int i); @@ -1685,7 +1680,8 @@ static int mgs_steal_llog_handler(const struct lu_env *env, struct cfg_marker *marker; marker = lustre_cfg_buf(lcfg, 1); if (!strncmp(marker->cm_comment,"add osc",7) && - (marker->cm_flags & CM_START)){ + (marker->cm_flags & CM_START) && + !(marker->cm_flags & CM_SKIP)) { got_an_osc_or_mdc = 1; strncpy(tmti->mti_svname, marker->cm_tgtname, sizeof(tmti->mti_svname)); @@ -1700,7 +1696,8 @@ static int mgs_steal_llog_handler(const struct lu_env *env, RETURN(rc); } if (!strncmp(marker->cm_comment,"add osc",7) && - (marker->cm_flags & CM_END)){ + (marker->cm_flags & CM_END) && + !(marker->cm_flags & CM_SKIP)) { LASSERT(last_step == marker->cm_step); last_step = -1; got_an_osc_or_mdc = 0; @@ -1714,7 +1711,8 @@ static int mgs_steal_llog_handler(const struct lu_env *env, RETURN(rc); } if (!strncmp(marker->cm_comment,"add mdc",7) && - (marker->cm_flags & CM_START)){ + (marker->cm_flags & CM_START) && + !(marker->cm_flags & CM_SKIP)) { got_an_osc_or_mdc = 2; last_step = marker->cm_step; memcpy(tmti->mti_svname, marker->cm_tgtname, @@ -1723,7 +1721,8 @@ static int mgs_steal_llog_handler(const struct lu_env *env, RETURN(rc); } if (!strncmp(marker->cm_comment,"add mdc",7) && - (marker->cm_flags & CM_END)){ + (marker->cm_flags & CM_END) && + !(marker->cm_flags & CM_SKIP)) { LASSERT(last_step == marker->cm_step); last_step = -1; got_an_osc_or_mdc = 0; @@ -1774,7 +1773,8 @@ static int mgs_steal_llog_handler(const struct lu_env *env, strlen(mti->mti_fsname)); tmti->mti_stripe_index = index; - rc = mgs_write_log_mdc_to_mdt(env, mgs, fsdb, tmti, + rc = mgs_write_log_osp_to_mdt(env, mgs, fsdb, tmti, + mti->mti_stripe_index, mti->mti_svname); memset(tmti, 0, sizeof(*tmti)); RETURN(rc); @@ -2087,83 +2087,158 @@ out_free: RETURN(rc); } +static inline int name_create_lov(char **lovname, char *mdtname, + struct fs_db *fsdb, int index) +{ + /* COMPAT_180 */ + if (index == 0 && test_bit(FSDB_OSCNAME18, &fsdb->fsdb_flags)) + return name_create(lovname, fsdb->fsdb_name, "-mdtlov"); + else + return name_create(lovname, mdtname, "-mdtlov"); +} + +static int name_create_mdt_and_lov(char **logname, char **lovname, + struct fs_db *fsdb, int i) +{ + int rc; + + rc = name_create_mdt(logname, fsdb->fsdb_name, i); + if (rc) + return rc; + /* COMPAT_180 */ + if (i == 0 && test_bit(FSDB_OSCNAME18, &fsdb->fsdb_flags)) + rc = name_create(lovname, fsdb->fsdb_name, "-mdtlov"); + else + rc = name_create(lovname, *logname, "-mdtlov"); + if (rc) { + name_destroy(logname); + *logname = NULL; + } + return rc; +} + +static inline int name_create_mdt_osc(char **oscname, char *ostname, + struct fs_db *fsdb, int i) +{ + char suffix[16]; + + if (i == 0 && test_bit(FSDB_OSCNAME18, &fsdb->fsdb_flags)) + sprintf(suffix, "-osc"); + else + sprintf(suffix, "-osc-MDT%04x", i); + return name_create(oscname, ostname, suffix); +} + /* add new mdc to already existent MDS */ -static int mgs_write_log_mdc_to_mdt(const struct lu_env *env, +static int mgs_write_log_osp_to_mdt(const struct lu_env *env, struct mgs_device *mgs, struct fs_db *fsdb, struct mgs_target_info *mti, - char *logname) -{ - struct llog_handle *llh = NULL; - char *nodeuuid = NULL; - char *mdcname = NULL; - char *mdcuuid = NULL; - char *mdtuuid = NULL; - int idx = mti->mti_stripe_index; - char index[9]; - int i, rc; + int mdt_index, char *logname) +{ + struct llog_handle *llh = NULL; + char *nodeuuid = NULL; + char *ospname = NULL; + char *lovuuid = NULL; + char *mdtuuid = NULL; + char *svname = NULL; + char *mdtname = NULL; + char *lovname = NULL; + char index_str[16]; + int i, rc; - ENTRY; - if (mgs_log_is_empty(env, mgs, logname)) { + ENTRY; + if (mgs_log_is_empty(env, mgs, mti->mti_svname)) { CERROR("log is empty! Logical error\n"); RETURN (-EINVAL); } - CDEBUG(D_MGS, "adding mdc index %d to %s\n", idx, logname); + CDEBUG(D_MGS, "adding osp index %d to %s\n", mti->mti_stripe_index, + logname); - rc = name_create(&nodeuuid, libcfs_nid2str(mti->mti_nids[0]), ""); + rc = name_create_mdt(&mdtname, fsdb->fsdb_name, mti->mti_stripe_index); if (rc) RETURN(rc); - snprintf(index, sizeof(index), "-mdc%04x", idx); - rc = name_create(&mdcname, logname, index); + + rc = name_create(&nodeuuid, libcfs_nid2str(mti->mti_nids[0]), ""); if (rc) - GOTO(out_free, rc); - rc = name_create(&mdcuuid, mdcname, "_UUID"); + GOTO(out_destory, rc); + + rc = name_create(&svname, mdtname, "-osp"); if (rc) - GOTO(out_free, rc); - rc = name_create(&mdtuuid, logname, "_UUID"); + GOTO(out_destory, rc); + + sprintf(index_str, "-MDT%04x", mdt_index); + rc = name_create(&ospname, svname, index_str); if (rc) - GOTO(out_free, rc); + GOTO(out_destory, rc); + + rc = name_create_lov(&lovname, logname, fsdb, mdt_index); + if (rc) + GOTO(out_destory, rc); + + rc = name_create(&lovuuid, lovname, "_UUID"); + if (rc) + GOTO(out_destory, rc); + + rc = name_create(&mdtuuid, mdtname, "_UUID"); + if (rc) + GOTO(out_destory, rc); rc = record_start_log(env, mgs, &llh, logname); if (rc) - GOTO(out_free, rc); - rc = record_marker(env, llh, fsdb, CM_START, mti->mti_svname, "add mdc"); + GOTO(out_destory, rc); + + rc = record_marker(env, llh, fsdb, CM_START, mti->mti_svname, + "add osp"); if (rc) - GOTO(out_end, rc); - for (i = 0; i < mti->mti_nid_count; i++) { - CDEBUG(D_MGS, "add nid %s for mdt\n", - libcfs_nid2str(mti->mti_nids[i])); + GOTO(out_destory, rc); + + for (i = 0; i < mti->mti_nid_count; i++) { + CDEBUG(D_MGS, "add nid %s for mdt\n", + libcfs_nid2str(mti->mti_nids[i])); rc = record_add_uuid(env, llh, mti->mti_nids[i], nodeuuid); if (rc) GOTO(out_end, rc); } - rc = record_attach(env, llh, mdcname, LUSTRE_MDC_NAME, mdcuuid); + + rc = record_attach(env, llh, ospname, LUSTRE_OSP_NAME, lovuuid); if (rc) GOTO(out_end, rc); - rc = record_setup(env, llh, mdcname, mti->mti_uuid, nodeuuid, 0, 0); + + rc = record_setup(env, llh, ospname, mti->mti_uuid, nodeuuid, + NULL, NULL); if (rc) GOTO(out_end, rc); - rc = mgs_write_log_failnids(env, mti, llh, mdcname); + + rc = mgs_write_log_failnids(env, mti, llh, ospname); if (rc) GOTO(out_end, rc); - snprintf(index, sizeof(index), "%d", idx); - rc = record_mdc_add(env, llh, logname, mdcuuid, mti->mti_uuid, - index, "1"); + /* Add mdc(osp) to lod */ + snprintf(index_str, sizeof(mti->mti_stripe_index), "%d", + mti->mti_stripe_index); + rc = record_base(env, llh, lovname, 0, LCFG_ADD_MDC, mti->mti_uuid, + index_str, "1", NULL); if (rc) GOTO(out_end, rc); - rc = record_marker(env, llh, fsdb, CM_END, mti->mti_svname, "add mdc"); + + rc = record_marker(env, llh, fsdb, CM_END, mti->mti_svname, "add osp"); if (rc) GOTO(out_end, rc); + out_end: record_end_log(env, &llh); -out_free: + +out_destory: name_destroy(&mdtuuid); - name_destroy(&mdcuuid); - name_destroy(&mdcname); + name_destroy(&lovuuid); + name_destroy(&lovname); + name_destroy(&ospname); + name_destroy(&svname); name_destroy(&nodeuuid); - RETURN(rc); + name_destroy(&mdtname); + RETURN(rc); } static int mgs_write_log_mdt0(const struct lu_env *env, @@ -2230,38 +2305,6 @@ out_free: RETURN(rc); } -static int name_create_mdt_and_lov(char **logname, char **lovname, - struct fs_db *fsdb, int i) -{ - int rc; - - rc = name_create_mdt(logname, fsdb->fsdb_name, i); - if (rc) - return rc; - /* COMPAT_180 */ - if (i == 0 && test_bit(FSDB_OSCNAME18, &fsdb->fsdb_flags)) - rc = name_create(lovname, fsdb->fsdb_name, "-mdtlov"); - else - rc = name_create(lovname, *logname, "-mdtlov"); - if (rc) { - name_destroy(logname); - *logname = NULL; - } - return rc; -} - -static inline int name_create_mdt_osc(char **oscname, char *ostname, - struct fs_db *fsdb, int i) -{ - char suffix[16]; - - if (i == 0 && test_bit(FSDB_OSCNAME18, &fsdb->fsdb_flags)) - sprintf(suffix, "-osc"); - else - sprintf(suffix, "-osc-MDT%04x", i); - return name_create(oscname, ostname, suffix); -} - /* envelope method for all layers log */ static int mgs_write_log_mdt(const struct lu_env *env, struct mgs_device *mgs, @@ -2343,20 +2386,24 @@ static int mgs_write_log_mdt(const struct lu_env *env, if (rc) GOTO(out_end, rc); + /* for_all_existing_mdt except current one */ - for (i = 0; i < INDEX_MAP_SIZE * 8; i++){ - char *mdtname; - if (i != mti->mti_stripe_index && - test_bit(i, fsdb->fsdb_mdt_index_map)) { - rc = name_create_mdt(&mdtname, mti->mti_fsname, i); + for (i = 0; i < INDEX_MAP_SIZE * 8; i++) { + if (i != mti->mti_stripe_index && + test_bit(i, fsdb->fsdb_mdt_index_map)) { + char *logname; + + rc = name_create_mdt(&logname, fsdb->fsdb_name, i); if (rc) GOTO(out_end, rc); - rc = mgs_write_log_mdc_to_mdt(env, mgs, fsdb, mti, mdtname); - name_destroy(&mdtname); + + rc = mgs_write_log_osp_to_mdt(env, mgs, fsdb, mti, + i, logname); + name_destroy(&logname); if (rc) GOTO(out_end, rc); - } - } + } + } out_end: record_end_log(env, &llh); out_free: @@ -2395,6 +2442,7 @@ static int mgs_write_log_osc_to_lov(const struct lu_env *env, rc = name_create(&svname, mti->mti_svname, "-osc"); if (rc) GOTO(out_free, rc); + /* for the system upgraded from old 1.8, keep using the old osc naming * style for mdt, see name_create_mdt_osc(). LU-1257 */ if (test_bit(FSDB_OSCNAME18, &fsdb->fsdb_flags)) @@ -2403,6 +2451,7 @@ static int mgs_write_log_osc_to_lov(const struct lu_env *env, rc = name_create(&oscname, svname, suffix); if (rc) GOTO(out_free, rc); + rc = name_create(&oscuuid, oscname, "_UUID"); if (rc) GOTO(out_free, rc); @@ -2410,6 +2459,7 @@ static int mgs_write_log_osc_to_lov(const struct lu_env *env, if (rc) GOTO(out_free, rc); + /* #03 L add_uuid nid=uml1@tcp(0x20000c0a80201) 0: 1:uml1_UUID multihomed (#4) @@ -2452,7 +2502,9 @@ static int mgs_write_log_osc_to_lov(const struct lu_env *env, rc = mgs_write_log_failnids(env, mti, llh, oscname); if (rc) GOTO(out_end, rc); - snprintf(index, sizeof(index), "%d", mti->mti_stripe_index); + + snprintf(index, sizeof(index), "%d", mti->mti_stripe_index); + rc = record_lov_add(env, llh, lovname, mti->mti_uuid, index, "1"); if (rc) GOTO(out_end, rc); @@ -2863,7 +2915,7 @@ static int mgs_write_log_quota(const struct lu_env *env, struct mgs_device *mgs, mti->mti_fsname, quota, 1); *ptr = sep; lustre_cfg_free(lcfg); - return rc; + return rc < 0 ? rc : 0; } static int mgs_srpc_set_param_disk(const struct lu_env *env, diff --git a/lustre/obdclass/obd_mount.c b/lustre/obdclass/obd_mount.c index 49df455..3eaf93f 100644 --- a/lustre/obdclass/obd_mount.c +++ b/lustre/obdclass/obd_mount.c @@ -1325,6 +1325,18 @@ static int lustre_disconnect_osp(struct super_block *sb) ENTRY; LASSERT(IS_OST(lsi) || IS_MDT(lsi)); + if (IS_MDT(lsi)) { + int index; + + /* Only disconnect MDT0-osp-MDT0 here, other osp on MDT + * will be disconnect during MDT stack cleanup. + * FIXME: remove later when quota on DNE is finished */ + rc = server_name2index(lsi->lsi_svname, &index, NULL); + if (rc < 0) + RETURN(rc); + if (index != 0) + RETURN(0); + } OBD_ALLOC(logname, MTI_NAME_MAXLEN); if (logname == NULL) RETURN(-ENOMEM); @@ -1395,7 +1407,7 @@ out: } /** - * Stop the osp(fsname-MDT0000-osp-OSTxxxx) for an OST target. + * Stop the osp(fsname-MDT0000-osp-OSTxxxx) or (fsname-MDT0000-osp-MDT0000) for an OST target. **/ static int lustre_stop_osp(struct super_block *sb) { @@ -1406,8 +1418,20 @@ static int lustre_stop_osp(struct super_block *sb) ENTRY; LASSERT(IS_OST(lsi) || IS_MDT(lsi)); - OBD_ALLOC(ospname, MTI_NAME_MAXLEN); + if (IS_MDT(lsi)) { + int index; + + /* Only disconnect MDT0-osp-MDT0 here, other osp on MDT + * will be disconnect during MDT stack cleanup. + * FIXME: remove later when quota on DNE is finished */ + rc = server_name2index(lsi->lsi_svname, &index, NULL); + if (rc < 0) + RETURN(rc); + if (index != 0) + RETURN(0); + } + OBD_ALLOC(ospname, MTI_NAME_MAXLEN); rc = tgt_name2ospname(lsi->lsi_svname, ospname); if (rc != 0) { CERROR("%s get fsname error: rc %d\n", diff --git a/lustre/osd-ldiskfs/osd_handler.c b/lustre/osd-ldiskfs/osd_handler.c index c07b928..6d0771e 100644 --- a/lustre/osd-ldiskfs/osd_handler.c +++ b/lustre/osd-ldiskfs/osd_handler.c @@ -4585,6 +4585,9 @@ static struct lu_device *osd_device_alloc(const struct lu_env *env, rc = dt_device_init(&o->od_dt_dev, t); if (rc == 0) { + /* Because the ctx might be revived in dt_device_init, + * refill the env here */ + lu_env_refill((struct lu_env *)env); rc = osd_device_init0(env, o, cfg); if (rc) dt_device_fini(&o->od_dt_dev); diff --git a/lustre/osp/lproc_osp.c b/lustre/osp/lproc_osp.c index c6dbbd9..790500c 100644 --- a/lustre/osp/lproc_osp.c +++ b/lustre/osp/lproc_osp.c @@ -479,22 +479,24 @@ void osp_lprocfs_init(struct osp_device *osp) ptlrpc_lprocfs_register_obd(obd); /* for compatibility we link old procfs's OSC entries to osp ones */ - osc_proc_dir = lprocfs_srch(proc_lustre_root, "osc"); - if (osc_proc_dir) { - cfs_proc_dir_entry_t *symlink = NULL; - char *name; - - OBD_ALLOC(name, strlen(obd->obd_name) + 1); - if (name == NULL) - return; - - strcpy(name, obd->obd_name); - if (strstr(name, "osc")) - symlink = lprocfs_add_symlink(name, osc_proc_dir, - "../osp/%s", - obd->obd_name); - OBD_FREE(name, strlen(obd->obd_name) + 1); - osp->opd_symlink = symlink; + if (!osp->opd_connect_mdt) { + osc_proc_dir = lprocfs_srch(proc_lustre_root, "osc"); + if (osc_proc_dir) { + cfs_proc_dir_entry_t *symlink = NULL; + char *name; + + OBD_ALLOC(name, strlen(obd->obd_name) + 1); + if (name == NULL) + return; + + strcpy(name, obd->obd_name); + if (strstr(name, "osc")) + symlink = lprocfs_add_symlink(name, + osc_proc_dir, "../osp/%s", + obd->obd_name); + OBD_FREE(name, strlen(obd->obd_name) + 1); + osp->opd_symlink = symlink; + } } } diff --git a/lustre/osp/osp_dev.c b/lustre/osp/osp_dev.c index 7357ec6..c8e490e 100644 --- a/lustre/osp/osp_dev.c +++ b/lustre/osp/osp_dev.c @@ -49,6 +49,7 @@ #include #include #include +#include #include "osp_internal.h" @@ -231,22 +232,25 @@ static int osp_shutdown(const struct lu_env *env, struct osp_device *d) int rc = 0; ENTRY; - if (is_osp_on_ost(d->opd_obd->obd_name)) { + if (is_osp_for_connection(d->opd_obd->obd_name)) { rc = osp_disconnect(d); RETURN(rc); } LASSERT(env); /* release last_used file */ - osp_last_used_fini(env, d); + if (!d->opd_connect_mdt) + osp_last_used_fini(env, d); rc = osp_disconnect(d); - /* stop precreate thread */ - osp_precreate_fini(d); + if (!d->opd_connect_mdt) { + /* stop precreate thread */ + osp_precreate_fini(d); - /* stop sync thread */ - osp_sync_fini(d); + /* stop sync thread */ + osp_sync_fini(d); + } obd_fid_fini(d->opd_obd); @@ -264,7 +268,7 @@ static int osp_process_config(const struct lu_env *env, switch (lcfg->lcfg_command) { case LCFG_CLEANUP: - if (!is_osp_on_ost(d->opd_obd->obd_name)) + if (!is_osp_for_connection(d->opd_obd->obd_name)) lu_dev_del_linkage(dev->ld_site, dev); rc = osp_shutdown(env, d); break; @@ -305,7 +309,8 @@ static int osp_recovery_complete(const struct lu_env *env, ENTRY; osp->opd_recovery_completed = 1; - cfs_waitq_signal(&osp->opd_pre_waitq); + if (!osp->opd_connect_mdt) + cfs_waitq_signal(&osp->opd_pre_waitq); RETURN(rc); } @@ -409,11 +414,11 @@ out: static int osp_init0(const struct lu_env *env, struct osp_device *m, struct lu_device_type *ldt, struct lustre_cfg *cfg) { - struct obd_device *obd; - struct obd_import *imp; - class_uuid_t uuid; - char *src, *ost, *mdt, *osdname = NULL; - int rc, idx; + struct obd_device *obd; + struct obd_import *imp; + class_uuid_t uuid; + char *src, *tgt, *mdt, *osdname = NULL; + int rc, idx; ENTRY; @@ -439,27 +444,75 @@ static int osp_init0(const struct lu_env *env, struct osp_device *m, if (src == NULL) RETURN(-EINVAL); - ost = strstr(src, "-OST"); - if (ost == NULL) + tgt = strrchr(src, '-'); + if (tgt == NULL) { + CERROR("%s: invalid target name %s\n", + m->opd_obd->obd_name, lustre_cfg_string(cfg, 0)); RETURN(-EINVAL); - - idx = simple_strtol(ost + 4, &mdt, 16); - if (mdt[0] != '-' || idx > INT_MAX || idx < 0) { - CERROR("%s: invalid OST index in '%s'\n", obd->obd_name, src); - GOTO(out_fini, rc = -EINVAL); } - m->opd_index = idx; - idx = ost - src; + if (strncmp(tgt, "-osc", 4) == 0) { + /* Old OSC name fsname-OSTXXXX-osc */ + for (tgt--; tgt > src && *tgt != '-'; tgt--) + ; + if (tgt == src) { + CERROR("%s: invalid target name %s\n", + m->opd_obd->obd_name, lustre_cfg_string(cfg, 0)); + RETURN(-EINVAL); + } + + if (strncmp(tgt, "-OST", 4) != 0) { + CERROR("%s: invalid target name %s\n", + m->opd_obd->obd_name, lustre_cfg_string(cfg, 0)); + RETURN(-EINVAL); + } + + idx = simple_strtol(tgt + 4, &mdt, 16); + if (mdt[0] != '-' || idx > INT_MAX || idx < 0) { + CERROR("%s: invalid OST index in '%s'\n", + m->opd_obd->obd_name, src); + RETURN(-EINVAL); + } + m->opd_index = idx; + idx = tgt - src; + } else { + /* New OSC name fsname-OSTXXXX-osc-MDTXXXX */ + if (strncmp(tgt, "-MDT", 4) != 0 && + strncmp(tgt, "-OST", 4) != 0) { + CERROR("%s: invalid target name %s\n", + m->opd_obd->obd_name, lustre_cfg_string(cfg, 0)); + RETURN(-EINVAL); + } + + if (tgt - src <= 12) { + CERROR("%s: invalid target name %s\n", + m->opd_obd->obd_name, lustre_cfg_string(cfg, 0)); + RETURN(-EINVAL); + } + + if (strncmp(tgt - 12, "-MDT", 4) == 0) + m->opd_connect_mdt = 1; + + idx = simple_strtol(tgt - 8, &mdt, 16); + if (mdt[0] != '-' || idx > INT_MAX || idx < 0) { + CERROR("%s: invalid OST index in '%s'\n", + m->opd_obd->obd_name, src); + RETURN(-EINVAL); + } + + m->opd_index = idx; + idx = tgt - src - 12; + } /* check the fsname length, and after this everything else will fit */ if (idx > MTI_NAME_MAXLEN) { - CERROR("%s: fsname too long in '%s'\n", obd->obd_name, src); - GOTO(out_fini, rc = -EINVAL); + CERROR("%s: fsname too long in '%s'\n", + m->opd_obd->obd_name, src); + RETURN(-EINVAL); } OBD_ALLOC(osdname, MAX_OBD_NAME); if (osdname == NULL) - GOTO(out_fini, rc = -ENOMEM); + RETURN(-ENOMEM); memcpy(osdname, src, idx); /* copy just the fsname part */ osdname[idx] = '\0'; @@ -472,6 +525,15 @@ static int osp_init0(const struct lu_env *env, struct osp_device *m, strcat(osdname, "-osd"); CDEBUG(D_HA, "%s: connect to %s (%s)\n", obd->obd_name, osdname, src); + if (m->opd_connect_mdt) { + struct client_obd *cli = &m->opd_obd->u.cli; + + OBD_ALLOC(cli->cl_rpc_lock, sizeof(*cli->cl_rpc_lock)); + if (!cli->cl_rpc_lock) + RETURN(-ENOMEM); + osp_init_rpc_lock(cli->cl_rpc_lock); + } + m->opd_dt_dev.dd_lu_dev.ld_ops = &osp_lu_ops; m->opd_dt_dev.dd_ops = &osp_dt_ops; obd->obd_lu_dev = &m->opd_dt_dev.dd_lu_dev; @@ -492,35 +554,33 @@ static int osp_init0(const struct lu_env *env, struct osp_device *m, osp_lprocfs_init(m); - /* - * Initialize last id from the storage - will be used in orphan cleanup - */ - rc = osp_last_used_init(env, m); - if (rc) - GOTO(out_proc, rc); - - /* - * Initialize precreation thread, it handles new connections as well - */ - rc = osp_init_precreate(m); - if (rc) - GOTO(out_last_used, rc); - - /* - * Initialize synhronization mechanism taking care of propogating - * changes to OST in near transactional manner - */ - rc = osp_sync_init(env, m); - if (rc) - GOTO(out_precreat, rc); + if (!m->opd_connect_mdt) { + /* Initialize last id from the storage - will be + * used in orphan cleanup. */ + rc = osp_last_used_init(env, m); + if (rc) + GOTO(out_proc, rc); + /* Initialize precreation thread, it handles new + * connections as well. */ + rc = osp_init_precreate(m); + if (rc) + GOTO(out_last_used, rc); + /* + * Initialize synhronization mechanism taking + * care of propogating changes to OST in near + * transactional manner. + */ + rc = osp_sync_init(env, m); + if (rc) + GOTO(out_precreat, rc); - rc = obd_fid_init(m->opd_obd, NULL, LUSTRE_SEQ_DATA); - if (rc) { - CERROR("%s: fid init error: rc = %d\n", - m->opd_obd->obd_name, rc); - GOTO(out, rc); + rc = obd_fid_init(m->opd_obd, NULL, LUSTRE_SEQ_DATA); + if (rc) { + CERROR("%s: fid init error: rc = %d\n", + m->opd_obd->obd_name, rc); + GOTO(out, rc); + } } - /* * Initiate connect to OST */ @@ -537,11 +597,13 @@ static int osp_init0(const struct lu_env *env, struct osp_device *m, RETURN(0); out: - /* stop sync thread */ - osp_sync_fini(m); + if (!m->opd_connect_mdt) + /* stop sync thread */ + osp_sync_fini(m); out_precreat: /* stop precreate thread */ - osp_precreate_fini(m); + if (!m->opd_connect_mdt) + osp_precreate_fini(m); out_last_used: osp_last_used_fini(env, m); out_proc: @@ -552,6 +614,13 @@ out_proc: out_ref: ptlrpcd_decref(); out_disconnect: + if (m->opd_connect_mdt) { + struct client_obd *cli = &m->opd_obd->u.cli; + if (cli->cl_rpc_lock != NULL) { + OBD_FREE_PTR(cli->cl_rpc_lock); + cli->cl_rpc_lock = NULL; + } + } obd_disconnect(m->opd_storage_exp); out_fini: if (osdname) @@ -590,7 +659,7 @@ static struct lu_device *osp_device_alloc(const struct lu_env *env, l = osp2lu_dev(m); dt_device_init(&m->opd_dt_dev, t); - if (is_osp_on_ost(lustre_cfg_string(lcfg, 0))) + if (is_osp_for_connection(lustre_cfg_string(lcfg, 0))) rc = osp_init_for_ost(env, m, t, lcfg); else rc = osp_init0(env, m, t, lcfg); @@ -614,7 +683,7 @@ static struct lu_device *osp_device_fini(const struct lu_env *env, if (m->opd_storage_exp) obd_disconnect(m->opd_storage_exp); - if (is_osp_on_ost(m->opd_obd->obd_name)) + if (is_osp_for_connection(m->opd_obd->obd_name)) osp_fini_for_ost(m); imp = m->opd_obd->u.cli.cl_import; @@ -633,6 +702,14 @@ static struct lu_device *osp_device_fini(const struct lu_env *env, ptlrpc_lprocfs_unregister_obd(m->opd_obd); lprocfs_obd_cleanup(m->opd_obd); + if (m->opd_connect_mdt) { + struct client_obd *cli = &m->opd_obd->u.cli; + if (cli->cl_rpc_lock != NULL) { + OBD_FREE_PTR(cli->cl_rpc_lock); + cli->cl_rpc_lock = NULL; + } + } + rc = client_obd_cleanup(m->opd_obd); LASSERTF(rc == 0, "error %d\n", rc); @@ -678,29 +755,21 @@ static int osp_obd_connect(const struct lu_env *env, struct obd_export **exp, osp->opd_connects++; LASSERT(osp->opd_connects == 1); + osp->opd_exp = *exp; + imp = osp->opd_obd->u.cli.cl_import; imp->imp_dlm_handle = conn; + LASSERT(data != NULL); + LASSERT(data->ocd_connect_flags & OBD_CONNECT_INDEX); ocd = &imp->imp_connect_data; - ocd->ocd_connect_flags = OBD_CONNECT_AT | - OBD_CONNECT_FULL20 | - OBD_CONNECT_INDEX | -#ifdef HAVE_LRU_RESIZE_SUPPORT - OBD_CONNECT_LRU_RESIZE | -#endif - OBD_CONNECT_MDS | - OBD_CONNECT_OSS_CAPA | - OBD_CONNECT_REQPORTAL | - OBD_CONNECT_SKIP_ORPHAN | - OBD_CONNECT_VERSION | - OBD_CONNECT_FID | - OBD_CONNECT_LVB_TYPE; - - if (is_osp_on_ost(osp->opd_obd->obd_name)) + *ocd = *data; + if (is_osp_for_connection(osp->opd_obd->obd_name)) ocd->ocd_connect_flags |= OBD_CONNECT_LIGHTWEIGHT; + imp->imp_connect_flags_orig = ocd->ocd_connect_flags; + ocd->ocd_version = LUSTRE_VERSION_CODE; - LASSERT(data->ocd_connect_flags & OBD_CONNECT_INDEX); ocd->ocd_index = data->ocd_index; imp->imp_connect_flags_orig = ocd->ocd_connect_flags; @@ -712,6 +781,16 @@ static int osp_obd_connect(const struct lu_env *env, struct obd_export **exp, ptlrpc_pinger_add_import(imp); + /* set seq controller export for MDC0 if exists */ + if (osp->opd_connect_mdt && !is_osp_for_connection(obd->obd_name) && + data->ocd_index == 0) { + struct seq_server_site *ss; + + ss = lu_site2seq(osp2lu_dev(osp)->ld_site); + ss->ss_control_exp = class_export_get(*exp); + ss->ss_server_fld->lsf_control_exp = *exp; + } + out: RETURN(rc); } @@ -739,7 +818,7 @@ static int osp_obd_disconnect(struct obd_export *exp) } /* destroy the device */ - if (!is_osp_on_ost(obd->obd_name)) + if (!is_osp_for_connection(obd->obd_name)) class_manual_cleanup(obd); RETURN(rc); @@ -820,7 +899,7 @@ static int osp_import_event(struct obd_device *obd, struct obd_import *imp, case IMP_EVENT_DISCON: d->opd_got_disconnected = 1; d->opd_imp_connected = 0; - if (is_osp_on_ost(d->opd_obd->obd_name)) + if (d->opd_connect_mdt) break; osp_pre_update_status(d, -ENODEV); cfs_waitq_signal(&d->opd_pre_waitq); @@ -828,7 +907,7 @@ static int osp_import_event(struct obd_device *obd, struct obd_import *imp, break; case IMP_EVENT_INACTIVE: d->opd_imp_active = 0; - if (is_osp_on_ost(d->opd_obd->obd_name)) + if (d->opd_connect_mdt) break; osp_pre_update_status(d, -ENODEV); cfs_waitq_signal(&d->opd_pre_waitq); @@ -840,7 +919,7 @@ static int osp_import_event(struct obd_device *obd, struct obd_import *imp, d->opd_new_connection = 1; d->opd_imp_connected = 1; d->opd_imp_seen_connected = 1; - if (is_osp_on_ost(d->opd_obd->obd_name)) + if (d->opd_connect_mdt) break; if (d->opd_obd->u.cli.cl_seq->lcs_exp == NULL) d->opd_obd->u.cli.cl_seq->lcs_exp = diff --git a/lustre/osp/osp_internal.h b/lustre/osp/osp_internal.h index 1815a0c..f9c9c0c 100644 --- a/lustre/osp/osp_internal.h +++ b/lustre/osp/osp_internal.h @@ -93,7 +93,8 @@ struct osp_device { int opd_got_disconnected; int opd_imp_connected; int opd_imp_active; - int opd_imp_seen_connected:1; + int opd_imp_seen_connected:1, + opd_connect_mdt:1; /* whether local recovery is completed: * reported via ->ldo_recovery_complete() */ @@ -202,6 +203,7 @@ struct osp_thread_info { }; struct llog_cookie osi_cookie; struct llog_catid osi_cid; + struct lu_seq_range osi_seq; }; static inline void osp_objid_buf_prep(struct osp_thread_info *osi, @@ -293,6 +295,10 @@ static inline struct dt_object *osp_object_child(struct osp_object *o) struct dt_object, do_lu); } +#define osp_init_rpc_lock(lck) mdc_init_rpc_lock(lck) +#define osp_get_rpc_lock(lck, it) mdc_get_rpc_lock(lck, it) +#define osp_put_rpc_lock(lck, it) mdc_put_rpc_lock(lck, it) + /* osp_dev.c */ void osp_update_last_id(struct osp_device *d, obd_id objid); extern struct llog_operations osp_mds_ost_orig_logops; diff --git a/lustre/osp/osp_ost.c b/lustre/osp/osp_ost.c index 60f5b33..a177e12 100644 --- a/lustre/osp/osp_ost.c +++ b/lustre/osp/osp_ost.c @@ -47,7 +47,7 @@ static int osp_name2fsname(char *ospname, char *fsname) LASSERT(ospname != NULL); LASSERT(fsname != NULL); - if (!is_osp_on_ost(ospname)) + if (!is_osp_for_connection(ospname)) return -EINVAL; sprintf(fsname, "-%s-", LUSTRE_OSP_NAME); @@ -106,6 +106,7 @@ static int osp_setup_for_ost(const struct lu_env *env, struct osp_device *osp, if (lcfg == NULL) GOTO(out, rc = -ENOMEM); + osp->opd_connect_mdt = 1; rc = client_obd_setup(osp->opd_obd, lcfg); if (rc != 0) { CERROR("%s: client obd setup error: rc = %d\n", -- 1.8.3.1