X-Git-Url: https://git.whamcloud.com/?p=fs%2Flustre-release.git;a=blobdiff_plain;f=lustre%2Flod%2Flod_dev.c;h=4a672dc43e83b6c4117258a26dfe2cc5de932212;hp=608f754abc247f410de669b538a653e151c8de7a;hb=303ea892488b985ba84e6c8e1481f6c7c914c7ed;hpb=74ec68346e14851ad8a1912185e1dccd3e6d12cd diff --git a/lustre/lod/lod_dev.c b/lustre/lod/lod_dev.c index 608f754..4a672dc 100644 --- a/lustre/lod/lod_dev.c +++ b/lustre/lod/lod_dev.c @@ -44,7 +44,9 @@ #define DEBUG_SUBSYSTEM S_MDS #include +#include #include +#include #include "lod_internal.h" @@ -94,6 +96,7 @@ int lod_fld_lookup(const struct lu_env *env, struct lod_device *lod, } extern struct lu_object_operations lod_lu_obj_ops; +extern struct lu_object_operations lod_lu_robj_ops; extern struct dt_object_operations lod_obj_ops; /* Slab for OSD object allocation */ @@ -117,19 +120,32 @@ struct lu_object *lod_object_alloc(const struct lu_env *env, const struct lu_object_header *hdr, struct lu_device *dev) { - struct lu_object *lu_obj; - struct lod_object *lo; + struct lod_object *lod_obj; + struct lu_object *lu_obj; + const struct lu_fid *fid = &hdr->loh_fid; + mdsno_t mds; + int rc = 0; + ENTRY; - OBD_SLAB_ALLOC_PTR_GFP(lo, lod_object_kmem, CFS_ALLOC_IO); - if (lo == NULL) - return NULL; + OBD_SLAB_ALLOC_PTR_GFP(lod_obj, lod_object_kmem, CFS_ALLOC_IO); + if (lod_obj == NULL) + RETURN(ERR_PTR(-ENOMEM)); - lu_obj = lod2lu_obj(lo); - dt_object_init(&lo->ldo_obj, NULL, dev); - lo->ldo_obj.do_ops = &lod_obj_ops; - lu_obj->lo_ops = &lod_lu_obj_ops; + rc = lod_fld_lookup(env, lu2lod_dev(dev), fid, &mds, LU_SEQ_RANGE_MDT); + if (rc) { + OBD_SLAB_FREE_PTR(lod_obj, lod_object_kmem); + RETURN(ERR_PTR(rc)); + } - return lu_obj; + lod_obj->ldo_mds_num = mds; + lu_obj = lod2lu_obj(lod_obj); + dt_object_init(&lod_obj->ldo_obj, NULL, dev); + lod_obj->ldo_obj.do_ops = &lod_obj_ops; + if (likely(mds == lu_site2seq(dev->ld_site)->ss_node_id)) + lu_obj->lo_ops = &lod_lu_obj_ops; + else + lu_obj->lo_ops = &lod_lu_robj_ops; + RETURN(lu_obj); } static int lod_cleanup_desc_tgts(const struct lu_env *env, @@ -199,6 +215,120 @@ static int lodname2mdt_index(char *lodname, int *index) return 0; } +/* + * Init client sequence manager which is used by local MDS to talk to sequence + * controller on remote node. + */ +static int lod_seq_init_cli(const struct lu_env *env, + struct lod_device *lod, + char *tgtuuid, int index) +{ + struct seq_server_site *ss; + struct obd_device *osp; + int rc; + char *prefix; + struct obd_uuid obd_uuid; + ENTRY; + + ss = lu_site2seq(lod2lu_dev(lod)->ld_site); + LASSERT(ss != NULL); + + /* check if this is adding the first MDC and controller is not yet + * initialized. */ + if (index != 0 || ss->ss_client_seq) + RETURN(0); + + obd_str2uuid(&obd_uuid, tgtuuid); + osp = class_find_client_obd(&obd_uuid, LUSTRE_OSP_NAME, + &lod->lod_dt_dev.dd_lu_dev.ld_obd->obd_uuid); + if (osp == NULL) { + CERROR("%s: can't find %s device\n", + lod->lod_dt_dev.dd_lu_dev.ld_obd->obd_name, + tgtuuid); + RETURN(-EINVAL); + } + + if (!osp->obd_set_up) { + CERROR("target %s not set up\n", osp->obd_name); + rc = -EINVAL; + } + + LASSERT(ss->ss_control_exp); + OBD_ALLOC_PTR(ss->ss_client_seq); + if (ss->ss_client_seq == NULL) + RETURN(-ENOMEM); + + OBD_ALLOC(prefix, MAX_OBD_NAME + 5); + if (!prefix) { + OBD_FREE_PTR(ss->ss_client_seq); + ss->ss_client_seq = NULL; + RETURN(-ENOMEM); + } + + snprintf(prefix, MAX_OBD_NAME + 5, "ctl-%s", osp->obd_name); + rc = seq_client_init(ss->ss_client_seq, ss->ss_control_exp, + LUSTRE_SEQ_METADATA, prefix, NULL); + OBD_FREE(prefix, MAX_OBD_NAME + 5); + if (rc) { + OBD_FREE_PTR(ss->ss_client_seq); + ss->ss_client_seq = NULL; + RETURN(rc); + } + + LASSERT(ss->ss_server_seq != NULL); + rc = seq_server_set_cli(ss->ss_server_seq, ss->ss_client_seq, + env); + + RETURN(rc); +} + +static void lod_seq_fini_cli(struct lod_device *lod) +{ + struct seq_server_site *ss; + + ENTRY; + + ss = lu_site2seq(lod2lu_dev(lod)->ld_site); + if (ss == NULL) { + EXIT; + return; + } + + if (ss->ss_server_seq) + seq_server_set_cli(ss->ss_server_seq, + NULL, NULL); + + if (ss->ss_control_exp) { + class_export_put(ss->ss_control_exp); + ss->ss_control_exp = NULL; + } + + EXIT; + return; +} + +/** + * Procss config log on LOD + * \param env environment info + * \param dev lod device + * \param lcfg config log + * + * Add osc config log, + * marker 20 (flags=0x01, v2.2.49.56) lustre-OST0001 'add osc' + * add_uuid nid=192.168.122.162@tcp(0x20000c0a87aa2) 0: 1:nidxxx + * attach 0:lustre-OST0001-osc-MDT0001 1:osc 2:lustre-MDT0001-mdtlov_UUID + * setup 0:lustre-OST0001-osc-MDT0001 1:lustre-OST0001_UUID 2:nid + * lov_modify_tgts add 0:lustre-MDT0001-mdtlov 1:lustre-OST0001_UUID 2:1 3:1 + * marker 20 (flags=0x02, v2.2.49.56) lustre-OST0001 'add osc' + * + * Add mdc config log + * marker 10 (flags=0x01, v2.2.49.56) lustre-MDT0000 'add osp' + * add_uuid nid=192.168.122.162@tcp(0x20000c0a87aa2) 0: 1:nid + * attach 0:lustre-MDT0000-osp-MDT0001 1:osp 2:lustre-MDT0001-mdtlov_UUID + * setup 0:lustre-MDT0000-osp-MDT0001 1:lustre-MDT0000_UUID 2:nid + * modify_mdc_tgts add 0:lustre-MDT0001 1:lustre-MDT0000_UUID 2:0 3:1 + * marker 10 (flags=0x02, v2.2.49.56) lustre-MDT0000_UUID 'add osp' + **/ static int lod_process_config(const struct lu_env *env, struct lu_device *dev, struct lustre_cfg *lcfg) @@ -246,6 +376,9 @@ static int lod_process_config(const struct lu_env *env, mdt_index = index; rc = lod_add_device(env, lod, arg1, index, gen, mdt_index, LUSTRE_MDC_NAME, 1); + if (rc == 0) + rc = lod_seq_init_cli(env, lod, arg1, + mdt_index); } else if (lcfg->lcfg_command == LCFG_LOV_ADD_INA) { /*FIXME: Add mdt_index for LCFG_LOV_ADD_INA*/ mdt_index = 0; @@ -275,6 +408,8 @@ static int lod_process_config(const struct lu_env *env, lu_dev_del_linkage(dev->ld_site, dev); lod_cleanup_desc_tgts(env, lod, &lod->lod_mdt_descs, lcfg); lod_cleanup_desc_tgts(env, lod, &lod->lod_ost_descs, lcfg); + + lod_seq_fini_cli(lod); /* * do cleanup on underlying storage only when * all OSPs are cleaned up, as they use that OSD as well @@ -373,19 +508,77 @@ static int lod_statfs(const struct lu_env *env, static struct thandle *lod_trans_create(const struct lu_env *env, struct dt_device *dev) { - return dt_trans_create(env, dt2lod_dev(dev)->lod_child); + struct thandle *th; + + th = dt_trans_create(env, dt2lod_dev(dev)->lod_child); + if (IS_ERR(th)) + return th; + + CFS_INIT_LIST_HEAD(&th->th_remote_update_list); + return th; +} + +static int lod_remote_sync(const struct lu_env *env, struct dt_device *dev, + struct thandle *th) +{ + struct update_request *update; + int rc = 0; + ENTRY; + + if (cfs_list_empty(&th->th_remote_update_list)) + RETURN(0); + + cfs_list_for_each_entry(update, &th->th_remote_update_list, + ur_list) { + /* In DNE phase I, there should be only one OSP + * here, so we will do send/receive one by one, + * instead of sending them parallel, will fix this + * in Phase II */ + th->th_current_request = update; + rc = dt_trans_start(env, update->ur_dt, th); + if (rc != 0) { + /* FIXME how to revert the partial results + * once error happened? Resolved by 2 Phase commit */ + update->ur_rc = rc; + break; + } + } + + RETURN(rc); } static int lod_trans_start(const struct lu_env *env, struct dt_device *dev, struct thandle *th) { - return dt_trans_start(env, dt2lod_dev(dev)->lod_child, th); + struct lod_device *lod = dt2lod_dev((struct dt_device *) dev); + int rc; + + rc = lod_remote_sync(env, dev, th); + if (rc) + return rc; + + return dt_trans_start(env, lod->lod_child, th); } static int lod_trans_stop(const struct lu_env *env, struct thandle *th) { - /* XXX: we don't know next device, will be fixed with DNE */ - return dt_trans_stop(env, th->th_dev, th); + struct update_request *update; + struct update_request *tmp; + int rc = 0; + int rc2 = 0; + + cfs_list_for_each_entry_safe(update, tmp, + &th->th_remote_update_list, + ur_list) { + th->th_current_request = update; + rc2 = dt_trans_stop(env, update->ur_dt, th); + if (unlikely(rc2 != 0 && rc == 0)) + rc = rc2; + } + + rc2 = dt_trans_stop(env, th->th_dev, th); + + return rc2 != 0 ? rc2 : rc; } static void lod_conf_get(const struct lu_env *env,