From df70a5c8938888686188cfc63748011f8c7557e0 Mon Sep 17 00:00:00 2001 From: Alex Zhuravlev Date: Thu, 20 Sep 2012 15:53:25 +0400 Subject: [PATCH] LU-1303 lod: object create to create underlying objects and generate LOV EA for just created striping. the policies choosing specific OSPs to use in the striping is in the subsequent patch. Signed-off-by: Alex Zhuravlev Change-Id: I2675dd3e2ba4455b2b0daaf4aa7af30b910e9a75 Reviewed-on: http://review.whamcloud.com/4057 Reviewed-by: wangdi Tested-by: Hudson Tested-by: Maloo Reviewed-by: Mike Pershin Reviewed-by: Oleg Drokin --- lustre/include/dt_object.h | 1 + lustre/include/lustre/lustre_idl.h | 7 +++ lustre/lod/lod_internal.h | 9 +++ lustre/lod/lod_lov.c | 67 +++++++++++++++++++++ lustre/lod/lod_object.c | 115 ++++++++++++++++++++++++++++++++++++- 5 files changed, 198 insertions(+), 1 deletion(-) diff --git a/lustre/include/dt_object.h b/lustre/include/dt_object.h index 9aa277e..3c342a8 100644 --- a/lustre/include/dt_object.h +++ b/lustre/include/dt_object.h @@ -262,6 +262,7 @@ struct dt_object_format { enum dt_format_type dof_type; union { struct dof_regular { + int striped; } dof_reg; struct dof_dir { } dof_dir; diff --git a/lustre/include/lustre/lustre_idl.h b/lustre/include/lustre/lustre_idl.h index 5426309..3bb6f37 100644 --- a/lustre/include/lustre/lustre_idl.h +++ b/lustre/include/lustre/lustre_idl.h @@ -534,6 +534,13 @@ static inline obd_id fid_idif_id(obd_seq seq, __u32 oid, __u32 ver) return ((__u64)ver << 48) | ((seq & 0xffff) << 32) | oid; } +/* extract ost index from IDIF FID */ +static inline __u32 fid_idif_ost_idx(const struct lu_fid *fid) +{ + LASSERT(fid_is_idif(fid)); + return (fid_seq(fid) >> 16) & 0xffff; +} + /* unpack an ostid (id/seq) from a wire/disk structure into an IDIF FID */ static inline void ostid_idif_unpack(struct ost_id *ostid, struct lu_fid *fid, __u32 ost_idx) diff --git a/lustre/lod/lod_internal.h b/lustre/lod/lod_internal.h index 8a76732..03614bf 100644 --- a/lustre/lod/lod_internal.h +++ b/lustre/lod/lod_internal.h @@ -262,6 +262,8 @@ int lod_initialize_objects(const struct lu_env *env, struct lod_object *mo, int lod_store_def_striping(const struct lu_env *env, struct dt_object *dt, struct thandle *th); int lod_verify_striping(struct lod_device *d, const struct lu_buf *buf, int specific); +int lod_generate_and_set_lovea(const struct lu_env *env, + struct lod_object *mo, struct thandle *th); /* lod_pool.c */ int lod_ost_pool_add(struct ost_pool *op, __u32 idx, unsigned int min_count); @@ -288,6 +290,13 @@ void lprocfs_lod_init_vars(struct lprocfs_static_vars *lvars); /* lod_object.c */ int lod_object_set_pool(struct lod_object *o, char *pool); +int lod_declare_striped_object(const struct lu_env *env, struct dt_object *dt, + struct lu_attr *attr, + const struct lu_buf *lovea, struct thandle *th); +int lod_striping_create(const struct lu_env *env, struct dt_object *dt, + struct lu_attr *attr, struct dt_object_format *dof, + struct thandle *th); +void lod_object_free_striping(const struct lu_env *env, struct lod_object *lo); #endif diff --git a/lustre/lod/lod_lov.c b/lustre/lod/lod_lov.c index 821012b..d823651 100644 --- a/lustre/lod/lod_lov.c +++ b/lustre/lod/lod_lov.c @@ -403,6 +403,73 @@ int lod_ea_store_resize(struct lod_thread_info *info, int size) RETURN(0); } +/* + * generate and write LOV EA for given striped object + */ +int lod_generate_and_set_lovea(const struct lu_env *env, + struct lod_object *lo, struct thandle *th) +{ + struct lod_thread_info *info = lod_env_info(env); + struct dt_object *next = dt_object_child(&lo->ldo_obj); + const struct lu_fid *fid = lu_object_fid(&lo->ldo_obj.do_lu); + struct lov_mds_md_v1 *lmm; + struct lov_ost_data_v1 *objs; + __u32 magic; + int i, rc, lmm_size; + ENTRY; + + LASSERT(lo); + LASSERT(lo->ldo_stripenr > 0); + + magic = lo->ldo_pool ? LOV_MAGIC_V3 : LOV_MAGIC_V1; + lmm_size = lov_mds_md_size(lo->ldo_stripenr, magic); + if (info->lti_ea_store_size < lmm_size) { + rc = lod_ea_store_resize(info, lmm_size); + if (rc) + RETURN(rc); + } + + lmm = info->lti_ea_store; + + lmm->lmm_magic = cpu_to_le32(magic); + lmm->lmm_pattern = cpu_to_le32(LOV_PATTERN_RAID0); + lmm->lmm_object_id = cpu_to_le64(fid_ver_oid(fid)); + lmm->lmm_object_seq = cpu_to_le64(fid_seq(fid)); + lmm->lmm_stripe_size = cpu_to_le32(lo->ldo_stripe_size); + lmm->lmm_stripe_count = cpu_to_le16(lo->ldo_stripenr); + lmm->lmm_layout_gen = 0; + if (magic == LOV_MAGIC_V1) { + objs = &lmm->lmm_objects[0]; + } else { + struct lov_mds_md_v3 *v3 = (struct lov_mds_md_v3 *) lmm; + strncpy(v3->lmm_pool_name, lo->ldo_pool, LOV_MAXPOOLNAME); + objs = &v3->lmm_objects[0]; + } + + for (i = 0; i < lo->ldo_stripenr; i++) { + const struct lu_fid *fid; + + LASSERT(lo->ldo_stripe[i]); + fid = lu_object_fid(&lo->ldo_stripe[i]->do_lu); + + rc = fid_ostid_pack(fid, &info->lti_ostid); + LASSERT(rc == 0); + LASSERT(info->lti_ostid.oi_seq == FID_SEQ_OST_MDT0); + + objs[i].l_object_id = cpu_to_le64(info->lti_ostid.oi_id); + objs[i].l_object_seq = cpu_to_le64(info->lti_ostid.oi_seq); + objs[i].l_ost_gen = cpu_to_le32(1); + objs[i].l_ost_idx = cpu_to_le32(fid_idif_ost_idx(fid)); + } + + info->lti_buf.lb_buf = lmm; + info->lti_buf.lb_len = lmm_size; + rc = dt_xattr_set(env, next, &info->lti_buf, XATTR_NAME_LOV, 0, + th, BYPASS_CAPA); + + RETURN(rc); +} + int lod_get_lov_ea(const struct lu_env *env, struct lod_object *lo) { struct lod_thread_info *info = lod_env_info(env); diff --git a/lustre/lod/lod_object.c b/lustre/lod/lod_object.c index 30760cd..816e6cd 100644 --- a/lustre/lod/lod_object.c +++ b/lustre/lod/lod_object.c @@ -292,9 +292,35 @@ static int lod_declare_xattr_set(const struct lu_env *env, struct thandle *th) { struct dt_object *next = dt_object_child(dt); + struct lu_attr *attr = &lod_env_info(env)->lti_attr; + __u32 mode; int rc; ENTRY; + /* + * allow to declare predefined striping on a new (!mode) object + * which is supposed to be replay of regular file creation + * (when LOV setting is declared) + */ + mode = dt->do_lu.lo_header->loh_attr & S_IFMT; + if ((S_ISREG(mode) || !mode) && !strcmp(name, XATTR_NAME_LOV)) { + /* + * this is a request to manipulate object's striping + */ + if (dt_object_exists(dt)) { + rc = dt_attr_get(env, next, attr, BYPASS_CAPA); + if (rc) + RETURN(rc); + } else { + memset(attr, 0, sizeof(attr)); + attr->la_valid = LA_TYPE | LA_MODE; + attr->la_mode = S_IFREG; + } + rc = lod_declare_striped_object(env, dt, attr, buf, th); + if (rc) + RETURN(rc); + } + rc = dt_declare_xattr_set(env, next, buf, name, fl, th); RETURN(rc); @@ -375,6 +401,14 @@ static int lod_xattr_set(const struct lu_env *env, else rc = dt_xattr_set(env, next, buf, name, fl, th, capa); + } else if (S_ISREG(attr) && !strcmp(name, XATTR_NAME_LOV)) { + /* + * XXX: check striping match what we already have + * during req replay, declare_xattr_set() defines striping, + * then create() does the work + */ + rc = lod_striping_create(env, dt, NULL, NULL, th); + RETURN(rc); } else { /* * behave transparantly for all other EAs @@ -600,6 +634,43 @@ static void lod_ah_init(const struct lu_env *env, EXIT; } +/** + * Create declaration of striped object + */ +int lod_declare_striped_object(const struct lu_env *env, struct dt_object *dt, + struct lu_attr *attr, + const struct lu_buf *lovea, struct thandle *th) +{ + struct lod_thread_info *info = lod_env_info(env); + struct dt_object *next = dt_object_child(dt); + struct lod_object *lo = lod_dt_obj(dt); + int rc; + ENTRY; + + if (OBD_FAIL_CHECK(OBD_FAIL_MDS_ALLOC_OBDO)) { + /* failed to create striping, let's reset + * config so that others don't get confused */ + lod_object_free_striping(env, lo); + GOTO(out, rc = -ENOMEM); + } + + /* XXX: there will be a call to QoS here */ + RETURN(0); + + /* + * declare storage for striping data + */ + info->lti_buf.lb_len = lov_mds_md_size(lo->ldo_stripenr, + lo->ldo_pool ? LOV_MAGIC_V3 : LOV_MAGIC_V1); + rc = dt_declare_xattr_set(env, next, &info->lti_buf, XATTR_NAME_LOV, + 0, th); + if (rc) + GOTO(out, rc); + +out: + RETURN(rc); +} + static int lod_declare_object_create(const struct lu_env *env, struct dt_object *dt, struct lu_attr *attr, @@ -627,7 +698,20 @@ static int lod_declare_object_create(const struct lu_env *env, if (dof->dof_type == DFT_SYM) dt->do_body_ops = &lod_body_lnk_ops; - if (dof->dof_type == DFT_DIR && lo->ldo_striping_cached) { + /* + * it's lod_ah_init() who has decided the object will striped + */ + if (dof->dof_type == DFT_REGULAR) { + /* callers don't want stripes */ + /* XXX: all tricky interactions with ->ah_make_hint() decided + * to use striping, then ->declare_create() behaving differently + * should be cleaned */ + if (dof->u.dof_reg.striped == 0) + lo->ldo_stripenr = 0; + if (lo->ldo_stripenr > 0) + rc = lod_declare_striped_object(env, dt, attr, + NULL, th); + } else if (dof->dof_type == DFT_DIR && lo->ldo_striping_cached) { struct lod_thread_info *info = lod_env_info(env); info->lti_buf.lb_buf = NULL; @@ -641,12 +725,39 @@ out: RETURN(rc); } +int lod_striping_create(const struct lu_env *env, struct dt_object *dt, + struct lu_attr *attr, struct dt_object_format *dof, + struct thandle *th) +{ + struct lod_object *lo = lod_dt_obj(dt); + int rc = 0, i; + ENTRY; + + LASSERT(lo->ldo_stripe); + LASSERT(lo->ldo_stripe > 0); + LASSERT(lo->ldo_striping_cached == 0); + + /* create all underlying objects */ + for (i = 0; i < lo->ldo_stripenr; i++) { + LASSERT(lo->ldo_stripe[i]); + rc = dt_create(env, lo->ldo_stripe[i], attr, NULL, dof, th); + + if (rc) + break; + } + if (rc == 0) + rc = lod_generate_and_set_lovea(env, lo, th); + + RETURN(rc); +} + static int lod_object_create(const struct lu_env *env, struct dt_object *dt, struct lu_attr *attr, struct dt_allocation_hint *hint, struct dt_object_format *dof, struct thandle *th) { struct dt_object *next = dt_object_child(dt); + struct lod_object *lo = lod_dt_obj(dt); int rc; ENTRY; @@ -656,6 +767,8 @@ static int lod_object_create(const struct lu_env *env, struct dt_object *dt, if (rc == 0) { if (S_ISDIR(dt->do_lu.lo_header->loh_attr)) rc = lod_store_def_striping(env, dt, th); + else if (lo->ldo_stripe) + rc = lod_striping_create(env, dt, attr, dof, th); } RETURN(rc); -- 1.8.3.1