From 2bd62ea92ade0a3980cb090e651f4c8baab59597 Mon Sep 17 00:00:00 2001 From: Jinshan Xiong Date: Tue, 26 Sep 2017 00:28:57 +0000 Subject: [PATCH] LU-9771 lod: define ldo_mirrors to manage mirrors Right now it only supports loading mirror from disk Test-Parameters: testlist=sanity-flr Signed-off-by: Jinshan Xiong Signed-off-by: Bobi Jam Change-Id: Id560070e6147465681ede5a02e1add585281ea89 Reviewed-on: https://review.whamcloud.com/29089 Tested-by: Jenkins Reviewed-by: Lai Siyao Reviewed-by: Bobi Jam Tested-by: Maloo --- lustre/lod/lod_internal.h | 22 ++++++-- lustre/lod/lod_lov.c | 140 ++++++++++++++++++++++++++-------------------- lustre/lod/lod_object.c | 110 +++++++++++++++++++++++++++++++----- lustre/lod/lod_qos.c | 15 ++++- 4 files changed, 206 insertions(+), 81 deletions(-) diff --git a/lustre/lod/lod_internal.h b/lustre/lod/lod_internal.h index fe7d21b..b475827 100644 --- a/lustre/lod/lod_internal.h +++ b/lustre/lod/lod_internal.h @@ -250,6 +250,7 @@ struct lod_default_striping { /* default LOV */ /* current layout component count */ __u16 lds_def_comp_cnt; + __u16 lds_def_mirror_cnt; /* the largest comp count ever used */ __u32 lds_def_comp_size_cnt; struct lod_layout_component *lds_def_comp_entries; @@ -264,6 +265,15 @@ struct lod_default_striping { lds_dir_def_striping_set:1; }; +struct lod_mirror_entry { + __u16 lme_stale:1; + /* mirror id */ + __u16 lme_id; + /* start,end index of this mirror in ldo_comp_entries */ + __u16 lme_start; + __u16 lme_end; +}; + struct lod_object { /* common fields for both files and directories */ struct dt_object ldo_obj; @@ -274,7 +284,10 @@ struct lod_object { /* Layout component count for a regular file. * It equals to 1 for non-composite layout. */ __u16 ldo_comp_cnt; + /* Layout mirror count for a PFLR file. + * It's 0 for files with non-composite layout. */ __u16 ldo_mirror_count; + struct lod_mirror_entry *ldo_mirrors; __u32 ldo_is_composite:1, ldo_flr_state:2, ldo_comp_cached:1; @@ -331,15 +344,13 @@ static inline int lod_set_pool(char **pool, const char *new_pool) static inline int lod_set_def_pool(struct lod_default_striping *lds, int i, const char *new_pool) { - return lod_set_pool(&lds->lds_def_comp_entries[i].llc_pool, - new_pool); + return lod_set_pool(&lds->lds_def_comp_entries[i].llc_pool, new_pool); } static inline int lod_obj_set_pool(struct lod_object *lo, int i, const char *new_pool) { - return lod_set_pool(&lo->ldo_comp_entries[i].llc_pool, - new_pool); + return lod_set_pool(&lo->ldo_comp_entries[i].llc_pool, new_pool); } /** @@ -608,7 +619,8 @@ int lod_ea_store_resize(struct lod_thread_info *info, size_t size); int lod_def_striping_comp_resize(struct lod_default_striping *lds, __u16 count); void lod_free_def_comp_entries(struct lod_default_striping *lds); void lod_free_comp_entries(struct lod_object *lo); -int lod_alloc_comp_entries(struct lod_object *lo, int cnt); +int lod_alloc_comp_entries(struct lod_object *lo, int mirror_cnt, int comp_cnt); +int lod_fill_mirrors(struct lod_object *lo); /* lod_pool.c */ int lod_ost_pool_add(struct ost_pool *op, __u32 idx, unsigned int min_count); diff --git a/lustre/lod/lod_lov.c b/lustre/lod/lod_lov.c index 1a3c9df..45f74c0 100644 --- a/lustre/lod/lod_lov.c +++ b/lustre/lod/lod_lov.c @@ -694,6 +694,12 @@ int lod_def_striping_comp_resize(struct lod_default_striping *lds, __u16 count) void lod_free_comp_entries(struct lod_object *lo) { + if (lo->ldo_mirrors) { + OBD_FREE(lo->ldo_mirrors, + sizeof(*lo->ldo_mirrors) * lo->ldo_mirror_count); + lo->ldo_mirrors = NULL; + lo->ldo_mirror_count = 0; + } lod_free_comp_buffer(lo->ldo_comp_entries, lo->ldo_comp_cnt, sizeof(*lo->ldo_comp_entries) * lo->ldo_comp_cnt); @@ -702,19 +708,75 @@ void lod_free_comp_entries(struct lod_object *lo) lo->ldo_is_composite = 0; } -int lod_alloc_comp_entries(struct lod_object *lo, int cnt) +int lod_alloc_comp_entries(struct lod_object *lo, + int mirror_count, int comp_count) { - LASSERT(cnt != 0); + LASSERT(comp_count != 0); LASSERT(lo->ldo_comp_cnt == 0 && lo->ldo_comp_entries == NULL); + if (mirror_count > 0) { + OBD_ALLOC(lo->ldo_mirrors, + sizeof(*lo->ldo_mirrors) * mirror_count); + if (!lo->ldo_mirrors) + return -ENOMEM; + + lo->ldo_mirror_count = mirror_count; + } + OBD_ALLOC_LARGE(lo->ldo_comp_entries, - sizeof(*lo->ldo_comp_entries) * cnt); - if (lo->ldo_comp_entries == NULL) + sizeof(*lo->ldo_comp_entries) * comp_count); + if (lo->ldo_comp_entries == NULL) { + OBD_FREE(lo->ldo_mirrors, + sizeof(*lo->ldo_mirrors) * mirror_count); + lo->ldo_mirror_count = 0; return -ENOMEM; - lo->ldo_comp_cnt = cnt; + } + + lo->ldo_comp_cnt = comp_count; return 0; } +int lod_fill_mirrors(struct lod_object *lo) +{ + struct lod_layout_component *lod_comp; + int mirror_idx = -1; + __u16 mirror_id = 0xffff; + int i; + ENTRY; + + LASSERT(equi(!lo->ldo_is_composite, lo->ldo_mirror_count == 0)); + + if (!lo->ldo_is_composite) + RETURN(0); + + lod_comp = &lo->ldo_comp_entries[0]; + for (i = 0; i < lo->ldo_comp_cnt; i++, lod_comp++) { + int stale = !!(lod_comp->llc_flags & LCME_FL_STALE); + + if (mirror_id_of(lod_comp->llc_id) == mirror_id) { + lo->ldo_mirrors[mirror_idx].lme_stale |= stale; + lo->ldo_mirrors[mirror_idx].lme_end = i; + continue; + } + + /* new mirror */ + ++mirror_idx; + if (mirror_idx >= lo->ldo_mirror_count) + RETURN(-EINVAL); + + mirror_id = mirror_id_of(lod_comp->llc_id); + + lo->ldo_mirrors[mirror_idx].lme_id = mirror_id; + lo->ldo_mirrors[mirror_idx].lme_stale = stale; + lo->ldo_mirrors[mirror_idx].lme_start = i; + lo->ldo_mirrors[mirror_idx].lme_end = i; + } + if (mirror_idx != lo->ldo_mirror_count - 1) + RETURN(-EINVAL); + + RETURN(0); +} + /** * Generate on-disk lov_mds_md structure for each layout component based on * the information in lod_object->ldo_comp_entries[i]. @@ -848,52 +910,6 @@ done: } /** - * Generate component ID for new created component. - * - * \param[in] lo LOD object - * \param[in] comp_idx index of ldo_comp_entries - * - * \retval component ID on success - * \retval LCME_ID_INVAL on failure - */ -static __u32 lod_gen_component_id(struct lod_object *lo, int comp_idx) -{ - struct lod_layout_component *lod_comp; - __u32 id, start, end; - int i; - - LASSERT(lo->ldo_comp_entries[comp_idx].llc_id == LCME_ID_INVAL); - - lod_obj_inc_layout_gen(lo); - id = lo->ldo_layout_gen; - if (likely(id <= LCME_ID_MAX)) - return id; - - /* Layout generation wraps, need to check collisions. */ - start = id & LCME_ID_MASK; - end = LCME_ID_MAX; -again: - for (id = start; id <= end; id++) { - for (i = 0; i < lo->ldo_comp_cnt; i++) { - lod_comp = &lo->ldo_comp_entries[i]; - if (id == lod_comp->llc_id) - break; - } - /* Found the ununsed ID */ - if (i == lo->ldo_comp_cnt) - return id; - } - if (end == LCME_ID_MAX) { - start = 1; - end = min(lo->ldo_layout_gen & LCME_ID_MASK, - (__u32)(LCME_ID_MAX - 1)); - goto again; - } - - return LCME_ID_INVAL; -} - -/** * Generate on-disk lov_mds_md structure based on the information in * the lod_object->ldo_comp_entries. * @@ -916,18 +932,20 @@ int lod_generate_lovea(const struct lu_env *env, struct lod_object *lo, struct lov_comp_md_entry_v1 *lcme; struct lov_comp_md_v1 *lcm; struct lod_layout_component *comp_entries; - __u16 comp_cnt; + __u16 comp_cnt, mirror_cnt; bool is_composite; int i, rc = 0, offset; ENTRY; if (is_dir) { comp_cnt = lo->ldo_def_striping->lds_def_comp_cnt; + mirror_cnt = lo->ldo_def_striping->lds_def_mirror_cnt; comp_entries = lo->ldo_def_striping->lds_def_comp_entries; is_composite = lo->ldo_def_striping->lds_def_striping_is_composite; } else { comp_cnt = lo->ldo_comp_cnt; + mirror_cnt = lo->ldo_mirror_count; comp_entries = lo->ldo_comp_entries; is_composite = lo->ldo_is_composite; } @@ -943,7 +961,7 @@ int lod_generate_lovea(const struct lu_env *env, struct lod_object *lo, lcm = (struct lov_comp_md_v1 *)lmm; lcm->lcm_magic = cpu_to_le32(LOV_MAGIC_COMP_V1); lcm->lcm_entry_count = cpu_to_le16(comp_cnt); - lcm->lcm_mirror_count = cpu_to_le16(lo->ldo_mirror_count); + lcm->lcm_mirror_count = cpu_to_le16(mirror_cnt - 1); lcm->lcm_flags = cpu_to_le16(lo->ldo_flr_state); offset = sizeof(*lcm) + sizeof(*lcme) * comp_cnt; @@ -957,11 +975,7 @@ int lod_generate_lovea(const struct lu_env *env, struct lod_object *lo, lod_comp = &comp_entries[i]; lcme = &lcm->lcm_entries[i]; - if (lod_comp->llc_id == LCME_ID_INVAL && !is_dir) { - lod_comp->llc_id = lod_gen_component_id(lo, i); - if (lod_comp->llc_id == LCME_ID_INVAL) - GOTO(out, rc = -ERANGE); - } + LASSERT(ergo(!is_dir, lod_comp->llc_id != LCME_ID_INVAL)); lcme->lcme_id = cpu_to_le32(lod_comp->llc_id); /* component could be un-inistantiated */ @@ -1201,6 +1215,7 @@ int lod_parse_striping(const struct lu_env *env, struct lod_object *lo, __u32 magic, pattern; int i, j, rc = 0; __u16 comp_cnt; + __u16 mirror_cnt = 0; ENTRY; LASSERT(buf); @@ -1225,14 +1240,14 @@ int lod_parse_striping(const struct lu_env *env, struct lod_object *lo, lo->ldo_is_composite = 1; lo->ldo_flr_state = le16_to_cpu(comp_v1->lcm_flags) & LCM_FL_FLR_MASK; - lo->ldo_mirror_count = le16_to_cpu(comp_v1->lcm_mirror_count); + mirror_cnt = le16_to_cpu(comp_v1->lcm_mirror_count) + 1; } else { comp_cnt = 1; lo->ldo_layout_gen = le16_to_cpu(lmm->lmm_layout_gen); lo->ldo_is_composite = 0; } - rc = lod_alloc_comp_entries(lo, comp_cnt); + rc = lod_alloc_comp_entries(lo, mirror_cnt, comp_cnt); if (rc) GOTO(out, rc); @@ -1332,6 +1347,11 @@ int lod_parse_striping(const struct lu_env *env, struct lod_object *lo, GOTO(out, rc); } } + + rc = lod_fill_mirrors(lo); + if (rc) + GOTO(out, rc); + out: if (rc) lod_object_free_striping(env, lo); diff --git a/lustre/lod/lod_object.c b/lustre/lod/lod_object.c index 7b54498..e53a438 100644 --- a/lustre/lod/lod_object.c +++ b/lustre/lod/lod_object.c @@ -2218,7 +2218,7 @@ static int lod_declare_layout_add(const struct lu_env *env, struct thandle *th) { struct lod_thread_info *info = lod_env_info(env); - struct lod_layout_component *comp_array, *lod_comp; + struct lod_layout_component *comp_array, *lod_comp, *old_array; struct lod_device *d = lu2lod_dev(dt->do_lu.lo_dev); struct dt_object *next = dt_object_child(dt); struct lov_desc *desc = &d->lod_desc; @@ -2226,11 +2226,14 @@ static int lod_declare_layout_add(const struct lu_env *env, struct lov_user_md_v3 *v3; struct lov_comp_md_v1 *comp_v1 = buf->lb_buf; __u32 magic; - int i, rc, array_cnt; + int i, rc, array_cnt, old_array_cnt; ENTRY; LASSERT(lo->ldo_is_composite); + if (lo->ldo_flr_state != LCM_FL_NOT_FLR) + RETURN(-EBUSY); + rc = lod_verify_striping(d, lo, buf, false); if (rc != 0) RETURN(rc); @@ -2287,7 +2290,9 @@ static int lod_declare_layout_add(const struct lu_env *env, } } - OBD_FREE(lo->ldo_comp_entries, sizeof(*lod_comp) * lo->ldo_comp_cnt); + old_array = lo->ldo_comp_entries; + old_array_cnt = lo->ldo_comp_cnt; + lo->ldo_comp_entries = comp_array; lo->ldo_comp_cnt = array_cnt; @@ -2297,8 +2302,16 @@ static int lod_declare_layout_add(const struct lu_env *env, info->lti_buf.lb_len = lod_comp_md_size(lo, false); rc = lod_sub_declare_xattr_set(env, next, &info->lti_buf, XATTR_NAME_LOV, 0, th); - if (rc) + if (rc) { + lo->ldo_comp_entries = old_array; + lo->ldo_comp_cnt = old_array_cnt; GOTO(error, rc); + } + + OBD_FREE(old_array, sizeof(*lod_comp) * old_array_cnt); + + LASSERT(lo->ldo_mirror_count == 1); + lo->ldo_mirrors[0].lme_end = array_cnt - 1; RETURN(0); @@ -2422,6 +2435,9 @@ static int lod_declare_layout_del(const struct lu_env *env, LASSERT(lo->ldo_is_composite); + if (lo->ldo_flr_state != LCM_FL_NOT_FLR) + RETURN(-EBUSY); + magic = comp_v1->lcm_magic; if (magic == __swab32(LOV_USER_MAGIC_COMP_V1)) { lustre_swab_lov_comp_md_v1(comp_v1); @@ -3453,6 +3469,9 @@ static int lod_layout_del(const struct lu_env *env, struct dt_object *dt, sizeof(*comp_array) * lo->ldo_comp_cnt); lo->ldo_comp_entries = comp_array; lo->ldo_comp_cnt = left; + + LASSERT(lo->ldo_mirror_count == 1); + lo->ldo_mirrors[0].lme_end = left - 1; lod_obj_inc_layout_gen(lo); } else { lod_free_comp_entries(lo); @@ -3700,6 +3719,7 @@ static int lod_get_default_lov_striping(const struct lu_env *env, struct lov_user_md_v3 *v3 = NULL; struct lov_comp_md_v1 *comp_v1 = NULL; __u16 comp_cnt; + __u16 mirror_cnt; bool composite; int rc, i; ENTRY; @@ -3733,9 +3753,11 @@ static int lod_get_default_lov_striping(const struct lu_env *env, comp_cnt = comp_v1->lcm_entry_count; if (comp_cnt == 0) RETURN(-EINVAL); + mirror_cnt = comp_v1->lcm_mirror_count + 1; composite = true; } else { comp_cnt = 1; + mirror_cnt = 0; composite = false; } @@ -3745,7 +3767,8 @@ static int lod_get_default_lov_striping(const struct lu_env *env, RETURN(rc); lds->lds_def_comp_cnt = comp_cnt; - lds->lds_def_striping_is_composite = composite ? 1 : 0; + lds->lds_def_striping_is_composite = composite; + lds->lds_def_mirror_cnt = mirror_cnt; for (i = 0; i < comp_cnt; i++) { struct lod_layout_component *lod_comp; @@ -3881,7 +3904,8 @@ static void lod_striping_from_default(struct lod_object *lo, int i, rc; if (lds->lds_def_striping_set && S_ISREG(mode)) { - rc = lod_alloc_comp_entries(lo, lds->lds_def_comp_cnt); + rc = lod_alloc_comp_entries(lo, lds->lds_def_mirror_cnt, + lds->lds_def_comp_cnt); if (rc != 0) return; @@ -4142,7 +4166,7 @@ out: if (lod_need_inherit_more(lc, false)) { if (lc->ldo_comp_cnt == 0) { - rc = lod_alloc_comp_entries(lc, 1); + rc = lod_alloc_comp_entries(lc, 0, 1); if (rc) /* fail to allocate memory, will create a * non-striped file. */ @@ -4429,6 +4453,53 @@ out: } /** + * Generate component ID for new created component. + * + * \param[in] lo LOD object + * \param[in] comp_idx index of ldo_comp_entries + * + * \retval component ID on success + * \retval LCME_ID_INVAL on failure + */ +static __u32 lod_gen_component_id(struct lod_object *lo, + int mirror_id, int comp_idx) +{ + struct lod_layout_component *lod_comp; + __u32 id, start, end; + int i; + + LASSERT(lo->ldo_comp_entries[comp_idx].llc_id == LCME_ID_INVAL); + + lod_obj_inc_layout_gen(lo); + id = lo->ldo_layout_gen; + if (likely(id <= SEQ_ID_MAX)) + RETURN(pflr_id(mirror_id, id & SEQ_ID_MASK)); + + /* Layout generation wraps, need to check collisions. */ + start = id & SEQ_ID_MASK; + end = SEQ_ID_MAX; +again: + for (id = start; id <= end; id++) { + for (i = 0; i < lo->ldo_comp_cnt; i++) { + lod_comp = &lo->ldo_comp_entries[i]; + if (pflr_id(mirror_id, id) == lod_comp->llc_id) + break; + } + /* Found the ununsed ID */ + if (i == lo->ldo_comp_cnt) + RETURN(pflr_id(mirror_id, id)); + } + if (end == LCME_ID_MAX) { + start = 1; + end = min(lo->ldo_layout_gen & LCME_ID_MASK, + (__u32)(LCME_ID_MAX - 1)); + goto again; + } + + RETURN(LCME_ID_INVAL); +} + +/** * Creation of a striped regular object. * * The function is called to create the stripe objects for a regular @@ -4462,6 +4533,12 @@ int lod_striped_create(const struct lu_env *env, struct dt_object *dt, for (i = 0; i < lo->ldo_comp_cnt; i++) { lod_comp = &lo->ldo_comp_entries[i]; + if (lod_comp->llc_id == LCME_ID_INVAL) { + lod_comp->llc_id = lod_gen_component_id(lo, 0, i); + if (lod_comp->llc_id == LCME_ID_INVAL) + GOTO(out, rc = -ERANGE); + } + if (lod_comp_inited(lod_comp)) continue; @@ -4480,19 +4557,24 @@ int lod_striped_create(const struct lu_env *env, struct dt_object *dt, LASSERT(object != NULL); rc = lod_sub_create(env, object, attr, NULL, dof, th); if (rc) - break; + GOTO(out, rc); } lod_comp_set_init(lod_comp); } - if (rc == 0) - rc = lod_generate_and_set_lovea(env, lo, th); + rc = lod_fill_mirrors(lo); + if (rc) + GOTO(out, rc); - if (rc == 0) - lo->ldo_comp_cached = 1; - else - lod_object_free_striping(env, lo); + rc = lod_generate_and_set_lovea(env, lo, th); + if (rc) + GOTO(out, rc); + + lo->ldo_comp_cached = 1; + RETURN(0); +out: + lod_object_free_striping(env, lo); RETURN(rc); } diff --git a/lustre/lod/lod_qos.c b/lustre/lod/lod_qos.c index 01beb59..4943bc2 100644 --- a/lustre/lod/lod_qos.c +++ b/lustre/lod/lod_qos.c @@ -1726,6 +1726,7 @@ int lod_use_defined_striping(const struct lu_env *env, struct lov_ost_data_v1 *objs; __u32 magic; __u16 comp_cnt; + __u16 mirror_cnt; int rc = 0, i; ENTRY; @@ -1740,13 +1741,16 @@ int lod_use_defined_striping(const struct lu_env *env, comp_cnt = le16_to_cpu(comp_v1->lcm_entry_count); if (comp_cnt == 0) RETURN(-EINVAL); + mirror_cnt = le16_to_cpu(comp_v1->lcm_mirror_count) + 1; mo->ldo_is_composite = 1; } else { mo->ldo_is_composite = 0; comp_cnt = 1; + mirror_cnt = 0; } + mo->ldo_layout_gen = le16_to_cpu(v1->lmm_layout_gen); - rc = lod_alloc_comp_entries(mo, comp_cnt); + rc = lod_alloc_comp_entries(mo, mirror_cnt, comp_cnt); if (rc) RETURN(rc); @@ -1806,6 +1810,10 @@ int lod_use_defined_striping(const struct lu_env *env, GOTO(out, rc); } } + + rc = lod_fill_mirrors(mo); + if (rc) + GOTO(out, rc); out: if (rc) lod_object_free_striping(env, mo); @@ -1841,6 +1849,7 @@ int lod_qos_parse_config(const struct lu_env *env, struct lod_object *lo, struct lov_comp_md_v1 *comp_v1 = NULL; __u32 magic; __u16 comp_cnt; + __u16 mirror_cnt; int i, rc; ENTRY; @@ -1903,13 +1912,15 @@ int lod_qos_parse_config(const struct lu_env *env, struct lod_object *lo, comp_cnt = comp_v1->lcm_entry_count; if (comp_cnt == 0) RETURN(-EINVAL); + mirror_cnt = comp_v1->lcm_mirror_count + 1; lo->ldo_is_composite = 1; } else { comp_cnt = 1; + mirror_cnt = 0; lo->ldo_is_composite = 0; } - rc = lod_alloc_comp_entries(lo, comp_cnt); + rc = lod_alloc_comp_entries(lo, mirror_cnt, comp_cnt); if (rc) RETURN(rc); -- 1.8.3.1