From 25faf70d7b46600ce35bed8e6e834318597ff9d3 Mon Sep 17 00:00:00 2001 From: Jinshan Xiong Date: Fri, 15 Sep 2017 21:29:58 +0000 Subject: [PATCH] LU-9771 lod: support setting mirror layout from CLI Parse and support mirror layout from CLI. Test-Parameters: testlist=sanity-flr Signed-off-by: Jinshan Xiong Signed-off-by: Bobi Jam Change-Id: I51d24aecb63af04cb3c1dd1bcfbf5728030b233b Reviewed-on: https://review.whamcloud.com/29098 Reviewed-by: Lai Siyao Tested-by: Jenkins Reviewed-by: Andreas Dilger Reviewed-by: Bobi Jam Tested-by: Maloo --- lustre/lod/lod_lov.c | 225 +++++++++++++++++++++++++++--------------------- lustre/lod/lod_object.c | 51 ++++++----- lustre/lod/lod_qos.c | 9 +- 3 files changed, 161 insertions(+), 124 deletions(-) diff --git a/lustre/lod/lod_lov.c b/lustre/lod/lod_lov.c index 45f74c0..e139fe1 100644 --- a/lustre/lod/lod_lov.c +++ b/lustre/lod/lod_lov.c @@ -1652,10 +1652,18 @@ out: int lod_verify_striping(struct lod_device *d, struct lod_object *lo, const struct lu_buf *buf, bool is_from_disk) { - struct lov_user_md_v1 *lum; - struct lov_comp_md_v1 *comp_v1; - __u32 magic; - int rc = 0, i; + struct lov_desc *desc = &d->lod_desc; + struct lov_user_md_v1 *lum; + struct lov_comp_md_v1 *comp_v1; + struct lov_comp_md_entry_v1 *ent; + struct lu_extent *ext; + struct lu_buf tmp; + __u64 prev_end = 0; + __u32 stripe_size = 0; + __u16 prev_mid = -1, mirror_id = -1; + __u32 mirror_count = 0; + __u32 magic; + int rc = 0, i; ENTRY; lum = buf->lb_buf; @@ -1676,125 +1684,142 @@ int lod_verify_striping(struct lod_device *d, struct lod_object *lo, RETURN(-EINVAL); } - if (magic == LOV_USER_MAGIC_COMP_V1) { - struct lov_comp_md_entry_v1 *ent; - struct lu_extent *ext; - struct lov_desc *desc = &d->lod_desc; - struct lu_buf tmp; - __u64 prev_end = 0; - __u32 stripe_size = 0; - - comp_v1 = buf->lb_buf; - if (buf->lb_len < le32_to_cpu(comp_v1->lcm_size)) { - CDEBUG(D_LAYOUT, "buf len %zu is less than %u\n", - buf->lb_len, le32_to_cpu(comp_v1->lcm_size)); - RETURN(-EINVAL); - } + if (magic != LOV_USER_MAGIC_COMP_V1) + RETURN(lod_verify_v1v3(d, buf, is_from_disk)); - if (le16_to_cpu(comp_v1->lcm_entry_count) == 0) { - CDEBUG(D_LAYOUT, "entry count is zero\n"); - RETURN(-EINVAL); - } + /* magic == LOV_USER_MAGIC_COMP_V1 */ + comp_v1 = buf->lb_buf; + if (buf->lb_len < le32_to_cpu(comp_v1->lcm_size)) { + CDEBUG(D_LAYOUT, "buf len %zu is less than %u\n", + buf->lb_len, le32_to_cpu(comp_v1->lcm_size)); + RETURN(-EINVAL); + } - if (S_ISREG(lod2lu_obj(lo)->lo_header->loh_attr) && - lo->ldo_comp_cnt > 0) { - __u32 cnt = lo->ldo_comp_cnt; + if (le16_to_cpu(comp_v1->lcm_entry_count) == 0) { + CDEBUG(D_LAYOUT, "entry count is zero\n"); + RETURN(-EINVAL); + } - ext = &lo->ldo_comp_entries[cnt - 1].llc_extent; - prev_end = ext->e_end; - } + if (S_ISREG(lod2lu_obj(lo)->lo_header->loh_attr) && + lo->ldo_comp_cnt > 0) { + /* could be called from lustre.lov.add */ + __u32 cnt = lo->ldo_comp_cnt; - for (i = 0; i < le16_to_cpu(comp_v1->lcm_entry_count); i++) { - ent = &comp_v1->lcm_entries[i]; - ext = &ent->lcme_extent; + ext = &lo->ldo_comp_entries[cnt - 1].llc_extent; + prev_end = ext->e_end; - if (is_from_disk && - (le32_to_cpu(ent->lcme_id) == 0 || - le32_to_cpu(ent->lcme_id) > LCME_ID_MAX)) { + ++mirror_count; + } + + for (i = 0; i < le16_to_cpu(comp_v1->lcm_entry_count); i++) { + ent = &comp_v1->lcm_entries[i]; + ext = &ent->lcme_extent; + + if (le64_to_cpu(ext->e_start) >= le64_to_cpu(ext->e_end)) { + CDEBUG(D_LAYOUT, "invalid extent "DEXT"\n", + le64_to_cpu(ext->e_start), + le64_to_cpu(ext->e_end)); + RETURN(-EINVAL); + } + + if (is_from_disk) { + /* lcme_id contains valid value */ + if (le32_to_cpu(ent->lcme_id) == 0 || + le32_to_cpu(ent->lcme_id) > LCME_ID_MAX) { CDEBUG(D_LAYOUT, "invalid id %u\n", le32_to_cpu(ent->lcme_id)); RETURN(-EINVAL); } - if (le64_to_cpu(ext->e_start) >= - le64_to_cpu(ext->e_end)) { - CDEBUG(D_LAYOUT, "invalid extent " - "[%llu, %llu)\n", - le64_to_cpu(ext->e_start), - le64_to_cpu(ext->e_end)); - RETURN(-EINVAL); - } + if (le16_to_cpu(comp_v1->lcm_mirror_count) > 0) { + mirror_id = mirror_id_of( + le32_to_cpu(ent->lcme_id)); - /* first component must start with 0, and the next - * must be adjacent with the previous one */ - if (le64_to_cpu(ext->e_start) != prev_end) { - CDEBUG(D_LAYOUT, "invalid start " - "actual:%llu, expect:%llu\n", - le64_to_cpu(ext->e_start), prev_end); - RETURN(-EINVAL); + /* first component must start with 0 */ + if (mirror_id != prev_mid && + le64_to_cpu(ext->e_start) != 0) { + CDEBUG(D_LAYOUT, + "invalid start:%llu, expect:0\n", + le64_to_cpu(ext->e_start)); + RETURN(-EINVAL); + } + + prev_mid = mirror_id; } + } - prev_end = le64_to_cpu(ext->e_end); + if (le64_to_cpu(ext->e_start) == 0) { + ++mirror_count; + prev_end = 0; + } - tmp.lb_buf = (char *)comp_v1 + - le32_to_cpu(ent->lcme_offset); - tmp.lb_len = le32_to_cpu(ent->lcme_size); + /* the next must be adjacent with the previous one */ + if (le64_to_cpu(ext->e_start) != prev_end) { + CDEBUG(D_LAYOUT, + "invalid start actual:%llu, expect:%llu\n", + le64_to_cpu(ext->e_start), prev_end); + RETURN(-EINVAL); + } - /* Checks for DoM entry in composite layout. */ - lum = tmp.lb_buf; - if (lov_pattern(le32_to_cpu(lum->lmm_pattern)) == - LOV_PATTERN_MDT) { - /* DoM component can be only the first entry */ - if (i > 0) { - CDEBUG(D_LAYOUT, "invalid DoM layout " - "entry found at %i index\n", i); - RETURN(-EINVAL); - } - stripe_size = le32_to_cpu(lum->lmm_stripe_size); - /* There is just one stripe on MDT and it must - * cover whole component size. */ - if (stripe_size != prev_end) { - CDEBUG(D_LAYOUT, "invalid DoM layout " - "stripe size %u != %llu " - "(component size)\n", - stripe_size, prev_end); - RETURN(-EINVAL); - } - /* Check stripe size againts per-MDT limit */ - if (stripe_size > d->lod_dom_max_stripesize) { - CDEBUG(D_LAYOUT, "DoM component size " - "%u is bigger than MDT limit " - "%u, check dom_max_stripesize" - " parameter\n", - stripe_size, - d->lod_dom_max_stripesize); - RETURN(-EINVAL); - } - } - rc = lod_verify_v1v3(d, &tmp, is_from_disk); - if (rc) - break; + prev_end = le64_to_cpu(ext->e_end); - lum = tmp.lb_buf; + tmp.lb_buf = (char *)comp_v1 + le32_to_cpu(ent->lcme_offset); + tmp.lb_len = le32_to_cpu(ent->lcme_size); - /* extent end must be aligned with the stripe_size */ + /* Check DoM entry is always the first one */ + lum = tmp.lb_buf; + if (lov_pattern(le32_to_cpu(lum->lmm_pattern)) == + LOV_PATTERN_MDT) { + /* DoM component can be only the first entry */ + if (i > 0) { + CDEBUG(D_LAYOUT, "invalid DoM layout " + "entry found at %i index\n", i); + RETURN(-EINVAL); + } stripe_size = le32_to_cpu(lum->lmm_stripe_size); - if (stripe_size == 0) - stripe_size = desc->ld_default_stripe_size; - if (stripe_size == 0 || - (prev_end != LUSTRE_EOF && - (prev_end & (stripe_size - 1)))) { - CDEBUG(D_LAYOUT, "stripe size isn't aligned. " - " stripe_sz: %u, [%llu, %llu)\n", - stripe_size, ext->e_start, prev_end); + /* There is just one stripe on MDT and it must + * cover whole component size. */ + if (stripe_size != prev_end) { + CDEBUG(D_LAYOUT, "invalid DoM layout " + "stripe size %u != %llu " + "(component size)\n", + stripe_size, prev_end); + RETURN(-EINVAL); + } + /* Check stripe size againts per-MDT limit */ + if (stripe_size > d->lod_dom_max_stripesize) { + CDEBUG(D_LAYOUT, "DoM component size " + "%u is bigger than MDT limit %u, check " + "dom_max_stripesize parameter\n", + stripe_size, d->lod_dom_max_stripesize); RETURN(-EINVAL); } } - } else { - rc = lod_verify_v1v3(d, buf, is_from_disk); + + rc = lod_verify_v1v3(d, &tmp, is_from_disk); + if (rc) + RETURN(rc); + + if (prev_end == LUSTRE_EOF) + continue; + + /* extent end must be aligned with the stripe_size */ + stripe_size = le32_to_cpu(lum->lmm_stripe_size); + if (stripe_size == 0) + stripe_size = desc->ld_default_stripe_size; + if (stripe_size == 0 || (prev_end & (stripe_size - 1))) { + CDEBUG(D_LAYOUT, "stripe size isn't aligned, " + "stripe_sz: %u, [%llu, %llu)\n", + stripe_size, ext->e_start, prev_end); + RETURN(-EINVAL); + } } - RETURN(rc); + /* make sure that the mirror_count is telling the truth */ + if (mirror_count != le16_to_cpu(comp_v1->lcm_mirror_count) + 1) + RETURN(-EINVAL); + + RETURN(0); } /** diff --git a/lustre/lod/lod_object.c b/lustre/lod/lod_object.c index 1372f31..24516ed 100644 --- a/lustre/lod/lod_object.c +++ b/lustre/lod/lod_object.c @@ -4241,7 +4241,6 @@ out: * in config log, use them. */ if (lod_need_inherit_more(lc, false)) { - if (lc->ldo_comp_cnt == 0) { rc = lod_alloc_comp_entries(lc, 0, 1); if (rc) @@ -4292,6 +4291,7 @@ static int lod_declare_init_size(const struct lu_env *env, struct lu_attr *attr = &lod_env_info(env)->lti_attr; uint64_t size, offs; int i, rc, stripe, stripe_count = 0, stripe_size = 0; + struct lu_extent size_ext; ENTRY; if (!lod_obj_is_striped(dt)) @@ -4306,6 +4306,7 @@ static int lod_declare_init_size(const struct lu_env *env, if (size == 0) RETURN(0); + size_ext = (typeof(size_ext)){ .e_start = size - 1, .e_end = size }; for (i = 0; i < lo->ldo_comp_cnt; i++) { struct lod_layout_component *lod_comp; struct lu_extent *extent; @@ -4316,35 +4317,34 @@ static int lod_declare_init_size(const struct lu_env *env, continue; extent = &lod_comp->llc_extent; - CDEBUG(D_INFO, "%lld [%lld, %lld)\n", - size, extent->e_start, extent->e_end); + CDEBUG(D_INFO, "%lld "DEXT"\n", size, PEXT(extent)); if (!lo->ldo_is_composite || - (size >= extent->e_start && size < extent->e_end)) { + lu_extent_is_overlapped(extent, &size_ext)) { objects = lod_comp->llc_stripe; stripe_count = lod_comp->llc_stripe_count; stripe_size = lod_comp->llc_stripe_size; - break; - } - } - if (stripe_count == 0) - RETURN(0); + /* next mirror */ + if (stripe_count == 0) + continue; - LASSERT(objects != NULL && stripe_size != 0); + LASSERT(objects != NULL && stripe_size != 0); + /* ll_do_div64(a, b) returns a % b, and a = a / b */ + ll_do_div64(size, (__u64)stripe_size); + stripe = ll_do_div64(size, (__u64)stripe_count); + LASSERT(objects[stripe] != NULL); - /* ll_do_div64(a, b) returns a % b, and a = a / b */ - ll_do_div64(size, (__u64)stripe_size); - stripe = ll_do_div64(size, (__u64)stripe_count); - LASSERT(objects[stripe] != NULL); + size = size * stripe_size; + offs = attr->la_size; + size += ll_do_div64(offs, stripe_size); - size = size * stripe_size; - offs = attr->la_size; - size += ll_do_div64(offs, stripe_size); + attr->la_valid = LA_SIZE; + attr->la_size = size; - attr->la_valid = LA_SIZE; - attr->la_size = size; - - rc = lod_sub_declare_attr_set(env, objects[stripe], attr, th); + rc = lod_sub_declare_attr_set(env, objects[stripe], + attr, th); + } + } RETURN(rc); } @@ -4601,17 +4601,24 @@ int lod_striped_create(const struct lu_env *env, struct dt_object *dt, { struct lod_layout_component *lod_comp; struct lod_object *lo = lod_dt_obj(dt); + __u16 mirror_id; int rc = 0, i, j; ENTRY; LASSERT(lo->ldo_comp_cnt != 0 && lo->ldo_comp_entries != NULL); + mirror_id = lo->ldo_mirror_count > 1 ? 1 : 0; + /* create all underlying objects */ for (i = 0; i < lo->ldo_comp_cnt; i++) { lod_comp = &lo->ldo_comp_entries[i]; + if (lod_comp->llc_extent.e_start == 0 && i > 0) /* new mirror */ + ++mirror_id; + if (lod_comp->llc_id == LCME_ID_INVAL) { - lod_comp->llc_id = lod_gen_component_id(lo, 0, i); + lod_comp->llc_id = lod_gen_component_id(lo, + mirror_id, i); if (lod_comp->llc_id == LCME_ID_INVAL) GOTO(out, rc = -ERANGE); } diff --git a/lustre/lod/lod_qos.c b/lustre/lod/lod_qos.c index 12aee55..17abe3c 100644 --- a/lustre/lod/lod_qos.c +++ b/lustre/lod/lod_qos.c @@ -1742,6 +1742,8 @@ int lod_use_defined_striping(const struct lu_env *env, if (comp_cnt == 0) RETURN(-EINVAL); mirror_cnt = le16_to_cpu(comp_v1->lcm_mirror_count) + 1; + mo->ldo_flr_state = le16_to_cpu(comp_v1->lcm_flags) & + LCM_FL_FLR_MASK; mo->ldo_is_composite = 1; } else { mo->ldo_is_composite = 0; @@ -1856,12 +1858,13 @@ int lod_qos_parse_config(const struct lu_env *env, struct lod_object *lo, if (buf == NULL || buf->lb_buf == NULL || buf->lb_len == 0) RETURN(0); + /* free default striping info */ + lod_free_comp_entries(lo); + rc = lod_verify_striping(d, lo, buf, false); if (rc) RETURN(-EINVAL); - lod_free_comp_entries(lo); - v3 = buf->lb_buf; v1 = buf->lb_buf; comp_v1 = buf->lb_buf; @@ -1913,6 +1916,8 @@ int lod_qos_parse_config(const struct lu_env *env, struct lod_object *lo, if (comp_cnt == 0) RETURN(-EINVAL); mirror_cnt = comp_v1->lcm_mirror_count + 1; + if (mirror_cnt > 1) + lo->ldo_flr_state = LCM_FL_RDONLY; lo->ldo_is_composite = 1; } else { comp_cnt = 1; -- 1.8.3.1