From 83c790cbf2f8f7452e1382051564af6f155b47cf Mon Sep 17 00:00:00 2001 From: Bobi Jam Date: Fri, 3 Sep 2021 00:27:34 +0800 Subject: [PATCH] LU-14514 flr: mirror split should not make stale file Mirror split could leave an all stale mirrors file, this patch prevent removing the last non-stale mirror from the file. Signed-off-by: Bobi Jam Change-Id: I63007784929a2cd18d2823e2250f7307ca7d8d45 Reviewed-on: https://review.whamcloud.com/42024 Reviewed-by: Andreas Dilger Reviewed-by: John L. Hammond Tested-by: jenkins Tested-by: Maloo Reviewed-by: Oleg Drokin --- lustre/lod/lod_internal.h | 8 +++++-- lustre/lod/lod_lov.c | 55 ++++++++++++++++++++++++++++++++++++------ lustre/lod/lod_object.c | 6 ++--- lustre/tests/sanity-flr.sh | 59 ++++++++++++++++++++++++++++++++++++++++++++-- 4 files changed, 114 insertions(+), 14 deletions(-) diff --git a/lustre/lod/lod_internal.h b/lustre/lod/lod_internal.h index 89da19f..61d5199 100644 --- a/lustre/lod/lod_internal.h +++ b/lustre/lod/lod_internal.h @@ -225,6 +225,10 @@ static inline __u8 lmv_inherit_rr_next(__u8 inherit_rr) return inherit_rr - 1; } +enum layout_verify_flags { + LVF_ALL_STALE = BIT(0), /* check not all stale mirrors */ +}; + struct lod_mirror_entry { __u16 lme_stale:1, lme_prefer:1; @@ -596,7 +600,7 @@ int lod_fini_tgt(const struct lu_env *env, struct lod_device *lod, struct lod_tgt_descs *ltd); int lod_striping_load(const struct lu_env *env, struct lod_object *lo); int lod_striping_reload(const struct lu_env *env, struct lod_object *lo, - const struct lu_buf *buf); + const struct lu_buf *buf, enum layout_verify_flags lvf); void lod_dom_stripesize_recalc(struct lod_device *d); int lod_get_ea(const struct lu_env *env, struct lod_object *lo, @@ -666,7 +670,7 @@ void lod_fix_lmv_desc_pattern(__u32 *val); int lod_pools_init(struct lod_device *m, struct lustre_cfg *cfg); int lod_pools_fini(struct lod_device *m); int lod_parse_striping(const struct lu_env *env, struct lod_object *mo, - const struct lu_buf *buf); + const struct lu_buf *buf, enum layout_verify_flags lvf); int lod_parse_dir_striping(const struct lu_env *env, struct lod_object *lo, const struct lu_buf *buf); int lod_initialize_objects(const struct lu_env *env, struct lod_object *mo, diff --git a/lustre/lod/lod_lov.c b/lustre/lod/lod_lov.c index 3dd8696..47ebff7 100644 --- a/lustre/lod/lod_lov.c +++ b/lustre/lod/lod_lov.c @@ -1143,13 +1143,15 @@ out: * \param[in] env execution environment for this thread * \param[in] lo LOD object * \param[in] buf buffer storing LOV EA to parse + * \param[in] lvf verify flags when parsing the layout * * \retval 0 if parsing and objects creation succeed * \retval negative error number on failure */ int lod_parse_striping(const struct lu_env *env, struct lod_object *lo, - const struct lu_buf *buf) + const struct lu_buf *buf, enum layout_verify_flags lvf) { + struct lod_device *d = lu2lod_dev(lo->ldo_obj.do_lu.lo_dev); struct lov_mds_md_v1 *lmm; struct lov_comp_md_v1 *comp_v1 = NULL; struct lov_foreign_md *foreign = NULL; @@ -1158,6 +1160,9 @@ int lod_parse_striping(const struct lu_env *env, struct lod_object *lo, __u16 mirror_cnt = 0; __u16 comp_cnt; int i, rc; + __u16 mirror_id = MIRROR_ID_NEG; + bool stale = false; + int stale_mirrors = 0; ENTRY; LASSERT(buf); @@ -1252,6 +1257,7 @@ int lod_parse_striping(const struct lu_env *env, struct lod_object *lo, lod_comp->llc_flags = le32_to_cpu(comp_v1->lcm_entries[i].lcme_flags); + if (lod_comp->llc_flags & LCME_FL_NOSYNC) lod_comp->llc_timestamp = le64_to_cpu( comp_v1->lcm_entries[i].lcme_timestamp); @@ -1260,11 +1266,31 @@ int lod_parse_striping(const struct lu_env *env, struct lod_object *lo, if (lod_comp->llc_id == LCME_ID_INVAL) GOTO(out, rc = -EINVAL); + if (lvf & LVF_ALL_STALE) { + if (mirror_id_of(lod_comp->llc_id) == + mirror_id) { + /* remaining comps in the mirror */ + stale |= lod_comp->llc_flags & + LCME_FL_STALE; + } else { + /* + * new mirror, check last mirror's + * stale-ness + */ + if (stale) + stale_mirrors++; + + mirror_id = + mirror_id_of(lod_comp->llc_id); + + /* the first comp of the new mirror */ + stale = lod_comp->llc_flags & + LCME_FL_STALE; + } + } + if ((lod_comp->llc_flags & LCME_FL_EXTENSION) && comp_v1->lcm_magic != cpu_to_le32(LOV_MAGIC_SEL)) { - struct lod_device *d = - lu2lod_dev(lo->ldo_obj.do_lu.lo_dev); - CWARN("%s: EXTENSION flags=%x set on component[%u]=%x of non-SEL file "DFID" with magic=%#08x\n", lod2obd(d)->obd_name, lod_comp->llc_flags, lod_comp->llc_id, i, @@ -1355,6 +1381,21 @@ int lod_parse_striping(const struct lu_env *env, struct lod_object *lo, } } + if (lo->ldo_is_composite && (lvf & LVF_ALL_STALE)) { + /* check the last mirror stale-ness */ + if (stale) + stale_mirrors++; + + if (mirror_cnt == stale_mirrors) { + rc = -EPERM; + CERROR("%s: can not set all stale mirrors for " + DFID": rc = %d\n", + lod2obd(d)->obd_name, PFID(lod_object_fid(lo)), + rc); + GOTO(out, rc); + } + } + rc = lod_fill_mirrors(lo); if (rc) GOTO(out, rc); @@ -1438,7 +1479,7 @@ int lod_striping_load(const struct lu_env *env, struct lod_object *lo) */ buf->lb_buf = info->lti_ea_store; buf->lb_len = info->lti_ea_store_size; - rc = lod_parse_striping(env, lo, buf); + rc = lod_parse_striping(env, lo, buf, 0); if (rc == 0) lo->ldo_comp_cached = 1; } else if (S_ISDIR(lod2lu_obj(lo)->lo_header->loh_attr)) { @@ -1499,14 +1540,14 @@ unlock: } int lod_striping_reload(const struct lu_env *env, struct lod_object *lo, - const struct lu_buf *buf) + const struct lu_buf *buf, enum layout_verify_flags lvf) { int rc; ENTRY; mutex_lock(&lo->ldo_layout_mutex); - rc = lod_parse_striping(env, lo, buf); + rc = lod_parse_striping(env, lo, buf, lvf); mutex_unlock(&lo->ldo_layout_mutex); RETURN(rc); diff --git a/lustre/lod/lod_object.c b/lustre/lod/lod_object.c index e4147b8..bf4a5bd 100644 --- a/lustre/lod/lod_object.c +++ b/lustre/lod/lod_object.c @@ -3417,7 +3417,7 @@ static int lod_declare_layout_merge(const struct lu_env *env, if ((le16_to_cpu(lcm->lcm_flags) & LCM_FL_FLR_MASK) == LCM_FL_NONE) lcm->lcm_flags = cpu_to_le32(LCM_FL_RDONLY); - rc = lod_striping_reload(env, lo, buf); + rc = lod_striping_reload(env, lo, buf, 0); if (rc) GOTO(out, rc); @@ -3460,7 +3460,7 @@ static int lod_declare_layout_split(const struct lu_env *env, int rc; ENTRY; - rc = lod_striping_reload(env, lo, mbuf); + rc = lod_striping_reload(env, lo, mbuf, LVF_ALL_STALE); if (rc) RETURN(rc); @@ -4879,7 +4879,7 @@ static int lod_xattr_set(const struct lu_env *env, if (rc) RETURN(rc); - rc = lod_striping_reload(env, lo, buf); + rc = lod_striping_reload(env, lo, buf, LVF_ALL_STALE); if (rc) RETURN(rc); diff --git a/lustre/tests/sanity-flr.sh b/lustre/tests/sanity-flr.sh index 803617d..fd811a2 100644 --- a/lustre/tests/sanity-flr.sh +++ b/lustre/tests/sanity-flr.sh @@ -2000,7 +2000,7 @@ test_43b() { } run_test 43b "allow writing to multiple preferred mirror file" -test_44() { +test_44a() { [ $MDSCOUNT -lt 2 ] && skip "needs >= 2 MDTs" && return rm -rf $DIR/$tdir rm -rf $DIR/$tdir-1 @@ -2075,7 +2075,62 @@ test_44() { diff $tf $tf.mirror~2 || error "splited file $tf.mirror~2 diffs from $tf" } -run_test 44 "lfs mirror split check" +run_test 44a "lfs mirror split check" + +test_44b() { + rm -rf $DIR/$tdir + local tf=$DIR/$tdir/$tfile + + mkdir -p $DIR/$tdir || error "create directory failed" + + echo XXX > $tf + + # create 2 mirrors file + $LFS mirror extend -N -c1 $tf + + echo YYY > $tf + + verify_flr_state $tf "wp" + + local str=$(cat $tf) + + [[ $str == "YYY" ]] || error "$tf content is not YYY" + + # get the non-stale mirror id + local ids=($($LFS getstripe $tf | awk '/lcme_id/{print $2}' | + tr '\n' ' ')) + local mirror_ids=($($LFS getstripe $tf | + awk '/lcme_mirror_id/{print $2}' | tr '\n' ' ')) + for ((i = 0; i < 2; i++)); do + $LFS getstripe -I${ids[$i]} --component-flags $tf | + grep stale > /dev/null || break + done + + [[ $i -ge 2 ]] && ( $LFS getstripe $tf; error "no stale mirror" ) + + $LFS getstripe $tf + + # split the updated mirror, should fail + echo "split mirror_id ${mirror_ids[$i]} id ${ids[$i]}, should fail" + $LFS mirror split --mirror-id=${mirror_ids[$i]} $tf &> /dev/null && + error "split --mirror-id=${mirror_ids[$i]} $tf should fail" + + i=$(( 1 - i )) + # split the stale mirror + echo "split mirror_id ${mirror_ids[$i]} id ${ids[$i]}" + $LFS mirror split --mirror-id=${mirror_ids[$i]} -d $tf || + error "mirror split --mirror-id=${mirror_ids[$i]} $tf failed" + + echo "make sure there's no stale comp in the file" + # make sure there's no stale comp in the file + $LFS getstripe $tf | awk '/lcme_flags/{print $2}' | grep stale && + ( $LFS getstripe $tf; error "stale mirror file" ) + + str=$(cat $tf) + [[ $str == "YYY" ]] || + ( cat $tf; error "$tf content is not YYY after split" ) +} +run_test 44b "mirror split does not create stale file" test_44c() { local tf=$DIR/$tdir/$tfile -- 1.8.3.1