LU-14514 flr: mirror split should not make stale file

Mirror split could leave an all stale mirrors file, this patch
prevent removing the last non-stale mirror from the file.

Signed-off-by: Bobi Jam <bobijam@whamcloud.com>
Change-Id: I63007784929a2cd18d2823e2250f7307ca7d8d45
Reviewed-on: https://review.whamcloud.com/42024
Reviewed-by: Andreas Dilger <adilger@whamcloud.com>
Reviewed-by: John L. Hammond <jhammond@whamcloud.com>
Tested-by: jenkins <devops@whamcloud.com>
Tested-by: Maloo <maloo@whamcloud.com>
Reviewed-by: Oleg Drokin <green@whamcloud.com>

diff --git a/lustre/lod/lod_internal.h b/lustre/lod/lod_internal.h
index 89da19f..61d5199 100644 (file)
--- a/lustre/lod/lod_internal.h
+++ b/lustre/lod/lod_internal.h
@@ -225,6 +225,10 @@ static inline __u8 lmv_inherit_rr_next(__u8 inherit_rr)
        return inherit_rr - 1;
 }
 
+enum layout_verify_flags {
+       LVF_ALL_STALE           = BIT(0), /* check not all stale mirrors */
+};
+
 struct lod_mirror_entry {
        __u16   lme_stale:1,
                lme_prefer:1;
@@ -596,7 +600,7 @@ int lod_fini_tgt(const struct lu_env *env, struct lod_device *lod,
                 struct lod_tgt_descs *ltd);
 int lod_striping_load(const struct lu_env *env, struct lod_object *lo);
 int lod_striping_reload(const struct lu_env *env, struct lod_object *lo,
-                       const struct lu_buf *buf);
+                       const struct lu_buf *buf, enum layout_verify_flags lvf);
 void lod_dom_stripesize_recalc(struct lod_device *d);
 
 int lod_get_ea(const struct lu_env *env, struct lod_object *lo,
@@ -666,7 +670,7 @@ void lod_fix_lmv_desc_pattern(__u32 *val);
 int lod_pools_init(struct lod_device *m, struct lustre_cfg *cfg);
 int lod_pools_fini(struct lod_device *m);
 int lod_parse_striping(const struct lu_env *env, struct lod_object *mo,
-                      const struct lu_buf *buf);
+                      const struct lu_buf *buf, enum layout_verify_flags lvf);
 int lod_parse_dir_striping(const struct lu_env *env, struct lod_object *lo,
                           const struct lu_buf *buf);
 int lod_initialize_objects(const struct lu_env *env, struct lod_object *mo,

diff --git a/lustre/lod/lod_lov.c b/lustre/lod/lod_lov.c
index 3dd8696..47ebff7 100644 (file)
--- a/lustre/lod/lod_lov.c
+++ b/lustre/lod/lod_lov.c
@@ -1143,13 +1143,15 @@ out:
  * \param[in] env              execution environment for this thread
  * \param[in] lo               LOD object
  * \param[in] buf              buffer storing LOV EA to parse
+ * \param[in] lvf              verify flags when parsing the layout
  *
  * \retval                     0 if parsing and objects creation succeed
  * \retval                     negative error number on failure
  */
 int lod_parse_striping(const struct lu_env *env, struct lod_object *lo,
-                      const struct lu_buf *buf)
+                      const struct lu_buf *buf, enum layout_verify_flags lvf)
 {
+       struct lod_device *d = lu2lod_dev(lo->ldo_obj.do_lu.lo_dev);
        struct lov_mds_md_v1 *lmm;
        struct lov_comp_md_v1 *comp_v1 = NULL;
        struct lov_foreign_md *foreign = NULL;
@@ -1158,6 +1160,9 @@ int lod_parse_striping(const struct lu_env *env, struct lod_object *lo,
        __u16 mirror_cnt = 0;
        __u16 comp_cnt;
        int i, rc;
+       __u16 mirror_id = MIRROR_ID_NEG;
+       bool stale = false;
+       int stale_mirrors = 0;
        ENTRY;
 
        LASSERT(buf);
@@ -1252,6 +1257,7 @@ int lod_parse_striping(const struct lu_env *env, struct lod_object *lo,
 
                        lod_comp->llc_flags =
                                le32_to_cpu(comp_v1->lcm_entries[i].lcme_flags);
+
                        if (lod_comp->llc_flags & LCME_FL_NOSYNC)
                                lod_comp->llc_timestamp = le64_to_cpu(
                                        comp_v1->lcm_entries[i].lcme_timestamp);
@@ -1260,11 +1266,31 @@ int lod_parse_striping(const struct lu_env *env, struct lod_object *lo,
                        if (lod_comp->llc_id == LCME_ID_INVAL)
                                GOTO(out, rc = -EINVAL);
 
+                       if (lvf & LVF_ALL_STALE) {
+                               if (mirror_id_of(lod_comp->llc_id) ==
+                                   mirror_id) {
+                                       /* remaining comps in the mirror */
+                                       stale |= lod_comp->llc_flags &
+                                                LCME_FL_STALE;
+                               } else {
+                                       /*
+                                        * new mirror, check last mirror's
+                                        * stale-ness
+                                        */
+                                       if (stale)
+                                               stale_mirrors++;
+
+                                       mirror_id =
+                                               mirror_id_of(lod_comp->llc_id);
+
+                                       /* the first comp of the new mirror */
+                                       stale = lod_comp->llc_flags &
+                                               LCME_FL_STALE;
+                               }
+                       }
+
                        if ((lod_comp->llc_flags & LCME_FL_EXTENSION) &&
                            comp_v1->lcm_magic != cpu_to_le32(LOV_MAGIC_SEL)) {
-                               struct lod_device *d =
-                                       lu2lod_dev(lo->ldo_obj.do_lu.lo_dev);
-
                                CWARN("%s: EXTENSION flags=%x set on component[%u]=%x of non-SEL file "DFID" with magic=%#08x\n",
                                      lod2obd(d)->obd_name,
                                      lod_comp->llc_flags, lod_comp->llc_id, i,
@@ -1355,6 +1381,21 @@ int lod_parse_striping(const struct lu_env *env, struct lod_object *lo,
                }
        }
 
+       if (lo->ldo_is_composite && (lvf & LVF_ALL_STALE)) {
+               /* check the last mirror stale-ness */
+               if (stale)
+                       stale_mirrors++;
+
+               if (mirror_cnt == stale_mirrors) {
+                       rc = -EPERM;
+                       CERROR("%s: can not set all stale mirrors for "
+                              DFID": rc = %d\n",
+                              lod2obd(d)->obd_name, PFID(lod_object_fid(lo)),
+                              rc);
+                       GOTO(out, rc);
+               }
+       }
+
        rc = lod_fill_mirrors(lo);
        if (rc)
                GOTO(out, rc);
@@ -1438,7 +1479,7 @@ int lod_striping_load(const struct lu_env *env, struct lod_object *lo)
                 */
                buf->lb_buf = info->lti_ea_store;
                buf->lb_len = info->lti_ea_store_size;
-               rc = lod_parse_striping(env, lo, buf);
+               rc = lod_parse_striping(env, lo, buf, 0);
                if (rc == 0)
                        lo->ldo_comp_cached = 1;
        } else if (S_ISDIR(lod2lu_obj(lo)->lo_header->loh_attr)) {
@@ -1499,14 +1540,14 @@ unlock:
 }
 
 int lod_striping_reload(const struct lu_env *env, struct lod_object *lo,
-                        const struct lu_buf *buf)
+                       const struct lu_buf *buf, enum layout_verify_flags lvf)
 {
        int rc;
 
        ENTRY;
 
        mutex_lock(&lo->ldo_layout_mutex);
-       rc = lod_parse_striping(env, lo, buf);
+       rc = lod_parse_striping(env, lo, buf, lvf);
        mutex_unlock(&lo->ldo_layout_mutex);
 
        RETURN(rc);

diff --git a/lustre/lod/lod_object.c b/lustre/lod/lod_object.c
index e4147b8..bf4a5bd 100644 (file)
--- a/lustre/lod/lod_object.c
+++ b/lustre/lod/lod_object.c
@@ -3417,7 +3417,7 @@ static int lod_declare_layout_merge(const struct lu_env *env,
        if ((le16_to_cpu(lcm->lcm_flags) & LCM_FL_FLR_MASK) == LCM_FL_NONE)
                lcm->lcm_flags = cpu_to_le32(LCM_FL_RDONLY);
 
-       rc = lod_striping_reload(env, lo, buf);
+       rc = lod_striping_reload(env, lo, buf, 0);
        if (rc)
                GOTO(out, rc);
 
@@ -3460,7 +3460,7 @@ static int lod_declare_layout_split(const struct lu_env *env,
        int rc;
        ENTRY;
 
-       rc = lod_striping_reload(env, lo, mbuf);
+       rc = lod_striping_reload(env, lo, mbuf, LVF_ALL_STALE);
        if (rc)
                RETURN(rc);
 
@@ -4879,7 +4879,7 @@ static int lod_xattr_set(const struct lu_env *env,
                        if (rc)
                                RETURN(rc);
 
-                       rc = lod_striping_reload(env, lo, buf);
+                       rc = lod_striping_reload(env, lo, buf, LVF_ALL_STALE);
                        if (rc)
                                RETURN(rc);
 

diff --git a/lustre/tests/sanity-flr.sh b/lustre/tests/sanity-flr.sh
index 803617d..fd811a2 100644 (file)
--- a/lustre/tests/sanity-flr.sh
+++ b/lustre/tests/sanity-flr.sh
@@ -2000,7 +2000,7 @@ test_43b() {
 }
 run_test 43b "allow writing to multiple preferred mirror file"
 
-test_44() {
+test_44a() {
        [ $MDSCOUNT -lt 2 ] && skip "needs >= 2 MDTs" && return
        rm -rf $DIR/$tdir
        rm -rf $DIR/$tdir-1
@@ -2075,7 +2075,62 @@ test_44() {
        diff $tf $tf.mirror~2 ||
                error "splited file $tf.mirror~2 diffs from $tf"
 }
-run_test 44 "lfs mirror split check"
+run_test 44a "lfs mirror split check"
+
+test_44b() {
+       rm -rf $DIR/$tdir
+       local tf=$DIR/$tdir/$tfile
+
+       mkdir -p $DIR/$tdir || error "create directory failed"
+
+       echo XXX > $tf
+
+       # create 2 mirrors file
+       $LFS mirror extend -N -c1 $tf
+
+       echo YYY > $tf
+
+       verify_flr_state $tf "wp"
+
+       local str=$(cat $tf)
+
+       [[ $str == "YYY" ]] || error "$tf content is not YYY"
+
+       # get the non-stale mirror id
+       local ids=($($LFS getstripe $tf | awk '/lcme_id/{print $2}' |
+                       tr '\n' ' '))
+       local mirror_ids=($($LFS getstripe $tf |
+                       awk '/lcme_mirror_id/{print $2}' | tr '\n' ' '))
+       for ((i = 0; i < 2; i++)); do
+               $LFS getstripe -I${ids[$i]} --component-flags $tf |
+                       grep stale > /dev/null || break
+       done
+
+       [[ $i -ge 2 ]] && ( $LFS getstripe $tf; error "no stale mirror" )
+
+       $LFS getstripe $tf
+
+       # split the updated mirror, should fail
+       echo "split mirror_id ${mirror_ids[$i]} id ${ids[$i]}, should fail"
+       $LFS mirror split --mirror-id=${mirror_ids[$i]} $tf &> /dev/null &&
+               error "split --mirror-id=${mirror_ids[$i]} $tf should fail"
+
+       i=$(( 1 - i ))
+       # split the stale mirror
+       echo "split mirror_id ${mirror_ids[$i]} id ${ids[$i]}"
+       $LFS mirror split --mirror-id=${mirror_ids[$i]} -d $tf ||
+               error "mirror split --mirror-id=${mirror_ids[$i]} $tf failed"
+
+       echo "make sure there's no stale comp in the file"
+       # make sure there's no stale comp in the file
+       $LFS getstripe $tf | awk '/lcme_flags/{print $2}' | grep stale &&
+               ( $LFS getstripe $tf; error "stale mirror file" )
+
+       str=$(cat $tf)
+       [[ $str == "YYY" ]] ||
+               ( cat $tf; error "$tf content is not YYY after split" )
+}
+run_test 44b "mirror split does not create stale file"
 
 test_44c() {
        local tf=$DIR/$tdir/$tfile