From b380a1f9a1daa1683593d0ee4a508da859277164 Mon Sep 17 00:00:00 2001 From: Jian Yu Date: Sun, 15 Sep 2019 22:56:33 -0700 Subject: [PATCH] LU-11485 lod: disallow setting the last non-stale mirror as stale "lfs setstripe" allows setting stale flag on the last non-stale mirror of a file, which makes the file have no valid component to read and return IO error. This patch fixes the above issue by disallowing that. It also disallows "lfs mirror split" to destroy the last non-stale mirror of a file. Change-Id: I6934cfe0190cd1ea83de1cf28ddf840b9f96193a Signed-off-by: Jian Yu Reviewed-on: https://review.whamcloud.com/36141 Reviewed-by: Bobi Jam Tested-by: jenkins Reviewed-by: Andreas Dilger Reviewed-by: Patrick Farrell Tested-by: Maloo --- lustre/doc/lfs-mirror-split.1 | 3 +- lustre/doc/lfs-setstripe.1 | 5 +- lustre/lod/lod_object.c | 46 +++++++++++++++- lustre/tests/sanity-flr.sh | 44 ++++++++++++++- lustre/utils/lfs.c | 121 ++++++++++++++++++++++++++++++------------ 5 files changed, 178 insertions(+), 41 deletions(-) diff --git a/lustre/doc/lfs-mirror-split.1 b/lustre/doc/lfs-mirror-split.1 index 34386f9..d5259a9 100644 --- a/lustre/doc/lfs-mirror-split.1 +++ b/lustre/doc/lfs-mirror-split.1 @@ -14,7 +14,8 @@ a component with <\fIcomp_id\fR> or containing a component on <\fIpool\fR> out of a mirrored file specified by the path name \fImirrored_file\fR. By default, the layout of the split mirror will be stored into a new file named <\fImirrored_file\fR>.mirror~<\fImirror_id\fR>. If \fB\-\-destroy\fR|\fB\-d\fR -option is specified, then the split mirror will be destroyed. +option is specified, and the split mirror is not the last non-stale mirror, +then it will be destroyed. If \fB\-f\fR <\fInew_file\fR> option is specified, then the layout of the split mirror will be stored into the named file. .br diff --git a/lustre/doc/lfs-setstripe.1 b/lustre/doc/lfs-setstripe.1 index a4cc7ef..97f74e5 100644 --- a/lustre/doc/lfs-setstripe.1 +++ b/lustre/doc/lfs-setstripe.1 @@ -387,8 +387,9 @@ to find uninitialized components. .B * prefer\fR - component preferred for read/write in a mirrored file .RE .RS -.B * stale\fR - component has outdated data in a mirrored file. Once a -component is marked +.B * stale\fR - component has outdated data in a mirrored file. This flag is +not allowed to be set on a component of the last non-stale mirror. +Once a component is marked .BR stale , it isn't permitted to clear this flag directly. \fBlfs-mirror-resync\fR(1) is required to clear the flag. diff --git a/lustre/lod/lod_object.c b/lustre/lod/lod_object.c index 13a7e31..db457a4 100644 --- a/lustre/lod/lod_object.c +++ b/lustre/lod/lod_object.c @@ -2860,6 +2860,42 @@ error: } /** + * lod_last_non_stale_mirror() - Check if a mirror is the last non-stale mirror. + * @mirror_id: Mirror id to be checked. + * @lo: LOD object. + * + * This function checks if a mirror with specified @mirror_id is the last + * non-stale mirror of a LOD object @lo. + * + * Return: true or false. + */ +static inline +bool lod_last_non_stale_mirror(__u16 mirror_id, struct lod_object *lo) +{ + struct lod_layout_component *lod_comp; + bool has_stale_flag; + int i; + + for (i = 0; i < lo->ldo_mirror_count; i++) { + if (lo->ldo_mirrors[i].lme_id == mirror_id || + lo->ldo_mirrors[i].lme_stale) + continue; + + has_stale_flag = false; + lod_foreach_mirror_comp(lod_comp, lo, i) { + if (lod_comp->llc_flags & LCME_FL_STALE) { + has_stale_flag = true; + break; + } + } + if (!has_stale_flag) + return false; + } + + return true; +} + +/** * Declare component set. The xattr is name XATTR_LUSTRE_LOV.set.$field, * the '$field' can only be 'flags' now. The xattr value is binary * lov_comp_md_v1 which contains the component ID(s) and the value of @@ -2914,6 +2950,7 @@ static int lod_declare_layout_set(const struct lu_env *env, __u32 id = comp_v1->lcm_entries[i].lcme_id; __u32 flags = comp_v1->lcm_entries[i].lcme_flags; __u32 mirror_flag = flags & LCME_MIRROR_FLAGS; + __u16 mirror_id = mirror_id_of(id); bool neg = flags & LCME_FL_NEG; if (flags & LCME_FL_INIT) { @@ -2928,7 +2965,7 @@ static int lod_declare_layout_set(const struct lu_env *env, /* lfs only put one flag in each entry */ if ((flags && id != lod_comp->llc_id) || - (mirror_flag && mirror_id_of(id) != + (mirror_flag && mirror_id != mirror_id_of(lod_comp->llc_id))) continue; @@ -2938,8 +2975,13 @@ static int lod_declare_layout_set(const struct lu_env *env, if (mirror_flag) lod_comp->llc_flags &= ~mirror_flag; } else { - if (flags) + if (flags) { + if ((flags & LCME_FL_STALE) && + lod_last_non_stale_mirror(mirror_id, + lo)) + RETURN(-EUCLEAN); lod_comp->llc_flags |= flags; + } if (mirror_flag) { lod_comp->llc_flags |= mirror_flag; if (mirror_flag & LCME_FL_NOSYNC) diff --git a/lustre/tests/sanity-flr.sh b/lustre/tests/sanity-flr.sh index 9b89264..8a93e26 100644 --- a/lustre/tests/sanity-flr.sh +++ b/lustre/tests/sanity-flr.sh @@ -720,6 +720,35 @@ test_0h() { error "error setting flag prefer" verify_comp_attr lcme_flags $tf 0x20003 prefer + + $LFS setstripe --comp-set -I 0x20003 --comp-flags=^prefer $tf || + error "error clearing prefer flag from component 0x20003" + + # MDS disallows setting stale flag on the last non-stale mirror + [[ "$MDS1_VERSION" -ge $(version_code 2.12.57) ]] || return 0 + + cp /etc/hosts $tf || error "error writing file '$tf'" + + verify_comp_attr lcme_flags $tf 0x10002 prefer + verify_comp_attr lcme_flags $tf 0x20003 stale + verify_comp_attr lcme_flags $tf 0x30004 stale + + ! $LFS setstripe --comp-set -I 0x10002 --comp-flags=^prefer,stale $tf \ + > /dev/null 2>&1 || + error "setting stale flag on component 0x10002 should fail" + + $LFS mirror resync $tf || error "error resync-ing file '$tf'" + + $LFS setstripe --comp-set -I 0x10001 --comp-flags=stale $tf || + error "error setting stale flag on component 0x10001" + $LFS setstripe --comp-set -I 0x20003 --comp-flags=stale $tf || + error "error setting stale flag on component 0x20003" + + ! $LFS setstripe --comp-set -I 0x30004 --comp-flags=stale $tf \ + > /dev/null 2>&1 || + error "setting stale flag on component 0x30004 should fail" + + $LFS mirror resync $tf || error "error resync-ing file '$tf'" } run_test 0h "set, clear and test flags for FLR files" @@ -1782,6 +1811,10 @@ test_44() { verify_flr_state $tf "wp" + # disallow destroying the last non-stale mirror + ! $LFS mirror split --mirror-id 1 -d $tf > /dev/null 2>&1 || + error "destroying mirror 1 should fail" + # synchronize all mirrors of the file $LFS mirror resync $tf || error "mirror resync $tf failed" @@ -1807,8 +1840,17 @@ test_44() { verify_mirror_count $tf 2 verify_mirror_count $tf.mirror~2 1 + $LFS setstripe --comp-set -I 0x30008 --comp-flags=stale $tf || + error "setting stale flag on component 0x30008 failed" + + # disallow destroying the last non-stale mirror + ! $LFS mirror split --mirror-id 4 -d $tf > /dev/null 2>&1 || + error "destroying mirror 4 should fail" + + $LFS mirror resync $tf || error "resynchronizing $tf failed" + $LFS mirror split --mirror-id 3 -d $tf || - error "split and delte mirror 3 failed" + error "destroying mirror 3 failed" verify_mirror_count $tf 1 # verify splitted file contains the same content as the orig file does diff --git a/lustre/utils/lfs.c b/lustre/utils/lfs.c index e5ecd75..a701711 100644 --- a/lustre/utils/lfs.c +++ b/lustre/utils/lfs.c @@ -1105,10 +1105,18 @@ static int lfs_component_set(char *fname, int comp_id, } rc = llapi_layout_file_comp_set(fname, ids, flags_array, count); - if (rc) - fprintf(stderr, - "%s: cannot change the flags of component '%#x' of file '%s': %x / ^(%x)\n", - progname, comp_id, fname, flags, neg_flags); + if (rc) { + if (errno == EUCLEAN) { + rc = -errno; + fprintf(stderr, + "%s: cannot set 'stale' flag on component '%#x' of the last non-stale mirror of '%s'\n", + progname, comp_id, fname); + } else { + fprintf(stderr, + "%s: cannot change the flags of component '%#x' of file '%s': %x / ^(%x)\n", + progname, comp_id, fname, flags, neg_flags); + } + } return rc; } @@ -1858,6 +1866,72 @@ static int find_comp_id_by_pool(struct llapi_layout *layout, void *cbdata) return LLAPI_LAYOUT_ITER_STOP; } +struct collect_ids_data { + __u16 *cid_ids; + int cid_count; + __u16 cid_exclude; +}; + +static int collect_mirror_id(struct llapi_layout *layout, void *cbdata) +{ + struct collect_ids_data *cid = cbdata; + uint32_t id; + int rc; + + rc = llapi_layout_mirror_id_get(layout, &id); + if (rc < 0) + return rc; + + if ((__u16)id != cid->cid_exclude) { + int i; + + for (i = 0; i < cid->cid_count; i++) { + /* already collected the mirror id */ + if (id == cid->cid_ids[i]) + return LLAPI_LAYOUT_ITER_CONT; + } + cid->cid_ids[cid->cid_count] = id; + cid->cid_count++; + } + + return LLAPI_LAYOUT_ITER_CONT; +} + +/** + * last_non_stale_mirror() - Check if a mirror is the last non-stale mirror. + * @mirror_id: Mirror id to be checked. + * @layout: Mirror component list. + * + * This function checks if a mirror with specified @mirror_id is the last + * non-stale mirror of a layout @layout. + * + * Return: true or false. + */ +static inline +bool last_non_stale_mirror(__u16 mirror_id, struct llapi_layout *layout) +{ + __u16 mirror_ids[128] = { 0 }; + struct collect_ids_data cid = { .cid_ids = mirror_ids, + .cid_count = 0, + .cid_exclude = mirror_id, }; + int i; + + llapi_layout_comp_iterate(layout, collect_mirror_id, &cid); + + for (i = 0; i < cid.cid_count; i++) { + struct llapi_resync_comp comp_array[1024] = { { 0 } }; + int comp_size = 0; + + comp_size = llapi_mirror_find_stale(layout, comp_array, + ARRAY_SIZE(comp_array), + &mirror_ids[i], 1); + if (comp_size == 0) + return false; + } + + return true; +} + static int mirror_split(const char *fname, __u32 id, const char *pool, enum mirror_flags mflags, const char *victim_file) { @@ -1964,6 +2038,14 @@ static int mirror_split(const char *fname, __u32 id, const char *pool, if (victim_file == NULL) { /* use a temp file to store the splitted layout */ if (mflags & MF_DESTROY) { + if (last_non_stale_mirror(id, layout)) { + rc = -EUCLEAN; + fprintf(stderr, + "%s: cannot destroy the last non-stale mirror of file '%s'\n", + progname, fname); + goto close_fd; + } + fdv = llapi_create_volatile_idx(parent, mdt_index, O_LOV_DELAY_CREATE); } else { @@ -9706,37 +9788,6 @@ close_fd: return rc; } -struct collect_ids_data { - __u16 *cid_ids; - int cid_count; - __u16 cid_exclude; -}; - -static int collect_mirror_id(struct llapi_layout *layout, void *cbdata) -{ - struct collect_ids_data *cid = cbdata; - uint32_t id; - int rc; - - rc = llapi_layout_mirror_id_get(layout, &id); - if (rc < 0) - return rc; - - if ((__u16)id != cid->cid_exclude) { - int i; - - for (i = 0; i < cid->cid_count; i++) { - /* already collected the mirror id */ - if (id == cid->cid_ids[i]) - return LLAPI_LAYOUT_ITER_CONT; - } - cid->cid_ids[cid->cid_count] = id; - cid->cid_count++; - } - - return LLAPI_LAYOUT_ITER_CONT; -} - static inline int get_other_mirror_ids(int fd, __u16 *ids, __u16 exclude_id) { struct llapi_layout *layout; -- 1.8.3.1