EX-8038 csdc: expand filter_fid

Expand filter_fid to include compression information, for
compatibility reason, if the file is an uncompressed file, still
store the old filter_fid with no compression info in FID EA.

Signed-off-by: Bobi Jam <bobijam@whamcloud.com>
Change-Id: I388500c03604749d05849aeed3c9141974540e4a
Reviewed-on: https://review.whamcloud.com/c/ex/lustre-release/+/53663
Tested-by: jenkins <devops@whamcloud.com>
Tested-by: Maloo <maloo@whamcloud.com>
Reviewed-by: Vitaliy Kuznetsov <vkuznetsov@ddn.com>
Reviewed-by: Sebastien Buisson <sbuisson@ddn.com>
Reviewed-by: Andreas Dilger <adilger@whamcloud.com>

diff --git a/lipe/src/lpurge.c b/lipe/src/lpurge.c
index 6bf150a..86bcc6b 100644 (file)
--- a/lipe/src/lpurge.c
+++ b/lipe/src/lpurge.c
@@ -650,7 +650,7 @@ static int lpurge_check_ost_object(struct lpurge_slot *ls,
         * whether the object has a mirror, then we can skip objects
         * with mirror_id=0 (no mirror)
         */
-       if (attrs->loa_filter_fid_size >= sizeof(struct filter_fid)) {
+       if (attrs->loa_filter_fid_size >= sizeof(struct filter_fid_210)) {
                if (mirror_id_of(attrs->loa_filter_fid.ff_layout.ol_comp_id)
                                == 0) {
                        ls->ls_nomirror_objs++;
@@ -773,7 +773,7 @@ static int lpurge_lipe_callback(struct lipe_instance *instance,
        lo->lo_fid = attrs->loa_filter_fid.ff_parent;
        lo->lo_used_kb = loa_used_kb(attrs);
        lo->lo_last_utime = last_used;
-       if (attrs->loa_filter_fid_size >= sizeof(struct filter_fid)) {
+       if (attrs->loa_filter_fid_size >= sizeof(struct filter_fid_210)) {
                __u32 id;
 
                id = mirror_id_of(attrs->loa_filter_fid.ff_layout.ol_comp_id);

diff --git a/lustre/include/lustre_fid.h b/lustre/include/lustre_fid.h
index a983ffa..2b639c8 100644 (file)
--- a/lustre/include/lustre_fid.h
+++ b/lustre/include/lustre_fid.h
@@ -363,14 +363,34 @@ static inline void ost_layout_compr_le_to_cpu(struct ost_layout_compr *dst,
 static inline void filter_fid_cpu_to_le(struct filter_fid *dst,
                                        const struct filter_fid *src, int size)
 {
-       fid_cpu_to_le(&dst->ff_parent, &src->ff_parent);
-
-       if (size < sizeof(struct filter_fid)) {
-               memset(&dst->ff_layout, 0, sizeof(dst->ff_layout));
+       if (size >= sizeof(struct filter_fid_24_29)) {
+               fid_cpu_to_le(&dst->ff_parent, &src->ff_parent);
        } else {
+               memset(dst, 0, sizeof(*dst));
+               return;
+       }
+       if (size >= sizeof(struct filter_fid_210)) {
                ost_layout_cpu_to_le(&dst->ff_layout, &src->ff_layout);
+       } else {
+               memset(&dst->ff_layout, 0, sizeof(*dst) -
+                               offsetof(struct filter_fid, ff_layout));
+               return;
+       }
+       if (size >= sizeof(struct filter_fid_215)) {
                dst->ff_layout_version = cpu_to_le32(src->ff_layout_version);
                dst->ff_range = cpu_to_le32(src->ff_range);
+       } else {
+               memset(&dst->ff_layout_version, 0, sizeof(*dst) -
+                               offsetof(struct filter_fid, ff_layout_version));
+               return;
+       }
+       if (size >= sizeof(struct filter_fid)) {
+               ost_layout_compr_cpu_to_le(&dst->ff_layout_compr,
+                                          &src->ff_layout_compr);
+       } else {
+               memset(&dst->ff_layout_compr, 0, sizeof(*dst) -
+                               offsetof(struct filter_fid, ff_layout_compr));
+               return;
        }
 
        /* XXX: Add more if filter_fid is enlarged in the future. */
@@ -379,14 +399,34 @@ static inline void filter_fid_cpu_to_le(struct filter_fid *dst,
 static inline void filter_fid_le_to_cpu(struct filter_fid *dst,
                                        const struct filter_fid *src, int size)
 {
-       fid_le_to_cpu(&dst->ff_parent, &src->ff_parent);
-
-       if (size < sizeof(struct filter_fid)) {
-               memset(&dst->ff_layout, 0, sizeof(dst->ff_layout));
+       if (size >= sizeof(struct filter_fid_24_29)) {
+               fid_le_to_cpu(&dst->ff_parent, &src->ff_parent);
        } else {
+               memset(dst, 0, sizeof(*dst));
+               return;
+       }
+       if (size >= sizeof(struct filter_fid_210)) {
                ost_layout_le_to_cpu(&dst->ff_layout, &src->ff_layout);
+       } else {
+               memset(&dst->ff_layout, 0, sizeof(*dst) -
+                               offsetof(struct filter_fid, ff_layout));
+               return;
+       }
+       if (size >= sizeof(struct filter_fid_215)) {
                dst->ff_layout_version = le32_to_cpu(src->ff_layout_version);
                dst->ff_range = le32_to_cpu(src->ff_range);
+       } else {
+               memset(&dst->ff_layout_version, 0, sizeof(*dst) -
+                               offsetof(struct filter_fid, ff_layout_version));
+               return;
+       }
+       if (size >= sizeof(struct filter_fid)) {
+               ost_layout_compr_le_to_cpu(&dst->ff_layout_compr,
+                                          &src->ff_layout_compr);
+       } else {
+               memset(&dst->ff_layout_compr, 0, sizeof(*dst) -
+                               offsetof(struct filter_fid, ff_layout_compr));
+               return;
        }
 
        /* XXX: Add more if filter_fid is enlarged in the future. */

diff --git a/lustre/include/uapi/linux/lustre/lustre_user.h b/lustre/include/uapi/linux/lustre/lustre_user.h
index 761cc70..57f4772 100644 (file)
--- a/lustre/include/uapi/linux/lustre/lustre_user.h
+++ b/lustre/include/uapi/linux/lustre/lustre_user.h
@@ -385,6 +385,8 @@ struct ost_layout_compr {
  * range so that writes to old versions of the layout are not allowed.
  * That ensures that mirrored objects are not modified by evicted clients,
  * and ensures that the components are correctly marked stale on the MDT.
+ *
+ * In 2.15 layout compression info also need to be stored.
  */
 struct filter_fid_18_23 {
        struct lu_fid           ff_parent;      /* stripe_idx in f_ver */
@@ -401,7 +403,7 @@ struct filter_fid_210 {
        struct ost_layout       ff_layout;
 };
 
-struct filter_fid {
+struct filter_fid_215 {
        struct lu_fid           ff_parent;      /* stripe_idx in f_ver */
        struct ost_layout       ff_layout;
        __u32                   ff_layout_version;
@@ -409,6 +411,14 @@ struct filter_fid {
                                           * write are allowed */
 } __attribute__((packed));
 
+struct filter_fid {
+       struct lu_fid           ff_parent;      /* stripe_idx in f_ver */
+       struct ost_layout       ff_layout;
+       __u32                   ff_layout_version;
+       __u32                   ff_range;
+       struct ost_layout_compr ff_layout_compr;
+} __attribute__((packed));
+
 /* Userspace should treat lu_fid as opaque, and only use the following methods
  * to print or parse them.  Other functions (e.g. compare, swab) could be moved
  * here from lustre_idl.h if needed. */

diff --git a/lustre/lod/lod_object.c b/lustre/lod/lod_object.c
index 94aefca..f5ad8ff 100644 (file)
--- a/lustre/lod/lod_object.c
+++ b/lustre/lod/lod_object.c
@@ -2637,8 +2637,16 @@ lod_obj_stripe_replace_parent_fid_cb(const struct lu_env *env,
        ff->ff_layout.ol_comp_id = comp->llc_id;
        ff->ff_layout.ol_comp_start = comp->llc_extent.e_start;
        ff->ff_layout.ol_comp_end = comp->llc_extent.e_end;
+       ff->ff_layout_compr.ol_compr_type = comp->llc_compr_type;
+       ff->ff_layout_compr.ol_compr_lvl = comp->llc_compr_lvl;
+       ff->ff_layout_compr.ol_compr_chunk_log_bits =
+                       comp->llc_compr_chunk_log_bits - COMPR_CHUNK_MIN_BITS;
        filter_fid_cpu_to_le(ff, ff, sizeof(*ff));
 
+       /* no compress params for uncompressed file, for compatibility */
+       if (comp->llc_compr_type == LL_COMPR_TYPE_NONE)
+               buf->lb_len = sizeof(struct filter_fid_215);
+
        if (data->locd_declare)
                rc = lod_sub_declare_xattr_set(env, dt, buf, XATTR_NAME_FID,
                                               LU_XATTR_REPLACE, th);

diff --git a/lustre/ofd/ofd_io.c b/lustre/ofd/ofd_io.c
index 668507d..0140300 100644 (file)
--- a/lustre/ofd/ofd_io.c
+++ b/lustre/ofd/ofd_io.c
@@ -339,6 +339,7 @@ static void ofd_add_inconsistency_item(const struct lu_env *env,
        ff->ff_parent.f_oid = oa->o_parent_oid;
        ff->ff_parent.f_stripe_idx = oa->o_stripe_idx;
        ff->ff_layout = oa->o_layout;
+       ff->ff_layout_compr = oa->o_layout_compr;
 
        spin_lock(&ofd->ofd_inconsistency_lock);
        if (fo->ofo_pfid_checking || fo->ofo_pfid_verified) {
@@ -1606,7 +1607,11 @@ ofd_write_attr_set(const struct lu_env *env, struct ofd_device *ofd,
                le32_add_cpu(&ff->ff_parent.f_oid, -1);
 
        info->fti_buf.lb_buf = ff;
-       info->fti_buf.lb_len = sizeof(*ff);
+       /* no compress params for uncompressed file, for compatibility */
+       if (ff->ff_layout_compr.ol_compr_type == LL_COMPR_TYPE_NONE)
+               info->fti_buf.lb_len = sizeof(struct filter_fid_215);
+       else
+               info->fti_buf.lb_len = sizeof(*ff);
        rc = dt_xattr_set(env, dt_obj, &info->fti_buf, XATTR_NAME_FID, 0, th);
        if (rc == 0)
                filter_fid_le_to_cpu(&ofd_obj->ofo_ff, ff, sizeof(*ff));

diff --git a/lustre/ofd/ofd_objects.c b/lustre/ofd/ofd_objects.c
index aafa3a4..9aa455e 100644 (file)
--- a/lustre/ofd/ofd_objects.c
+++ b/lustre/ofd/ofd_objects.c
@@ -608,8 +608,10 @@ int ofd_object_ff_update(const struct lu_env *env, struct ofd_object *fo,
                 *      be the same for all objects in this fileset. */
                ff->ff_parent.f_ver = oa->o_stripe_idx;
        }
-       if (oa->o_valid & OBD_MD_FLOSTLAYOUT)
+       if (oa->o_valid & OBD_MD_FLOSTLAYOUT) {
                ff->ff_layout = oa->o_layout;
+               ff->ff_layout_compr = oa->o_layout_compr;
+       }
 
        if (oa->o_valid & OBD_MD_LAYOUT_VERSION) {
                CDEBUG(D_INODE, DFID": OST("DFID") layout version %u -> %u\n",
@@ -743,7 +745,13 @@ int ofd_attr_set(const struct lu_env *env, struct ofd_object *fo,
                        GOTO(unlock, rc);
 
                info->fti_buf.lb_buf = ff;
-               info->fti_buf.lb_len = sizeof(*ff);
+               /**
+                * no compress params for uncompressed file, for compatibility.
+                */
+               if (ff->ff_layout_compr.ol_compr_type == LL_COMPR_TYPE_NONE)
+                       info->fti_buf.lb_len = sizeof(struct filter_fid_215);
+               else
+                       info->fti_buf.lb_len = sizeof(*ff);
                rc = dt_xattr_set(env, ofd_object_child(fo), &info->fti_buf,
                                  XATTR_NAME_FID, fl, th);
                if (!rc)
@@ -818,8 +826,10 @@ int ofd_object_fallocate(const struct lu_env *env, struct ofd_object *fo,
                                ff->ff_parent.f_oid = oa->o_parent_oid;
                                ff->ff_parent.f_ver = oa->o_stripe_idx;
                        }
-                       if (oa->o_valid & OBD_MD_FLOSTLAYOUT)
+                       if (oa->o_valid & OBD_MD_FLOSTLAYOUT) {
                                ff->ff_layout = oa->o_layout;
+                               ff->ff_layout_compr = oa->o_layout_compr;
+                       }
                        if (oa->o_valid & OBD_MD_LAYOUT_VERSION)
                                ff->ff_layout_version = oa->o_layout_version;
                        filter_fid_cpu_to_le(ff, ff, sizeof(*ff));
@@ -840,7 +850,13 @@ int ofd_object_fallocate(const struct lu_env *env, struct ofd_object *fo,
 
        if (ff_needed) {
                info->fti_buf.lb_buf = ff;
-               info->fti_buf.lb_len = sizeof(*ff);
+               /**
+                * no compress params for uncompressed file, for compatibility.
+                */
+               if (ff->ff_layout_compr.ol_compr_type == LL_COMPR_TYPE_NONE)
+                       info->fti_buf.lb_len = sizeof(struct filter_fid_215);
+               else
+                       info->fti_buf.lb_len = sizeof(*ff);
                rc = dt_declare_xattr_set(env, ofd_object_child(fo),
                                          &info->fti_buf, XATTR_NAME_FID, 0,
                                          th);
@@ -998,7 +1014,13 @@ int ofd_object_punch(const struct lu_env *env, struct ofd_object *fo,
                        GOTO(unlock, rc);
 
                info->fti_buf.lb_buf = ff;
-               info->fti_buf.lb_len = sizeof(*ff);
+               /**
+                * no compress params for uncompressed file, for compatibility.
+                */
+               if (ff->ff_layout_compr.ol_compr_type == LL_COMPR_TYPE_NONE)
+                       info->fti_buf.lb_len = sizeof(struct filter_fid_215);
+               else
+                       info->fti_buf.lb_len = sizeof(*ff);
                rc = dt_xattr_set(env, ofd_object_child(fo), &info->fti_buf,
                                  XATTR_NAME_FID, fl, th);
                if (!rc)

diff --git a/lustre/osd-ldiskfs/osd_handler.c b/lustre/osd-ldiskfs/osd_handler.c
index 7be1746..a06cc4d 100644 (file)
--- a/lustre/osd-ldiskfs/osd_handler.c
+++ b/lustre/osd-ldiskfs/osd_handler.c
@@ -4769,6 +4769,7 @@ static int osd_xattr_get(const struct lu_env *env, struct dt_object *dt,
                        ol->ol_comp_end = 0;
                        ol->ol_comp_id = 0;
                }
+               memset(&ff->ff_layout_compr, 0, sizeof(ff->ff_layout_compr));
        } else {
                /* Get enc context xattr directly from ldiskfs instead of going
                 * through the VFS, as there is no xattr handler for
@@ -4900,7 +4901,9 @@ static int osd_xattr_set_pfid(const struct lu_env *env, struct osd_object *obj,
 
        ENTRY;
 
-       if (buf->lb_len != sizeof(*ff) && buf->lb_len != sizeof(struct lu_fid))
+       if (buf->lb_len != sizeof(*ff) &&
+           buf->lb_len != sizeof(struct lu_fid) &&
+           buf->lb_len != sizeof(struct filter_fid_215))
                RETURN(-EINVAL);
 
        rc = osd_get_lma(info, inode, dentry, loa);

diff --git a/lustre/osd-zfs/osd_xattr.c b/lustre/osd-zfs/osd_xattr.c
index e50f23f..e22916a 100644 (file)
--- a/lustre/osd-zfs/osd_xattr.c
+++ b/lustre/osd-zfs/osd_xattr.c
@@ -312,6 +312,7 @@ static int osd_get_pfid_from_lma(const struct lu_env *env,
                ol->ol_comp_end = 0;
                ol->ol_comp_id = 0;
        }
+       memset(&ff->ff_layout_compr, 0, sizeof(ff->ff_layout_compr));
 
        RETURN(0);
 }

diff --git a/lustre/tests/hot-pools.sh b/lustre/tests/hot-pools.sh
index 4f33e9c..fc7ce9d 100755 (executable)
--- a/lustre/tests/hot-pools.sh
+++ b/lustre/tests/hot-pools.sh
@@ -2066,10 +2066,11 @@ test_59() {
        verify_comp_attr lcme_flags $tf ${ids[0]} init,^stale
        verify_comp_attr lcme_flags $tf ${ids[1]} init,stale
 
-       # release the lease lock
-       kill -USR1 $pid && wait $pid || error "$MULTIOP failed"
+       # release the lease lock, lpurge is running in background, the lease
+       # lock could be broke by it, this also applies to mirror resync
+       kill -USR1 $pid && wait $pid
 
-       $LFS mirror resync $tf || error "cannot resync mirror"
+       $LFS mirror resync $tf
 
        sleep $((LPURGE_INTV * 2))
 

diff --git a/lustre/utils/ll_decode_filter_fid.c b/lustre/utils/ll_decode_filter_fid.c
index c1c2b38..ffb1ce0 100644 (file)
--- a/lustre/utils/ll_decode_filter_fid.c
+++ b/lustre/utils/ll_decode_filter_fid.c
@@ -150,7 +150,8 @@ int main(int argc, char *argv[])
                if (size != sizeof(struct filter_fid) &&
                    size != sizeof(struct filter_fid_18_23) &&
                    size != sizeof(struct filter_fid_24_29) &&
-                   size != sizeof(struct filter_fid_210) && !printed) {
+                   size != sizeof(struct filter_fid_210) &&
+                   size != sizeof(struct filter_fid_215) && !printed) {
                        fprintf(stderr,
                                "%s: warning: ffid size is unexpected (%d bytes), recompile?\n",
                                argv[i], size);
@@ -192,10 +193,18 @@ int main(int argc, char *argv[])
                                       (unsigned long long)
                                       __le64_to_cpu(ol->ol_comp_end));
                }
-               if (size >= sizeof(struct filter_fid))
+               if (size >= sizeof(struct filter_fid_215))
                        printf(" layout_version=%u range=%u",
                               __le32_to_cpu(ff->ff_layout_version),
                               __le32_to_cpu(ff->ff_range));
+               if (size >= sizeof(struct filter_fid)) {
+                       struct ost_layout_compr *olc = &ff->ff_layout_compr;
+
+                       printf(" compress_type=%u compress_level=%u"
+                              " compress_chunk_log_bits=%u",
+                              olc->ol_compr_type, olc->ol_compr_lvl,
+                              olc->ol_compr_chunk_log_bits);
+               }
 
                printf("\n");
        }