LU-10026 csdc: reserve layout bits for compress component

Add layout bits for compress component layout.

* lcme_compr_type: compression type (gzip, lz4, lz4hc, lzo, etc.)
* lcme_compr_lvl: compression level (0=default, 1-15)
* lcme_compr_chunk_log_bits: chunk size = 2^(16+chunk_log_bits)

Component pattern:
* LOV_PATTERN_COMPRESS - file contains compressed data chunks and
       cannot be read by a client without decompression support.

Compress component flags:
* LCME_FL_COMPRESS - the component should be compressed with the
       compression algorithm stored in lcme_comp_type, at level
       lcme_comp_level, with chunk size 2^(16+lcme_chunk_log_bits)
* LCME_FL_PARTIAL - the component holds some uncompressed chunks due
       to poor write size/alignment, and may benefit from being
       recompressed as the full file data is available
* LCME_FL_NOCOMPR - the component should not be compressed because
       the data was found to be incompressible, or by user request

Signed-off-by: Bobi Jam <bobijam@whamcloud.com>
Change-Id: Idca22cca87b01bba8a5b3c85ca62044abe1d30eb
Reviewed-on: https://review.whamcloud.com/c/fs/lustre-release/+/49170
Reviewed-by: Artem Blagodarenko <ablagodarenko@ddn.com>
Reviewed-by: Oleg Drokin <green@whamcloud.com>
Reviewed-by: Andreas Dilger <adilger@whamcloud.com>
Tested-by: jenkins <devops@whamcloud.com>
Tested-by: Maloo <maloo@whamcloud.com>

diff --git a/lustre/include/lustre/lustreapi.h b/lustre/include/lustre/lustreapi.h
index e257eb4..1e486fd 100644 (file)
--- a/lustre/include/lustre/lustreapi.h
+++ b/lustre/include/lustre/lustreapi.h
@@ -1124,6 +1124,9 @@ static const struct comp_flag_name {
        { LCME_FL_NOSYNC,       "nosync" },
        { LCME_FL_EXTENSION,    "extension" },
        { LCME_FL_PARITY,       "parity" },
+       { LCME_FL_COMPRESS,     "compress" },
+       { LCME_FL_PARTIAL,      "partial" },
+       { LCME_FL_NOCOMPR,      "nocompr" },
 };
 
 /* HSM component flags table */

diff --git a/lustre/include/uapi/linux/lustre/lustre_user.h b/lustre/include/uapi/linux/lustre/lustre_user.h
index dc1af4e..70949e8 100644 (file)
--- a/lustre/include/uapi/linux/lustre/lustre_user.h
+++ b/lustre/include/uapi/linux/lustre/lustre_user.h
@@ -724,6 +724,7 @@ struct fsxattr {
 #define LOV_PATTERN_MDT                        0x100
 #define LOV_PATTERN_OVERSTRIPING       0x200
 #define LOV_PATTERN_FOREIGN            0x400
+#define LOV_PATTERN_COMPRESS           0x800
 
 #define LOV_PATTERN_F_MASK     0xffff0000
 #define LOV_PATTERN_F_HOLE     0x40000000 /* there is hole in LOV EA */
@@ -893,6 +894,13 @@ enum lov_comp_md_entry_flags {
        LCME_FL_NOSYNC    = 0x00000020, /* FLR: no sync for the mirror */
        LCME_FL_EXTENSION = 0x00000040, /* extension comp, never init */
        LCME_FL_PARITY    = 0x00000080, /* EC: a parity code component */
+       LCME_FL_COMPRESS  = 0x00000100, /* the component should be compressed */
+       LCME_FL_PARTIAL   = 0x00000200, /* some chunks in the component are
+                                        * uncompressed
+                                        */
+       LCME_FL_NOCOMPR   = 0x00000400, /* the component should not be
+                                        * compressed
+                                        */
        LCME_FL_NEG       = 0x80000000  /* used to indicate a negative flag,
                                         * won't be stored on disk
                                         */
@@ -963,7 +971,11 @@ struct lov_comp_md_entry_v1 {
        __u8                    lcme_cstripe_count;     /* code stripe count,
                                                         * p value in EC
                                                         */
-       __u16                   lcme_padding_1;
+       __u8                    lcme_compr_type;        /* compress type */
+       __u8                    lcme_compr_lvl:4;       /* compress level */
+       __u8                    lcme_compr_chunk_log_bits:4;
+                                    /* chunk_size = 2^(16+chunk_log_bits)
+                                     * i.e. power-of-two multiple of 64KiB */
 } __attribute__((packed));
 
 #define SEQ_ID_MAX             0x0000FFFF

diff --git a/lustre/ptlrpc/pack_generic.c b/lustre/ptlrpc/pack_generic.c
index 8c3f839..20a742a 100644 (file)
--- a/lustre/ptlrpc/pack_generic.c
+++ b/lustre/ptlrpc/pack_generic.c
@@ -2522,7 +2522,6 @@ void lustre_swab_lov_comp_md_v1(struct lov_comp_md_v1 *lum)
                __swab32s(&ent->lcme_layout_gen);
                /* no need to swab lcme_dstripe_count */
                /* no need to swab lcme_cstripe_count */
-               BUILD_BUG_ON(offsetof(typeof(*ent), lcme_padding_1) == 0);
 
                v1 = (struct lov_user_md_v1 *)((char *)lum + off);
                stripe_count = v1->lmm_stripe_count;

diff --git a/lustre/ptlrpc/wiretest.c b/lustre/ptlrpc/wiretest.c
index 1b5a59d..63ff323 100644 (file)
--- a/lustre/ptlrpc/wiretest.c
+++ b/lustre/ptlrpc/wiretest.c
@@ -1844,10 +1844,10 @@ void lustre_assert_wire_constants(void)
                 (long long)(int)offsetof(struct lov_comp_md_entry_v1, lcme_cstripe_count));
        LASSERTF((int)sizeof(((struct lov_comp_md_entry_v1 *)0)->lcme_cstripe_count) == 1, "found %lld\n",
                 (long long)(int)sizeof(((struct lov_comp_md_entry_v1 *)0)->lcme_cstripe_count));
-       LASSERTF((int)offsetof(struct lov_comp_md_entry_v1, lcme_padding_1) == 46, "found %lld\n",
-                (long long)(int)offsetof(struct lov_comp_md_entry_v1, lcme_padding_1));
-       LASSERTF((int)sizeof(((struct lov_comp_md_entry_v1 *)0)->lcme_padding_1) == 2, "found %lld\n",
-                (long long)(int)sizeof(((struct lov_comp_md_entry_v1 *)0)->lcme_padding_1));
+       LASSERTF((int)offsetof(struct lov_comp_md_entry_v1, lcme_compr_type) == 46, "found %lld\n",
+                (long long)(int)offsetof(struct lov_comp_md_entry_v1, lcme_compr_type));
+       LASSERTF((int)sizeof(((struct lov_comp_md_entry_v1 *)0)->lcme_compr_type) == 1, "found %lld\n",
+                (long long)(int)sizeof(((struct lov_comp_md_entry_v1 *)0)->lcme_compr_type));
        BUILD_BUG_ON(LCME_FL_STALE != 0x00000001);
        BUILD_BUG_ON(LCME_FL_PREF_RD != 0x00000002);
        BUILD_BUG_ON(LCME_FL_PREF_WR != 0x00000004);
@@ -1857,6 +1857,9 @@ void lustre_assert_wire_constants(void)
        BUILD_BUG_ON(LCME_FL_NOSYNC != 0x00000020);
        BUILD_BUG_ON(LCME_FL_EXTENSION != 0x00000040);
        BUILD_BUG_ON(LCME_FL_PARITY != 0x00000080);
+       BUILD_BUG_ON(LCME_FL_COMPRESS != 0x00000100);
+       BUILD_BUG_ON(LCME_FL_PARTIAL != 0x00000200);
+       BUILD_BUG_ON(LCME_FL_NOCOMPR != 0x00000400);
        BUILD_BUG_ON(LCME_FL_NEG != 0x80000000);
 
        /* Checks for struct lov_comp_md_v1 */
@@ -1917,6 +1920,8 @@ void lustre_assert_wire_constants(void)
                 (long long)LCM_FL_SYNC_PENDING);
        LASSERTF(LCM_FL_PCC_RDONLY == 8, "found %lld\n",
                 (long long)LCM_FL_PCC_RDONLY);
+       LASSERTF(LCM_FL_FLR_MASK == 11, "found %lld\n",
+                (long long)LCM_FL_FLR_MASK);
 
        /* Checks for struct lmv_mds_md_v1 */
        LASSERTF((int)sizeof(struct lmv_mds_md_v1) == 56, "found %lld\n",

diff --git a/lustre/utils/wirecheck.c b/lustre/utils/wirecheck.c
index 59beef9..15fb0ec 100644 (file)
--- a/lustre/utils/wirecheck.c
+++ b/lustre/utils/wirecheck.c
@@ -895,7 +895,9 @@ check_lov_comp_md_entry_v1(void)
        CHECK_MEMBER(lov_comp_md_entry_v1, lcme_timestamp);
        CHECK_MEMBER(lov_comp_md_entry_v1, lcme_dstripe_count);
        CHECK_MEMBER(lov_comp_md_entry_v1, lcme_cstripe_count);
-       CHECK_MEMBER(lov_comp_md_entry_v1, lcme_padding_1);
+       CHECK_MEMBER(lov_comp_md_entry_v1, lcme_compr_type);
+       CHECK_BITFIELD(lov_comp_md_entry_v1, lcme_compr_lvl);
+       CHECK_BITFIELD(lov_comp_md_entry_v1, lcme_compr_chunk_log_bits);
 
        CHECK_CVALUE_X(LCME_FL_STALE);
        CHECK_CVALUE_X(LCME_FL_PREF_RD);
@@ -906,6 +908,9 @@ check_lov_comp_md_entry_v1(void)
        CHECK_CVALUE_X(LCME_FL_NOSYNC);
        CHECK_CVALUE_X(LCME_FL_EXTENSION);
        CHECK_CVALUE_X(LCME_FL_PARITY);
+       CHECK_CVALUE_X(LCME_FL_COMPRESS);
+       CHECK_CVALUE_X(LCME_FL_PARTIAL);
+       CHECK_CVALUE_X(LCME_FL_NOCOMPR);
        CHECK_CVALUE_X(LCME_FL_NEG);
 }
 
@@ -933,6 +938,7 @@ check_lov_comp_md_v1(void)
        CHECK_VALUE(LCM_FL_WRITE_PENDING);
        CHECK_VALUE(LCM_FL_SYNC_PENDING);
        CHECK_VALUE(LCM_FL_PCC_RDONLY);
+       CHECK_VALUE(LCM_FL_FLR_MASK);
 }
 
 static void

diff --git a/lustre/utils/wiretest.c b/lustre/utils/wiretest.c
index d110255..9822479 100644 (file)
--- a/lustre/utils/wiretest.c
+++ b/lustre/utils/wiretest.c
@@ -1907,10 +1907,10 @@ void lustre_assert_wire_constants(void)
                 (long long)(int)offsetof(struct lov_comp_md_entry_v1, lcme_cstripe_count));
        LASSERTF((int)sizeof(((struct lov_comp_md_entry_v1 *)0)->lcme_cstripe_count) == 1, "found %lld\n",
                 (long long)(int)sizeof(((struct lov_comp_md_entry_v1 *)0)->lcme_cstripe_count));
-       LASSERTF((int)offsetof(struct lov_comp_md_entry_v1, lcme_padding_1) == 46, "found %lld\n",
-                (long long)(int)offsetof(struct lov_comp_md_entry_v1, lcme_padding_1));
-       LASSERTF((int)sizeof(((struct lov_comp_md_entry_v1 *)0)->lcme_padding_1) == 2, "found %lld\n",
-                (long long)(int)sizeof(((struct lov_comp_md_entry_v1 *)0)->lcme_padding_1));
+       LASSERTF((int)offsetof(struct lov_comp_md_entry_v1, lcme_compr_type) == 46, "found %lld\n",
+                (long long)(int)offsetof(struct lov_comp_md_entry_v1, lcme_compr_type));
+       LASSERTF((int)sizeof(((struct lov_comp_md_entry_v1 *)0)->lcme_compr_type) == 1, "found %lld\n",
+                (long long)(int)sizeof(((struct lov_comp_md_entry_v1 *)0)->lcme_compr_type));
        BUILD_BUG_ON(LCME_FL_STALE != 0x00000001);
        BUILD_BUG_ON(LCME_FL_PREF_RD != 0x00000002);
        BUILD_BUG_ON(LCME_FL_PREF_WR != 0x00000004);
@@ -1920,6 +1920,9 @@ void lustre_assert_wire_constants(void)
        BUILD_BUG_ON(LCME_FL_NOSYNC != 0x00000020);
        BUILD_BUG_ON(LCME_FL_EXTENSION != 0x00000040);
        BUILD_BUG_ON(LCME_FL_PARITY != 0x00000080);
+       BUILD_BUG_ON(LCME_FL_COMPRESS != 0x00000100);
+       BUILD_BUG_ON(LCME_FL_PARTIAL != 0x00000200);
+       BUILD_BUG_ON(LCME_FL_NOCOMPR != 0x00000400);
        BUILD_BUG_ON(LCME_FL_NEG != 0x80000000);
 
        /* Checks for struct lov_comp_md_v1 */
@@ -1980,6 +1983,8 @@ void lustre_assert_wire_constants(void)
                 (long long)LCM_FL_SYNC_PENDING);
        LASSERTF(LCM_FL_PCC_RDONLY == 8, "found %lld\n",
                 (long long)LCM_FL_PCC_RDONLY);
+       LASSERTF(LCM_FL_FLR_MASK == 11, "found %lld\n",
+                (long long)LCM_FL_FLR_MASK);
 
        /* Checks for struct lmv_mds_md_v1 */
        LASSERTF((int)sizeof(struct lmv_mds_md_v1) == 56, "found %lld\n",