LU-10026 csdc: reserve layout bits for compress component

Add layout bits for compress component layout.

* lcme_compr_type: compression type (gzip, lz4, lz4hc, lzo, etc.)
* lcme_compr_lvl: compression level (0=default, 1-15)
* lcme_compr_chunk_log_bits: chunk size = 2^(16+chunk_log_bits)

Component pattern:
* LOV_PATTERN_COMPRESS - file contains compressed data chunks and
       cannot be read by a client without decompression support.

Compress component flags:
* LCME_FL_COMPRESS - the component should be compressed with the
       compression algorithm stored in lcme_comp_type, at level
       lcme_comp_level, with chunk size 2^(16+lcme_chunk_log_bits)
* LCME_FL_PARTIAL - the component holds some uncompressed chunks due
       to poor write size/alignment, and may benefit from being
       recompressed as the full file data is available
* LCME_FL_NOCOMPR - the component should not be compressed because
       the data was found to be incompressible, or by user request

Lustre-change: https://review.whamcloud.com/49170
Lustre-commit: TBD (from 147d4eb27b85b4994a47539be6aceff212365ee5)

Signed-off-by: Bobi Jam <bobijam@whamcloud.com>
Change-Id: Idca22cca87b01bba8a5b3c85ca62044abe1d30eb
Reviewed-on: https://review.whamcloud.com/c/ex/lustre-release/+/49321
Tested-by: jenkins <devops@whamcloud.com>
Tested-by: Maloo <maloo@whamcloud.com>
Reviewed-by: Andreas Dilger <adilger@whamcloud.com>

diff --git a/lustre/include/lustre/lustreapi.h b/lustre/include/lustre/lustreapi.h
index 05c72e0..2e65763 100644 (file)
--- a/lustre/include/lustre/lustreapi.h
+++ b/lustre/include/lustre/lustreapi.h
@@ -1103,6 +1103,9 @@ static const struct comp_flag_name {
        { LCME_FL_OFFLINE,      "offline" },
        { LCME_FL_NOSYNC,       "nosync" },
        { LCME_FL_EXTENSION,    "extension" },
+       { LCME_FL_COMPRESS,     "compress" },
+       { LCME_FL_PARTIAL,      "partial" },
+       { LCME_FL_NOCOMPR,      "nocompr" },
 };
 
 /* HSM component flags table */

diff --git a/lustre/include/uapi/linux/lustre/lustre_user.h b/lustre/include/uapi/linux/lustre/lustre_user.h
index 7e5807f..7388144 100644 (file)
--- a/lustre/include/uapi/linux/lustre/lustre_user.h
+++ b/lustre/include/uapi/linux/lustre/lustre_user.h
@@ -717,6 +717,7 @@ struct fsxattr {
 #define LOV_PATTERN_MDT                        0x100
 #define LOV_PATTERN_OVERSTRIPING       0x200
 #define LOV_PATTERN_FOREIGN            0x400
+#define LOV_PATTERN_COMPRESS           0x800
 
 #define LOV_PATTERN_F_MASK     0xffff0000
 #define LOV_PATTERN_F_HOLE     0x40000000 /* there is hole in LOV EA */
@@ -868,6 +869,13 @@ enum lov_comp_md_entry_flags {
        LCME_FL_INIT      = 0x00000010, /* instantiated */
        LCME_FL_NOSYNC    = 0x00000020, /* FLR: no sync for the mirror */
        LCME_FL_EXTENSION = 0x00000040, /* extension comp, never init */
+       LCME_FL_COMPRESS  = 0x00000100, /* the component should be compressed */
+       LCME_FL_PARTIAL   = 0x00000200, /* some chunks in the component are
+                                        * uncompressed
+                                        */
+       LCME_FL_NOCOMPR   = 0x00000400, /* the component should not be
+                                        * compressed
+                                        */
        LCME_FL_NEG       = 0x80000000  /* used to indicate a negative flag,
                                         * won't be stored on disk
                                         */
@@ -927,7 +935,12 @@ struct lov_comp_md_entry_v1 {
        __u32                   lcme_size;      /* size of component blob */
        __u32                   lcme_layout_gen;
        __u64                   lcme_timestamp; /* snapshot time if applicable*/
-       __u32                   lcme_padding_1;
+       __u16                   lcme_padding_1;
+       __u8                    lcme_compr_type;        /* compress type */
+       __u8                    lcme_compr_lvl:4;       /* compress level */
+       __u8                    lcme_compr_chunk_log_bits:4;
+                                    /* chunk_size = 2^(16+chunk_log_bits)
+                                     * i.e. power-of-two multiple of 64KiB */
 } __attribute__((packed));
 
 #define SEQ_ID_MAX             0x0000FFFF

diff --git a/lustre/ptlrpc/wiretest.c b/lustre/ptlrpc/wiretest.c
index 05d835d..20d459b 100644 (file)
--- a/lustre/ptlrpc/wiretest.c
+++ b/lustre/ptlrpc/wiretest.c
@@ -45,7 +45,6 @@
 #include <uapi/linux/lustre/lustre_lfsck_user.h>
 #include <uapi/linux/lustre/lustre_cfg.h>
 
-
 void lustre_assert_wire_constants(void)
 {
        /* Wire protocol assertions generated by 'wirecheck'
@@ -1904,8 +1903,14 @@ void lustre_assert_wire_constants(void)
                 (long long)(int)sizeof(((struct lov_comp_md_entry_v1 *)0)->lcme_timestamp));
        LASSERTF((int)offsetof(struct lov_comp_md_entry_v1, lcme_padding_1) == 44, "found %lld\n",
                 (long long)(int)offsetof(struct lov_comp_md_entry_v1, lcme_padding_1));
-       LASSERTF((int)sizeof(((struct lov_comp_md_entry_v1 *)0)->lcme_padding_1) == 4, "found %lld\n",
+       LASSERTF((int)sizeof(((struct lov_comp_md_entry_v1 *)0)->lcme_padding_1) == 2, "found %lld\n",
                 (long long)(int)sizeof(((struct lov_comp_md_entry_v1 *)0)->lcme_padding_1));
+       LASSERTF((int)offsetof(struct lov_comp_md_entry_v1, lcme_compr_type) == 46, "found %lld\n",
+                (long long)(int)offsetof(struct lov_comp_md_entry_v1, lcme_compr_type));
+       LASSERTF((int)sizeof(((struct lov_comp_md_entry_v1 *)0)->lcme_compr_type) == 1, "found %lld\n",
+                (long long)(int)sizeof(((struct lov_comp_md_entry_v1 *)0)->lcme_compr_type));
+       /* lov_comp_md_entry_v1.lcme_compr_lvl is a bitfield and cannot be checked */
+       /* lov_comp_md_entry_v1.lcme_compr_chunk_log_bits is a bitfield and cannot be checked */
        BUILD_BUG_ON(LCME_FL_STALE != 0x00000001);
        BUILD_BUG_ON(LCME_FL_PREF_RD != 0x00000002);
        BUILD_BUG_ON(LCME_FL_PREF_WR != 0x00000004);
@@ -1914,6 +1919,9 @@ void lustre_assert_wire_constants(void)
        BUILD_BUG_ON(LCME_FL_INIT != 0x00000010);
        BUILD_BUG_ON(LCME_FL_NOSYNC != 0x00000020);
        BUILD_BUG_ON(LCME_FL_EXTENSION != 0x00000040);
+       BUILD_BUG_ON(LCME_FL_COMPRESS != 0x00000100);
+       BUILD_BUG_ON(LCME_FL_PARTIAL != 0x00000200);
+       BUILD_BUG_ON(LCME_FL_NOCOMPR != 0x00000400);
        BUILD_BUG_ON(LCME_FL_NEG != 0x80000000);
 
        /* Checks for struct lov_comp_md_v1 */

diff --git a/lustre/utils/wirecheck.c b/lustre/utils/wirecheck.c
index 5678f45..eb620f8 100644 (file)
--- a/lustre/utils/wirecheck.c
+++ b/lustre/utils/wirecheck.c
@@ -897,6 +897,9 @@ check_lov_comp_md_entry_v1(void)
        CHECK_MEMBER(lov_comp_md_entry_v1, lcme_layout_gen);
        CHECK_MEMBER(lov_comp_md_entry_v1, lcme_timestamp);
        CHECK_MEMBER(lov_comp_md_entry_v1, lcme_padding_1);
+       CHECK_MEMBER(lov_comp_md_entry_v1, lcme_compr_type);
+       CHECK_BITFIELD(lov_comp_md_entry_v1, lcme_compr_lvl);
+       CHECK_BITFIELD(lov_comp_md_entry_v1, lcme_compr_chunk_log_bits);
 
        CHECK_CVALUE_X(LCME_FL_STALE);
        CHECK_CVALUE_X(LCME_FL_PREF_RD);
@@ -906,6 +909,9 @@ check_lov_comp_md_entry_v1(void)
        CHECK_CVALUE_X(LCME_FL_INIT);
        CHECK_CVALUE_X(LCME_FL_NOSYNC);
        CHECK_CVALUE_X(LCME_FL_EXTENSION);
+       CHECK_CVALUE_X(LCME_FL_COMPRESS);
+       CHECK_CVALUE_X(LCME_FL_PARTIAL);
+       CHECK_CVALUE_X(LCME_FL_NOCOMPR);
        CHECK_CVALUE_X(LCME_FL_NEG);
 }
 

diff --git a/lustre/utils/wiretest.c b/lustre/utils/wiretest.c
index bf1126b..5b35499 100644 (file)
--- a/lustre/utils/wiretest.c
+++ b/lustre/utils/wiretest.c
@@ -1940,8 +1940,14 @@ void lustre_assert_wire_constants(void)
                 (long long)(int)sizeof(((struct lov_comp_md_entry_v1 *)0)->lcme_timestamp));
        LASSERTF((int)offsetof(struct lov_comp_md_entry_v1, lcme_padding_1) == 44, "found %lld\n",
                 (long long)(int)offsetof(struct lov_comp_md_entry_v1, lcme_padding_1));
-       LASSERTF((int)sizeof(((struct lov_comp_md_entry_v1 *)0)->lcme_padding_1) == 4, "found %lld\n",
+       LASSERTF((int)sizeof(((struct lov_comp_md_entry_v1 *)0)->lcme_padding_1) == 2, "found %lld\n",
                 (long long)(int)sizeof(((struct lov_comp_md_entry_v1 *)0)->lcme_padding_1));
+       LASSERTF((int)offsetof(struct lov_comp_md_entry_v1, lcme_compr_type) == 46, "found %lld\n",
+                (long long)(int)offsetof(struct lov_comp_md_entry_v1, lcme_compr_type));
+       LASSERTF((int)sizeof(((struct lov_comp_md_entry_v1 *)0)->lcme_compr_type) == 1, "found %lld\n",
+                (long long)(int)sizeof(((struct lov_comp_md_entry_v1 *)0)->lcme_compr_type));
+       /* lov_comp_md_entry_v1.lcme_compr_lvl is a bitfield and cannot be checked */
+       /* lov_comp_md_entry_v1.lcme_compr_chunk_log_bits is a bitfield and cannot be checked */
        BUILD_BUG_ON(LCME_FL_STALE != 0x00000001);
        BUILD_BUG_ON(LCME_FL_PREF_RD != 0x00000002);
        BUILD_BUG_ON(LCME_FL_PREF_WR != 0x00000004);
@@ -1950,6 +1956,9 @@ void lustre_assert_wire_constants(void)
        BUILD_BUG_ON(LCME_FL_INIT != 0x00000010);
        BUILD_BUG_ON(LCME_FL_NOSYNC != 0x00000020);
        BUILD_BUG_ON(LCME_FL_EXTENSION != 0x00000040);
+       BUILD_BUG_ON(LCME_FL_COMPRESS != 0x00000100);
+       BUILD_BUG_ON(LCME_FL_PARTIAL != 0x00000200);
+       BUILD_BUG_ON(LCME_FL_NOCOMPR != 0x00000400);
        BUILD_BUG_ON(LCME_FL_NEG != 0x80000000);
 
        /* Checks for struct lov_comp_md_v1 */