From: Bobi Jam Date: Mon, 14 Nov 2022 08:25:05 +0000 (+0800) Subject: LU-10026 csdc: reserve layout bits for compress component X-Git-Url: https://git.whamcloud.com/gitweb?a=commitdiff_plain;h=9a74cc47fad5b2e09a840a273fa8b0a784972e68;p=fs%2Flustre-release.git LU-10026 csdc: reserve layout bits for compress component Add layout bits for compress component layout. * lcme_compr_type: compression type (gzip, lz4, lz4hc, lzo, etc.) * lcme_compr_lvl: compression level (0=default, 1-15) * lcme_compr_chunk_log_bits: chunk size = 2^(16+chunk_log_bits) Component pattern: * LOV_PATTERN_COMPRESS - file contains compressed data chunks and cannot be read by a client without decompression support. Compress component flags: * LCME_FL_COMPRESS - the component should be compressed with the compression algorithm stored in lcme_comp_type, at level lcme_comp_level, with chunk size 2^(16+lcme_chunk_log_bits) * LCME_FL_PARTIAL - the component holds some uncompressed chunks due to poor write size/alignment, and may benefit from being recompressed as the full file data is available * LCME_FL_NOCOMPR - the component should not be compressed because the data was found to be incompressible, or by user request Lustre-change: https://review.whamcloud.com/49170 Lustre-commit: TBD (from 147d4eb27b85b4994a47539be6aceff212365ee5) Signed-off-by: Bobi Jam Change-Id: Idca22cca87b01bba8a5b3c85ca62044abe1d30eb Reviewed-on: https://review.whamcloud.com/c/ex/lustre-release/+/49321 Tested-by: jenkins Tested-by: Maloo Reviewed-by: Andreas Dilger --- diff --git a/lustre/include/lustre/lustreapi.h b/lustre/include/lustre/lustreapi.h index 05c72e0..2e65763 100644 --- a/lustre/include/lustre/lustreapi.h +++ b/lustre/include/lustre/lustreapi.h @@ -1103,6 +1103,9 @@ static const struct comp_flag_name { { LCME_FL_OFFLINE, "offline" }, { LCME_FL_NOSYNC, "nosync" }, { LCME_FL_EXTENSION, "extension" }, + { LCME_FL_COMPRESS, "compress" }, + { LCME_FL_PARTIAL, "partial" }, + { LCME_FL_NOCOMPR, "nocompr" }, }; /* HSM component flags table */ diff --git a/lustre/include/uapi/linux/lustre/lustre_user.h b/lustre/include/uapi/linux/lustre/lustre_user.h index 7e5807f..7388144 100644 --- a/lustre/include/uapi/linux/lustre/lustre_user.h +++ b/lustre/include/uapi/linux/lustre/lustre_user.h @@ -717,6 +717,7 @@ struct fsxattr { #define LOV_PATTERN_MDT 0x100 #define LOV_PATTERN_OVERSTRIPING 0x200 #define LOV_PATTERN_FOREIGN 0x400 +#define LOV_PATTERN_COMPRESS 0x800 #define LOV_PATTERN_F_MASK 0xffff0000 #define LOV_PATTERN_F_HOLE 0x40000000 /* there is hole in LOV EA */ @@ -868,6 +869,13 @@ enum lov_comp_md_entry_flags { LCME_FL_INIT = 0x00000010, /* instantiated */ LCME_FL_NOSYNC = 0x00000020, /* FLR: no sync for the mirror */ LCME_FL_EXTENSION = 0x00000040, /* extension comp, never init */ + LCME_FL_COMPRESS = 0x00000100, /* the component should be compressed */ + LCME_FL_PARTIAL = 0x00000200, /* some chunks in the component are + * uncompressed + */ + LCME_FL_NOCOMPR = 0x00000400, /* the component should not be + * compressed + */ LCME_FL_NEG = 0x80000000 /* used to indicate a negative flag, * won't be stored on disk */ @@ -927,7 +935,12 @@ struct lov_comp_md_entry_v1 { __u32 lcme_size; /* size of component blob */ __u32 lcme_layout_gen; __u64 lcme_timestamp; /* snapshot time if applicable*/ - __u32 lcme_padding_1; + __u16 lcme_padding_1; + __u8 lcme_compr_type; /* compress type */ + __u8 lcme_compr_lvl:4; /* compress level */ + __u8 lcme_compr_chunk_log_bits:4; + /* chunk_size = 2^(16+chunk_log_bits) + * i.e. power-of-two multiple of 64KiB */ } __attribute__((packed)); #define SEQ_ID_MAX 0x0000FFFF diff --git a/lustre/ptlrpc/wiretest.c b/lustre/ptlrpc/wiretest.c index 05d835d..20d459b 100644 --- a/lustre/ptlrpc/wiretest.c +++ b/lustre/ptlrpc/wiretest.c @@ -45,7 +45,6 @@ #include #include - void lustre_assert_wire_constants(void) { /* Wire protocol assertions generated by 'wirecheck' @@ -1904,8 +1903,14 @@ void lustre_assert_wire_constants(void) (long long)(int)sizeof(((struct lov_comp_md_entry_v1 *)0)->lcme_timestamp)); LASSERTF((int)offsetof(struct lov_comp_md_entry_v1, lcme_padding_1) == 44, "found %lld\n", (long long)(int)offsetof(struct lov_comp_md_entry_v1, lcme_padding_1)); - LASSERTF((int)sizeof(((struct lov_comp_md_entry_v1 *)0)->lcme_padding_1) == 4, "found %lld\n", + LASSERTF((int)sizeof(((struct lov_comp_md_entry_v1 *)0)->lcme_padding_1) == 2, "found %lld\n", (long long)(int)sizeof(((struct lov_comp_md_entry_v1 *)0)->lcme_padding_1)); + LASSERTF((int)offsetof(struct lov_comp_md_entry_v1, lcme_compr_type) == 46, "found %lld\n", + (long long)(int)offsetof(struct lov_comp_md_entry_v1, lcme_compr_type)); + LASSERTF((int)sizeof(((struct lov_comp_md_entry_v1 *)0)->lcme_compr_type) == 1, "found %lld\n", + (long long)(int)sizeof(((struct lov_comp_md_entry_v1 *)0)->lcme_compr_type)); + /* lov_comp_md_entry_v1.lcme_compr_lvl is a bitfield and cannot be checked */ + /* lov_comp_md_entry_v1.lcme_compr_chunk_log_bits is a bitfield and cannot be checked */ BUILD_BUG_ON(LCME_FL_STALE != 0x00000001); BUILD_BUG_ON(LCME_FL_PREF_RD != 0x00000002); BUILD_BUG_ON(LCME_FL_PREF_WR != 0x00000004); @@ -1914,6 +1919,9 @@ void lustre_assert_wire_constants(void) BUILD_BUG_ON(LCME_FL_INIT != 0x00000010); BUILD_BUG_ON(LCME_FL_NOSYNC != 0x00000020); BUILD_BUG_ON(LCME_FL_EXTENSION != 0x00000040); + BUILD_BUG_ON(LCME_FL_COMPRESS != 0x00000100); + BUILD_BUG_ON(LCME_FL_PARTIAL != 0x00000200); + BUILD_BUG_ON(LCME_FL_NOCOMPR != 0x00000400); BUILD_BUG_ON(LCME_FL_NEG != 0x80000000); /* Checks for struct lov_comp_md_v1 */ diff --git a/lustre/utils/wirecheck.c b/lustre/utils/wirecheck.c index 5678f45..eb620f8 100644 --- a/lustre/utils/wirecheck.c +++ b/lustre/utils/wirecheck.c @@ -897,6 +897,9 @@ check_lov_comp_md_entry_v1(void) CHECK_MEMBER(lov_comp_md_entry_v1, lcme_layout_gen); CHECK_MEMBER(lov_comp_md_entry_v1, lcme_timestamp); CHECK_MEMBER(lov_comp_md_entry_v1, lcme_padding_1); + CHECK_MEMBER(lov_comp_md_entry_v1, lcme_compr_type); + CHECK_BITFIELD(lov_comp_md_entry_v1, lcme_compr_lvl); + CHECK_BITFIELD(lov_comp_md_entry_v1, lcme_compr_chunk_log_bits); CHECK_CVALUE_X(LCME_FL_STALE); CHECK_CVALUE_X(LCME_FL_PREF_RD); @@ -906,6 +909,9 @@ check_lov_comp_md_entry_v1(void) CHECK_CVALUE_X(LCME_FL_INIT); CHECK_CVALUE_X(LCME_FL_NOSYNC); CHECK_CVALUE_X(LCME_FL_EXTENSION); + CHECK_CVALUE_X(LCME_FL_COMPRESS); + CHECK_CVALUE_X(LCME_FL_PARTIAL); + CHECK_CVALUE_X(LCME_FL_NOCOMPR); CHECK_CVALUE_X(LCME_FL_NEG); } diff --git a/lustre/utils/wiretest.c b/lustre/utils/wiretest.c index bf1126b..5b35499 100644 --- a/lustre/utils/wiretest.c +++ b/lustre/utils/wiretest.c @@ -1940,8 +1940,14 @@ void lustre_assert_wire_constants(void) (long long)(int)sizeof(((struct lov_comp_md_entry_v1 *)0)->lcme_timestamp)); LASSERTF((int)offsetof(struct lov_comp_md_entry_v1, lcme_padding_1) == 44, "found %lld\n", (long long)(int)offsetof(struct lov_comp_md_entry_v1, lcme_padding_1)); - LASSERTF((int)sizeof(((struct lov_comp_md_entry_v1 *)0)->lcme_padding_1) == 4, "found %lld\n", + LASSERTF((int)sizeof(((struct lov_comp_md_entry_v1 *)0)->lcme_padding_1) == 2, "found %lld\n", (long long)(int)sizeof(((struct lov_comp_md_entry_v1 *)0)->lcme_padding_1)); + LASSERTF((int)offsetof(struct lov_comp_md_entry_v1, lcme_compr_type) == 46, "found %lld\n", + (long long)(int)offsetof(struct lov_comp_md_entry_v1, lcme_compr_type)); + LASSERTF((int)sizeof(((struct lov_comp_md_entry_v1 *)0)->lcme_compr_type) == 1, "found %lld\n", + (long long)(int)sizeof(((struct lov_comp_md_entry_v1 *)0)->lcme_compr_type)); + /* lov_comp_md_entry_v1.lcme_compr_lvl is a bitfield and cannot be checked */ + /* lov_comp_md_entry_v1.lcme_compr_chunk_log_bits is a bitfield and cannot be checked */ BUILD_BUG_ON(LCME_FL_STALE != 0x00000001); BUILD_BUG_ON(LCME_FL_PREF_RD != 0x00000002); BUILD_BUG_ON(LCME_FL_PREF_WR != 0x00000004); @@ -1950,6 +1956,9 @@ void lustre_assert_wire_constants(void) BUILD_BUG_ON(LCME_FL_INIT != 0x00000010); BUILD_BUG_ON(LCME_FL_NOSYNC != 0x00000020); BUILD_BUG_ON(LCME_FL_EXTENSION != 0x00000040); + BUILD_BUG_ON(LCME_FL_COMPRESS != 0x00000100); + BUILD_BUG_ON(LCME_FL_PARTIAL != 0x00000200); + BUILD_BUG_ON(LCME_FL_NOCOMPR != 0x00000400); BUILD_BUG_ON(LCME_FL_NEG != 0x80000000); /* Checks for struct lov_comp_md_v1 */