From 570d76dc3fd677357d76187ba718e339dd695788 Mon Sep 17 00:00:00 2001 From: Bobi Jam Date: Mon, 14 Nov 2022 16:25:05 +0800 Subject: [PATCH] LU-10026 csdc: reserve layout bits for compress component Add layout bits for compress component layout. * lcme_compr_type: compression type (gzip, lz4, lz4hc, lzo, etc.) * lcme_compr_lvl: compression level (0=default, 1-15) * lcme_compr_chunk_log_bits: chunk size = 2^(16+chunk_log_bits) Component pattern: * LOV_PATTERN_COMPRESS - file contains compressed data chunks and cannot be read by a client without decompression support. Compress component flags: * LCME_FL_COMPRESS - the component should be compressed with the compression algorithm stored in lcme_comp_type, at level lcme_comp_level, with chunk size 2^(16+lcme_chunk_log_bits) * LCME_FL_PARTIAL - the component holds some uncompressed chunks due to poor write size/alignment, and may benefit from being recompressed as the full file data is available * LCME_FL_NOCOMPR - the component should not be compressed because the data was found to be incompressible, or by user request Signed-off-by: Bobi Jam Change-Id: Idca22cca87b01bba8a5b3c85ca62044abe1d30eb Reviewed-on: https://review.whamcloud.com/c/fs/lustre-release/+/49170 Reviewed-by: Artem Blagodarenko Reviewed-by: Oleg Drokin Reviewed-by: Andreas Dilger Tested-by: jenkins Tested-by: Maloo --- lustre/include/lustre/lustreapi.h | 3 +++ lustre/include/uapi/linux/lustre/lustre_user.h | 14 +++++++++++++- lustre/ptlrpc/pack_generic.c | 1 - lustre/ptlrpc/wiretest.c | 13 +++++++++---- lustre/utils/wirecheck.c | 8 +++++++- lustre/utils/wiretest.c | 13 +++++++++---- 6 files changed, 41 insertions(+), 11 deletions(-) diff --git a/lustre/include/lustre/lustreapi.h b/lustre/include/lustre/lustreapi.h index e257eb4..1e486fd 100644 --- a/lustre/include/lustre/lustreapi.h +++ b/lustre/include/lustre/lustreapi.h @@ -1124,6 +1124,9 @@ static const struct comp_flag_name { { LCME_FL_NOSYNC, "nosync" }, { LCME_FL_EXTENSION, "extension" }, { LCME_FL_PARITY, "parity" }, + { LCME_FL_COMPRESS, "compress" }, + { LCME_FL_PARTIAL, "partial" }, + { LCME_FL_NOCOMPR, "nocompr" }, }; /* HSM component flags table */ diff --git a/lustre/include/uapi/linux/lustre/lustre_user.h b/lustre/include/uapi/linux/lustre/lustre_user.h index dc1af4e..70949e8 100644 --- a/lustre/include/uapi/linux/lustre/lustre_user.h +++ b/lustre/include/uapi/linux/lustre/lustre_user.h @@ -724,6 +724,7 @@ struct fsxattr { #define LOV_PATTERN_MDT 0x100 #define LOV_PATTERN_OVERSTRIPING 0x200 #define LOV_PATTERN_FOREIGN 0x400 +#define LOV_PATTERN_COMPRESS 0x800 #define LOV_PATTERN_F_MASK 0xffff0000 #define LOV_PATTERN_F_HOLE 0x40000000 /* there is hole in LOV EA */ @@ -893,6 +894,13 @@ enum lov_comp_md_entry_flags { LCME_FL_NOSYNC = 0x00000020, /* FLR: no sync for the mirror */ LCME_FL_EXTENSION = 0x00000040, /* extension comp, never init */ LCME_FL_PARITY = 0x00000080, /* EC: a parity code component */ + LCME_FL_COMPRESS = 0x00000100, /* the component should be compressed */ + LCME_FL_PARTIAL = 0x00000200, /* some chunks in the component are + * uncompressed + */ + LCME_FL_NOCOMPR = 0x00000400, /* the component should not be + * compressed + */ LCME_FL_NEG = 0x80000000 /* used to indicate a negative flag, * won't be stored on disk */ @@ -963,7 +971,11 @@ struct lov_comp_md_entry_v1 { __u8 lcme_cstripe_count; /* code stripe count, * p value in EC */ - __u16 lcme_padding_1; + __u8 lcme_compr_type; /* compress type */ + __u8 lcme_compr_lvl:4; /* compress level */ + __u8 lcme_compr_chunk_log_bits:4; + /* chunk_size = 2^(16+chunk_log_bits) + * i.e. power-of-two multiple of 64KiB */ } __attribute__((packed)); #define SEQ_ID_MAX 0x0000FFFF diff --git a/lustre/ptlrpc/pack_generic.c b/lustre/ptlrpc/pack_generic.c index 8c3f839..20a742a 100644 --- a/lustre/ptlrpc/pack_generic.c +++ b/lustre/ptlrpc/pack_generic.c @@ -2522,7 +2522,6 @@ void lustre_swab_lov_comp_md_v1(struct lov_comp_md_v1 *lum) __swab32s(&ent->lcme_layout_gen); /* no need to swab lcme_dstripe_count */ /* no need to swab lcme_cstripe_count */ - BUILD_BUG_ON(offsetof(typeof(*ent), lcme_padding_1) == 0); v1 = (struct lov_user_md_v1 *)((char *)lum + off); stripe_count = v1->lmm_stripe_count; diff --git a/lustre/ptlrpc/wiretest.c b/lustre/ptlrpc/wiretest.c index 1b5a59d..63ff323 100644 --- a/lustre/ptlrpc/wiretest.c +++ b/lustre/ptlrpc/wiretest.c @@ -1844,10 +1844,10 @@ void lustre_assert_wire_constants(void) (long long)(int)offsetof(struct lov_comp_md_entry_v1, lcme_cstripe_count)); LASSERTF((int)sizeof(((struct lov_comp_md_entry_v1 *)0)->lcme_cstripe_count) == 1, "found %lld\n", (long long)(int)sizeof(((struct lov_comp_md_entry_v1 *)0)->lcme_cstripe_count)); - LASSERTF((int)offsetof(struct lov_comp_md_entry_v1, lcme_padding_1) == 46, "found %lld\n", - (long long)(int)offsetof(struct lov_comp_md_entry_v1, lcme_padding_1)); - LASSERTF((int)sizeof(((struct lov_comp_md_entry_v1 *)0)->lcme_padding_1) == 2, "found %lld\n", - (long long)(int)sizeof(((struct lov_comp_md_entry_v1 *)0)->lcme_padding_1)); + LASSERTF((int)offsetof(struct lov_comp_md_entry_v1, lcme_compr_type) == 46, "found %lld\n", + (long long)(int)offsetof(struct lov_comp_md_entry_v1, lcme_compr_type)); + LASSERTF((int)sizeof(((struct lov_comp_md_entry_v1 *)0)->lcme_compr_type) == 1, "found %lld\n", + (long long)(int)sizeof(((struct lov_comp_md_entry_v1 *)0)->lcme_compr_type)); BUILD_BUG_ON(LCME_FL_STALE != 0x00000001); BUILD_BUG_ON(LCME_FL_PREF_RD != 0x00000002); BUILD_BUG_ON(LCME_FL_PREF_WR != 0x00000004); @@ -1857,6 +1857,9 @@ void lustre_assert_wire_constants(void) BUILD_BUG_ON(LCME_FL_NOSYNC != 0x00000020); BUILD_BUG_ON(LCME_FL_EXTENSION != 0x00000040); BUILD_BUG_ON(LCME_FL_PARITY != 0x00000080); + BUILD_BUG_ON(LCME_FL_COMPRESS != 0x00000100); + BUILD_BUG_ON(LCME_FL_PARTIAL != 0x00000200); + BUILD_BUG_ON(LCME_FL_NOCOMPR != 0x00000400); BUILD_BUG_ON(LCME_FL_NEG != 0x80000000); /* Checks for struct lov_comp_md_v1 */ @@ -1917,6 +1920,8 @@ void lustre_assert_wire_constants(void) (long long)LCM_FL_SYNC_PENDING); LASSERTF(LCM_FL_PCC_RDONLY == 8, "found %lld\n", (long long)LCM_FL_PCC_RDONLY); + LASSERTF(LCM_FL_FLR_MASK == 11, "found %lld\n", + (long long)LCM_FL_FLR_MASK); /* Checks for struct lmv_mds_md_v1 */ LASSERTF((int)sizeof(struct lmv_mds_md_v1) == 56, "found %lld\n", diff --git a/lustre/utils/wirecheck.c b/lustre/utils/wirecheck.c index 59beef9..15fb0ec 100644 --- a/lustre/utils/wirecheck.c +++ b/lustre/utils/wirecheck.c @@ -895,7 +895,9 @@ check_lov_comp_md_entry_v1(void) CHECK_MEMBER(lov_comp_md_entry_v1, lcme_timestamp); CHECK_MEMBER(lov_comp_md_entry_v1, lcme_dstripe_count); CHECK_MEMBER(lov_comp_md_entry_v1, lcme_cstripe_count); - CHECK_MEMBER(lov_comp_md_entry_v1, lcme_padding_1); + CHECK_MEMBER(lov_comp_md_entry_v1, lcme_compr_type); + CHECK_BITFIELD(lov_comp_md_entry_v1, lcme_compr_lvl); + CHECK_BITFIELD(lov_comp_md_entry_v1, lcme_compr_chunk_log_bits); CHECK_CVALUE_X(LCME_FL_STALE); CHECK_CVALUE_X(LCME_FL_PREF_RD); @@ -906,6 +908,9 @@ check_lov_comp_md_entry_v1(void) CHECK_CVALUE_X(LCME_FL_NOSYNC); CHECK_CVALUE_X(LCME_FL_EXTENSION); CHECK_CVALUE_X(LCME_FL_PARITY); + CHECK_CVALUE_X(LCME_FL_COMPRESS); + CHECK_CVALUE_X(LCME_FL_PARTIAL); + CHECK_CVALUE_X(LCME_FL_NOCOMPR); CHECK_CVALUE_X(LCME_FL_NEG); } @@ -933,6 +938,7 @@ check_lov_comp_md_v1(void) CHECK_VALUE(LCM_FL_WRITE_PENDING); CHECK_VALUE(LCM_FL_SYNC_PENDING); CHECK_VALUE(LCM_FL_PCC_RDONLY); + CHECK_VALUE(LCM_FL_FLR_MASK); } static void diff --git a/lustre/utils/wiretest.c b/lustre/utils/wiretest.c index d110255..9822479 100644 --- a/lustre/utils/wiretest.c +++ b/lustre/utils/wiretest.c @@ -1907,10 +1907,10 @@ void lustre_assert_wire_constants(void) (long long)(int)offsetof(struct lov_comp_md_entry_v1, lcme_cstripe_count)); LASSERTF((int)sizeof(((struct lov_comp_md_entry_v1 *)0)->lcme_cstripe_count) == 1, "found %lld\n", (long long)(int)sizeof(((struct lov_comp_md_entry_v1 *)0)->lcme_cstripe_count)); - LASSERTF((int)offsetof(struct lov_comp_md_entry_v1, lcme_padding_1) == 46, "found %lld\n", - (long long)(int)offsetof(struct lov_comp_md_entry_v1, lcme_padding_1)); - LASSERTF((int)sizeof(((struct lov_comp_md_entry_v1 *)0)->lcme_padding_1) == 2, "found %lld\n", - (long long)(int)sizeof(((struct lov_comp_md_entry_v1 *)0)->lcme_padding_1)); + LASSERTF((int)offsetof(struct lov_comp_md_entry_v1, lcme_compr_type) == 46, "found %lld\n", + (long long)(int)offsetof(struct lov_comp_md_entry_v1, lcme_compr_type)); + LASSERTF((int)sizeof(((struct lov_comp_md_entry_v1 *)0)->lcme_compr_type) == 1, "found %lld\n", + (long long)(int)sizeof(((struct lov_comp_md_entry_v1 *)0)->lcme_compr_type)); BUILD_BUG_ON(LCME_FL_STALE != 0x00000001); BUILD_BUG_ON(LCME_FL_PREF_RD != 0x00000002); BUILD_BUG_ON(LCME_FL_PREF_WR != 0x00000004); @@ -1920,6 +1920,9 @@ void lustre_assert_wire_constants(void) BUILD_BUG_ON(LCME_FL_NOSYNC != 0x00000020); BUILD_BUG_ON(LCME_FL_EXTENSION != 0x00000040); BUILD_BUG_ON(LCME_FL_PARITY != 0x00000080); + BUILD_BUG_ON(LCME_FL_COMPRESS != 0x00000100); + BUILD_BUG_ON(LCME_FL_PARTIAL != 0x00000200); + BUILD_BUG_ON(LCME_FL_NOCOMPR != 0x00000400); BUILD_BUG_ON(LCME_FL_NEG != 0x80000000); /* Checks for struct lov_comp_md_v1 */ @@ -1980,6 +1983,8 @@ void lustre_assert_wire_constants(void) (long long)LCM_FL_SYNC_PENDING); LASSERTF(LCM_FL_PCC_RDONLY == 8, "found %lld\n", (long long)LCM_FL_PCC_RDONLY); + LASSERTF(LCM_FL_FLR_MASK == 11, "found %lld\n", + (long long)LCM_FL_FLR_MASK); /* Checks for struct lmv_mds_md_v1 */ LASSERTF((int)sizeof(struct lmv_mds_md_v1) == 56, "found %lld\n", -- 1.8.3.1