From 7ac05313beaf1492d63a5d78170de3d7ab82460b Mon Sep 17 00:00:00 2001 From: Patrick Farrell Date: Wed, 20 Dec 2023 14:51:55 -0500 Subject: [PATCH] EX-8851 lustre: add uncompressed size to compression header It's useful to have the uncompressed size of the data in the compression header. Also, we have three checksum fields - compressed, uncompressed, and header, but in practice, checksumming the compressed data including the header is enough to cover all of these. This patch cleans up all of this at the same time. Signed-off-by: Patrick Farrell Change-Id: Ie82e0dbe9c862ddc88999b109cea1f27577dbbff Reviewed-on: https://review.whamcloud.com/c/ex/lustre-release/+/53520 Tested-by: jenkins Tested-by: Maloo Reviewed-by: Andreas Dilger --- lustre/include/uapi/linux/lustre/lustre_idl.h | 8 ++++---- lustre/obdclass/lustre_compr.c | 4 ++-- lustre/ofd/ofd_compress.c | 9 +++++++++ lustre/ptlrpc/wiretest.c | 10 +++++----- lustre/utils/wirecheck.c | 2 +- lustre/utils/wiretest.c | 10 +++++----- 6 files changed, 26 insertions(+), 17 deletions(-) diff --git a/lustre/include/uapi/linux/lustre/lustre_idl.h b/lustre/include/uapi/linux/lustre/lustre_idl.h index f840eef..1f785d4 100644 --- a/lustre/include/uapi/linux/lustre/lustre_idl.h +++ b/lustre/include/uapi/linux/lustre/lustre_idl.h @@ -3834,10 +3834,10 @@ struct ll_compr_hdr { __u8 llch_compr_level:4, /* per-algorithm mapped level */ llch_chunk_log_bits:4; __u32 llch_compr_size; /* bytes of compressed data */ - __u32 llch_reserved; /* unused, initialize to 0 */ - __u32 llch_uncompr_csum; /* crc32 of raw data, or 0 */ - __u32 llch_compr_csum; /* crc32 of compressed data, or 0 */ - __u32 llch_hdr_csum; /* crc32 of magic..compr_csum, or 0 */ + __u32 llch_uncompr_size; /* bytes of uncompressed data */ + __u32 llch_reserved; /* reserved for future use */ + __u32 llch_compr_csum; /* crc32 of compressed data, or 0 */ + __u32 llch_hdr_csum; /* crc32 of magic..compr_csum, or 0 */ }; #if defined(__cplusplus) diff --git a/lustre/obdclass/lustre_compr.c b/lustre/obdclass/lustre_compr.c index 2805a91..e5268013 100644 --- a/lustre/obdclass/lustre_compr.c +++ b/lustre/obdclass/lustre_compr.c @@ -311,9 +311,9 @@ int compress_chunk(const char *obd_name, struct crypto_comp *cc, llch->llch_chunk_log_bits = chunk_bits - COMPR_CHUNK_MIN_BITS; llch->llch_flags = 0; llch->llch_compr_size = len; - llch->llch_compr_csum = 0; - llch->llch_uncompr_csum = 0; + llch->llch_uncompr_size = in_len; llch->llch_reserved = 0; + llch->llch_compr_csum = 0; llch->llch_hdr_csum = 0; *out_len = len + sizeof(*llch); diff --git a/lustre/ofd/ofd_compress.c b/lustre/ofd/ofd_compress.c index cfb87a5..9778693 100644 --- a/lustre/ofd/ofd_compress.c +++ b/lustre/ofd/ofd_compress.c @@ -117,6 +117,15 @@ static int decompress_chunk_in_lnb(const char *obd_name, lnbs[lnb_start].lnb_file_offset, rc); GOTO(out, rc); } + /* uncompressed size is not set by older clients, but older clients + * reliably zero unused parts of the header, so we skip if it's zero + */ + if (llch->llch_uncompr_size != 0 && dst_size != llch->llch_uncompr_size) { + CERROR("%s: Compressed chunk at %llu invalid, uncompressed size from disk %d disagrees with result of decompression %d, rc: %d\n", + obd_name, lnbs[lnb_start].lnb_file_offset, + llch->llch_uncompr_size, dst_size, rc); + GOTO(out, rc = -EUCLEAN); + } LASSERT(dst_size <= chunk_size); /* now that we've successfully decompressed this chunk, we copy diff --git a/lustre/ptlrpc/wiretest.c b/lustre/ptlrpc/wiretest.c index c04cc26..11a99f0 100644 --- a/lustre/ptlrpc/wiretest.c +++ b/lustre/ptlrpc/wiretest.c @@ -803,14 +803,14 @@ void lustre_assert_wire_constants(void) (long long)(int)offsetof(struct ll_compr_hdr, llch_compr_size)); LASSERTF((int)sizeof(((struct ll_compr_hdr *)0)->llch_compr_size) == 4, "found %lld\n", (long long)(int)sizeof(((struct ll_compr_hdr *)0)->llch_compr_size)); - LASSERTF((int)offsetof(struct ll_compr_hdr, llch_reserved) == 16, "found %lld\n", + LASSERTF((int)offsetof(struct ll_compr_hdr, llch_uncompr_size) == 16, "found %lld\n", + (long long)(int)offsetof(struct ll_compr_hdr, llch_uncompr_size)); + LASSERTF((int)sizeof(((struct ll_compr_hdr *)0)->llch_uncompr_size) == 4, "found %lld\n", + (long long)(int)sizeof(((struct ll_compr_hdr *)0)->llch_uncompr_size)); + LASSERTF((int)offsetof(struct ll_compr_hdr, llch_reserved) == 20, "found %lld\n", (long long)(int)offsetof(struct ll_compr_hdr, llch_reserved)); LASSERTF((int)sizeof(((struct ll_compr_hdr *)0)->llch_reserved) == 4, "found %lld\n", (long long)(int)sizeof(((struct ll_compr_hdr *)0)->llch_reserved)); - LASSERTF((int)offsetof(struct ll_compr_hdr, llch_uncompr_csum) == 20, "found %lld\n", - (long long)(int)offsetof(struct ll_compr_hdr, llch_uncompr_csum)); - LASSERTF((int)sizeof(((struct ll_compr_hdr *)0)->llch_uncompr_csum) == 4, "found %lld\n", - (long long)(int)sizeof(((struct ll_compr_hdr *)0)->llch_uncompr_csum)); LASSERTF((int)offsetof(struct ll_compr_hdr, llch_compr_csum) == 24, "found %lld\n", (long long)(int)offsetof(struct ll_compr_hdr, llch_compr_csum)); LASSERTF((int)sizeof(((struct ll_compr_hdr *)0)->llch_compr_csum) == 4, "found %lld\n", diff --git a/lustre/utils/wirecheck.c b/lustre/utils/wirecheck.c index 56c6295..48e070f 100644 --- a/lustre/utils/wirecheck.c +++ b/lustre/utils/wirecheck.c @@ -407,8 +407,8 @@ check_ll_compr_hdr(void) CHECK_BITFIELD(ll_compr_hdr, llch_comp_level); CHECK_BITFIELD(ll_compr_hdr, llch_chunk_log_bits); CHECK_MEMBER(ll_compr_hdr, llch_compr_size); + CHECK_MEMBER(ll_compr_hdr, llch_uncompr_size); CHECK_MEMBER(ll_compr_hdr, llch_reserved); - CHECK_MEMBER(ll_compr_hdr, llch_uncompr_csum); CHECK_MEMBER(ll_compr_hdr, llch_compr_csum); CHECK_MEMBER(ll_compr_hdr, llch_hdr_csum); diff --git a/lustre/utils/wiretest.c b/lustre/utils/wiretest.c index f387f3b..18126a3 100644 --- a/lustre/utils/wiretest.c +++ b/lustre/utils/wiretest.c @@ -839,14 +839,14 @@ void lustre_assert_wire_constants(void) (long long)(int)offsetof(struct ll_compr_hdr, llch_compr_size)); LASSERTF((int)sizeof(((struct ll_compr_hdr *)0)->llch_compr_size) == 4, "found %lld\n", (long long)(int)sizeof(((struct ll_compr_hdr *)0)->llch_compr_size)); - LASSERTF((int)offsetof(struct ll_compr_hdr, llch_reserved) == 16, "found %lld\n", + LASSERTF((int)offsetof(struct ll_compr_hdr, llch_uncompr_size) == 16, "found %lld\n", + (long long)(int)offsetof(struct ll_compr_hdr, llch_uncompr_size)); + LASSERTF((int)sizeof(((struct ll_compr_hdr *)0)->llch_uncompr_size) == 4, "found %lld\n", + (long long)(int)sizeof(((struct ll_compr_hdr *)0)->llch_uncompr_size)); + LASSERTF((int)offsetof(struct ll_compr_hdr, llch_reserved) == 20, "found %lld\n", (long long)(int)offsetof(struct ll_compr_hdr, llch_reserved)); LASSERTF((int)sizeof(((struct ll_compr_hdr *)0)->llch_reserved) == 4, "found %lld\n", (long long)(int)sizeof(((struct ll_compr_hdr *)0)->llch_reserved)); - LASSERTF((int)offsetof(struct ll_compr_hdr, llch_uncompr_csum) == 20, "found %lld\n", - (long long)(int)offsetof(struct ll_compr_hdr, llch_uncompr_csum)); - LASSERTF((int)sizeof(((struct ll_compr_hdr *)0)->llch_uncompr_csum) == 4, "found %lld\n", - (long long)(int)sizeof(((struct ll_compr_hdr *)0)->llch_uncompr_csum)); LASSERTF((int)offsetof(struct ll_compr_hdr, llch_compr_csum) == 24, "found %lld\n", (long long)(int)offsetof(struct ll_compr_hdr, llch_compr_csum)); LASSERTF((int)sizeof(((struct ll_compr_hdr *)0)->llch_compr_csum) == 4, "found %lld\n", -- 1.8.3.1