EX-8851 lustre: add uncompressed size to compression header

It's useful to have the uncompressed size of the data in the
compression header.  Also, we have three checksum fields -
compressed, uncompressed, and header, but in practice,
checksumming the compressed data including the header is
enough to cover all of these.

This patch cleans up all of this at the same time.

Signed-off-by: Patrick Farrell <pfarrell@whamcloud.com>
Change-Id: Ie82e0dbe9c862ddc88999b109cea1f27577dbbff
Reviewed-on: https://review.whamcloud.com/c/ex/lustre-release/+/53520
Tested-by: jenkins <devops@whamcloud.com>
Tested-by: Maloo <maloo@whamcloud.com>
Reviewed-by: Andreas Dilger <adilger@whamcloud.com>

diff --git a/lustre/include/uapi/linux/lustre/lustre_idl.h b/lustre/include/uapi/linux/lustre/lustre_idl.h
index f840eef..1f785d4 100644 (file)
--- a/lustre/include/uapi/linux/lustre/lustre_idl.h
+++ b/lustre/include/uapi/linux/lustre/lustre_idl.h
@@ -3834,10 +3834,10 @@ struct ll_compr_hdr {
        __u8  llch_compr_level:4, /* per-algorithm mapped level */
              llch_chunk_log_bits:4;
        __u32 llch_compr_size;    /* bytes of compressed data */
-       __u32 llch_reserved;     /* unused, initialize to 0 */
-       __u32 llch_uncompr_csum; /* crc32 of raw data, or 0 */
-       __u32 llch_compr_csum;   /* crc32 of compressed data, or 0 */
-       __u32 llch_hdr_csum;     /* crc32 of magic..compr_csum, or 0 */
+       __u32 llch_uncompr_size;  /* bytes of uncompressed data */
+       __u32 llch_reserved;      /* reserved for future use */
+       __u32 llch_compr_csum;    /* crc32 of compressed data, or 0 */
+       __u32 llch_hdr_csum;      /* crc32 of magic..compr_csum, or 0 */
 };
 
 #if defined(__cplusplus)

diff --git a/lustre/obdclass/lustre_compr.c b/lustre/obdclass/lustre_compr.c
index 2805a91..e526801 100644 (file)
--- a/lustre/obdclass/lustre_compr.c
+++ b/lustre/obdclass/lustre_compr.c
@@ -311,9 +311,9 @@ int compress_chunk(const char *obd_name, struct crypto_comp *cc,
        llch->llch_chunk_log_bits = chunk_bits - COMPR_CHUNK_MIN_BITS;
        llch->llch_flags = 0;
        llch->llch_compr_size = len;
-       llch->llch_compr_csum = 0;
-       llch->llch_uncompr_csum = 0;
+       llch->llch_uncompr_size = in_len;
        llch->llch_reserved = 0;
+       llch->llch_compr_csum = 0;
        llch->llch_hdr_csum = 0;
 
        *out_len = len + sizeof(*llch);

diff --git a/lustre/ofd/ofd_compress.c b/lustre/ofd/ofd_compress.c
index cfb87a5..9778693 100644 (file)
--- a/lustre/ofd/ofd_compress.c
+++ b/lustre/ofd/ofd_compress.c
@@ -117,6 +117,15 @@ static int decompress_chunk_in_lnb(const char *obd_name,
                       lnbs[lnb_start].lnb_file_offset, rc);
                GOTO(out, rc);
        }
+       /* uncompressed size is not set by older clients, but older clients
+        * reliably zero unused parts of the header, so we skip if it's zero
+        */
+       if (llch->llch_uncompr_size != 0 && dst_size != llch->llch_uncompr_size) {
+               CERROR("%s: Compressed chunk at %llu invalid, uncompressed size from disk %d disagrees with result of decompression %d, rc: %d\n",
+                      obd_name, lnbs[lnb_start].lnb_file_offset,
+                      llch->llch_uncompr_size, dst_size, rc);
+               GOTO(out, rc = -EUCLEAN);
+       }
        LASSERT(dst_size <= chunk_size);
 
        /* now that we've successfully decompressed this chunk, we copy

diff --git a/lustre/ptlrpc/wiretest.c b/lustre/ptlrpc/wiretest.c
index c04cc26..11a99f0 100644 (file)
--- a/lustre/ptlrpc/wiretest.c
+++ b/lustre/ptlrpc/wiretest.c
@@ -803,14 +803,14 @@ void lustre_assert_wire_constants(void)
                 (long long)(int)offsetof(struct ll_compr_hdr, llch_compr_size));
        LASSERTF((int)sizeof(((struct ll_compr_hdr *)0)->llch_compr_size) == 4, "found %lld\n",
                 (long long)(int)sizeof(((struct ll_compr_hdr *)0)->llch_compr_size));
-       LASSERTF((int)offsetof(struct ll_compr_hdr, llch_reserved) == 16, "found %lld\n",
+       LASSERTF((int)offsetof(struct ll_compr_hdr, llch_uncompr_size) == 16, "found %lld\n",
+                (long long)(int)offsetof(struct ll_compr_hdr, llch_uncompr_size));
+       LASSERTF((int)sizeof(((struct ll_compr_hdr *)0)->llch_uncompr_size) == 4, "found %lld\n",
+                (long long)(int)sizeof(((struct ll_compr_hdr *)0)->llch_uncompr_size));
+       LASSERTF((int)offsetof(struct ll_compr_hdr, llch_reserved) == 20, "found %lld\n",
                 (long long)(int)offsetof(struct ll_compr_hdr, llch_reserved));
        LASSERTF((int)sizeof(((struct ll_compr_hdr *)0)->llch_reserved) == 4, "found %lld\n",
                 (long long)(int)sizeof(((struct ll_compr_hdr *)0)->llch_reserved));
-       LASSERTF((int)offsetof(struct ll_compr_hdr, llch_uncompr_csum) == 20, "found %lld\n",
-                (long long)(int)offsetof(struct ll_compr_hdr, llch_uncompr_csum));
-       LASSERTF((int)sizeof(((struct ll_compr_hdr *)0)->llch_uncompr_csum) == 4, "found %lld\n",
-                (long long)(int)sizeof(((struct ll_compr_hdr *)0)->llch_uncompr_csum));
        LASSERTF((int)offsetof(struct ll_compr_hdr, llch_compr_csum) == 24, "found %lld\n",
                 (long long)(int)offsetof(struct ll_compr_hdr, llch_compr_csum));
        LASSERTF((int)sizeof(((struct ll_compr_hdr *)0)->llch_compr_csum) == 4, "found %lld\n",

diff --git a/lustre/utils/wirecheck.c b/lustre/utils/wirecheck.c
index 56c6295..48e070f 100644 (file)
--- a/lustre/utils/wirecheck.c
+++ b/lustre/utils/wirecheck.c
@@ -407,8 +407,8 @@ check_ll_compr_hdr(void)
        CHECK_BITFIELD(ll_compr_hdr, llch_comp_level);
        CHECK_BITFIELD(ll_compr_hdr, llch_chunk_log_bits);
        CHECK_MEMBER(ll_compr_hdr, llch_compr_size);
+       CHECK_MEMBER(ll_compr_hdr, llch_uncompr_size);
        CHECK_MEMBER(ll_compr_hdr, llch_reserved);
-       CHECK_MEMBER(ll_compr_hdr, llch_uncompr_csum);
        CHECK_MEMBER(ll_compr_hdr, llch_compr_csum);
        CHECK_MEMBER(ll_compr_hdr, llch_hdr_csum);
 

diff --git a/lustre/utils/wiretest.c b/lustre/utils/wiretest.c
index f387f3b..18126a3 100644 (file)
--- a/lustre/utils/wiretest.c
+++ b/lustre/utils/wiretest.c
@@ -839,14 +839,14 @@ void lustre_assert_wire_constants(void)
                 (long long)(int)offsetof(struct ll_compr_hdr, llch_compr_size));
        LASSERTF((int)sizeof(((struct ll_compr_hdr *)0)->llch_compr_size) == 4, "found %lld\n",
                 (long long)(int)sizeof(((struct ll_compr_hdr *)0)->llch_compr_size));
-       LASSERTF((int)offsetof(struct ll_compr_hdr, llch_reserved) == 16, "found %lld\n",
+       LASSERTF((int)offsetof(struct ll_compr_hdr, llch_uncompr_size) == 16, "found %lld\n",
+                (long long)(int)offsetof(struct ll_compr_hdr, llch_uncompr_size));
+       LASSERTF((int)sizeof(((struct ll_compr_hdr *)0)->llch_uncompr_size) == 4, "found %lld\n",
+                (long long)(int)sizeof(((struct ll_compr_hdr *)0)->llch_uncompr_size));
+       LASSERTF((int)offsetof(struct ll_compr_hdr, llch_reserved) == 20, "found %lld\n",
                 (long long)(int)offsetof(struct ll_compr_hdr, llch_reserved));
        LASSERTF((int)sizeof(((struct ll_compr_hdr *)0)->llch_reserved) == 4, "found %lld\n",
                 (long long)(int)sizeof(((struct ll_compr_hdr *)0)->llch_reserved));
-       LASSERTF((int)offsetof(struct ll_compr_hdr, llch_uncompr_csum) == 20, "found %lld\n",
-                (long long)(int)offsetof(struct ll_compr_hdr, llch_uncompr_csum));
-       LASSERTF((int)sizeof(((struct ll_compr_hdr *)0)->llch_uncompr_csum) == 4, "found %lld\n",
-                (long long)(int)sizeof(((struct ll_compr_hdr *)0)->llch_uncompr_csum));
        LASSERTF((int)offsetof(struct ll_compr_hdr, llch_compr_csum) == 24, "found %lld\n",
                 (long long)(int)offsetof(struct ll_compr_hdr, llch_compr_csum));
        LASSERTF((int)sizeof(((struct ll_compr_hdr *)0)->llch_compr_csum) == 4, "found %lld\n",