EX-7683 utils: update handling of compr level for lz4

The way lz4 compression level or acceleration factor is handled needs
to be adapted in order to match what is provided by the lz4 userspace
tool:
- any level between 0 and 2 is interpreted as the default lz4
  acceleration factor of 1;
- any level from 3 and up to 16 is interpreted as a compression level
  for internal lz4hc. Increasing the compression level trades CPU
  time for improved compression ratio;
- acceleration factor can be specified for the lz4fast compression
  type, from 1 to 26. This acceleration factor trades compression
  ratio for faster speed.

Signed-off-by: Sebastien Buisson <sbuisson@ddn.com>
Change-Id: I4711217c1a6601f29f78d262567da5998f657fc9
Reviewed-on: https://review.whamcloud.com/c/ex/lustre-release/+/51380
Tested-by: jenkins <devops@whamcloud.com>
Tested-by: Maloo <maloo@whamcloud.com>
Reviewed-by: Andreas Dilger <adilger@whamcloud.com>

diff --git a/lustre/include/lustre/lustreapi.h b/lustre/include/lustre/lustreapi.h
index 51e0124..44ccbd4 100644 (file)
--- a/lustre/include/lustre/lustreapi.h
+++ b/lustre/include/lustre/lustreapi.h
@@ -1159,37 +1159,8 @@ static const struct comp_flag_name {
 /* map compression algorithm compress level to 4bits value */
 #ifndef LZ4_ACCELERATION_DEFAULT
 #define LZ4_ACCELERATION_DEFAULT 1
-
-/* Level as stored in the Lustre file layout is limited to 4 bits, i.e.
- * between 0 and 15. But for lz4, acceleration factor can be fine tuned
- * with each successive value providing roughly +~3% to speed.
- * So we map the provided level to the lz4 acceleration factor by keeping 1-9
- * as-is, and then going by steps of 3 so 12 15 18 21 24 27.
- *
- */
-static inline __u8 to_lz4_level(__u8 level)
-{
-       if (level == LZ4_ACCELERATION_DEFAULT)
-               return level;
-       if (level < 10)
-               return level;
-       return (level - 9) * 3 + 9;
-}
-
 #endif
 
-/* if user does not set level (-1), it would be set to the default level */
-static inline __u8 from_lz4_level(int level)
-{
-       if (level == -1 )
-               return LZ4_ACCELERATION_DEFAULT;
-       if (level < 10)
-               return level;
-       if ((level - 9) / 3 + 9 > 0xf)
-               return 0xf;
-       return (level - 9) / 3 + 9;
-}
-
 #ifndef LZ4HC_DEFAULT_CLEVEL
 #define LZ4HC_DEFAULT_CLEVEL 9
 
@@ -1200,25 +1171,51 @@ static inline __u8 from_lz4_level(int level)
  */
 static inline __u8 to_lz4hc_level(__u8 level)
 {
-       if (level == LZ4HC_DEFAULT_CLEVEL)
-               return level;
        return level + 1;
 }
 
 #endif
 
+#ifndef LZ4HC_MIN_CLEVEL
+#define LZ4HC_MIN_CLEVEL 3
+#endif
+
 /* if user does not set level (-1), it would be set to the default level */
 static inline __u8 from_lz4hc_level(int level)
 {
        if (level == -1)
-               return LZ4HC_DEFAULT_CLEVEL;
-       if (level == 0)
-               return 0;
+               return LZ4_ACCELERATION_DEFAULT - 1;
        if ((level - 1) > 0xf)
                return 0xf;
        return level - 1;
 }
 
+/* Level as stored in the Lustre file layout is limited to 4 bits, i.e.
+ * between 0 and 15. But for lz4fast, acceleration factor can be fine tuned
+ * with each successive value providing roughly +~3% to speed.
+ * So we map the provided level to the lz4 acceleration factor by keeping 1-9
+ * as-is, and then going by steps of 3 so 12 15 18 21 24 27.
+ * And to mock lz4 command line utility behavior, the user provided factor is
+ * actually applied internally as 'value + 1'.
+ */
+static inline __u8 to_lz4fast_lvl(__u8 level)
+{
+       if (level < 10)
+               return level - 1;
+       return (level - 9) * 3 + 9 - 1;
+}
+
+static inline __u8 from_lz4fast_lvl(int level)
+{
+       if (level <= 0)
+               return LZ4_ACCELERATION_DEFAULT + 1;
+       if (level < 10)
+               return level + 1;
+       if ((level - 9) / 3 + 9 + 1 > 0xf)
+               return 0xf;
+       return (level - 9) / 3 + 9 + 1;
+}
+
 static inline __u8 from_gzip_level(int level)
 {
        if (level == -1)
@@ -1239,13 +1236,13 @@ static const struct compr_type_name {
        from_compress_level     ctn_from_compr_level;
        to_compress_level       ctn_to_compr_level;
 } compr_type_table[] = {
-       { LL_COMPR_TYPE_NONE,   "none",  NULL, NULL },
-       { LL_COMPR_TYPE_FAST,   "fast",  NULL, NULL },
-       { LL_COMPR_TYPE_BEST,   "best",  NULL, NULL },
-       { LL_COMPR_TYPE_GZIP,   "gzip",  from_gzip_level, NULL },
-       { LL_COMPR_TYPE_LZ4,    "lz4",   from_lz4_level,   to_lz4_level },
-       { LL_COMPR_TYPE_LZ4HC,  "lz4hc", from_lz4hc_level, to_lz4hc_level },
-       { LL_COMPR_TYPE_LZO,    "lzo",   NULL, NULL },
+       { LL_COMPR_TYPE_NONE,    "none",    NULL,             NULL },
+       { LL_COMPR_TYPE_FAST,    "fast",    NULL,             NULL },
+       { LL_COMPR_TYPE_BEST,    "best",    NULL,             NULL },
+       { LL_COMPR_TYPE_GZIP,    "gzip",    from_gzip_level,  NULL },
+       { LL_COMPR_TYPE_LZ4FAST, "lz4fast", from_lz4fast_lvl, to_lz4fast_lvl },
+       { LL_COMPR_TYPE_LZ4HC,   "lz4",     from_lz4hc_level, to_lz4hc_level },
+       { LL_COMPR_TYPE_LZO,     "lzo",     NULL,             NULL },
 };
 
 /* HSM component flags table */

diff --git a/lustre/include/lustre_crypto.h b/lustre/include/lustre_crypto.h
index f737c18..fbe7c5e 100644 (file)
--- a/lustre/include/lustre_crypto.h
+++ b/lustre/include/lustre_crypto.h
@@ -41,6 +41,10 @@ void ll_sbi_set_name_encrypt(struct ll_sb_info *sbi, bool set);
 /* sizeof(struct fscrypt_context_v2) = 40 */
 #define LLCRYPT_ENC_CTX_SIZE 40
 
+#ifndef LZ4HC_MIN_CLEVEL
+#define LZ4HC_MIN_CLEVEL 3
+#endif
+
 /* Only the lower bits of the crt_flags field (u32) of struct crypto_tfm are
  * currently used. So use the top 4 bits to store the compression level.
  */

diff --git a/lustre/include/uapi/linux/lustre/lustre_user.h b/lustre/include/uapi/linux/lustre/lustre_user.h
index 446af93..77057cb 100644 (file)
--- a/lustre/include/uapi/linux/lustre/lustre_user.h
+++ b/lustre/include/uapi/linux/lustre/lustre_user.h
@@ -2927,7 +2927,7 @@ enum ll_compr_type {
        LL_COMPR_TYPE_FAST      = 1,
        LL_COMPR_TYPE_BEST      = 2,
        LL_COMPR_TYPE_GZIP      = 3,
-       LL_COMPR_TYPE_LZ4       = 4,
+       LL_COMPR_TYPE_LZ4FAST   = 4,
        LL_COMPR_TYPE_LZ4HC     = 5,
        LL_COMPR_TYPE_LZO       = 6,
        LL_COMPR_TYPE_MAX,

diff --git a/lustre/osc/osc_compress.c b/lustre/osc/osc_compress.c
index e41b9b6..804ac0d 100644 (file)
--- a/lustre/osc/osc_compress.c
+++ b/lustre/osc/osc_compress.c
@@ -98,9 +98,9 @@ static int alloc_comp(enum ll_compr_type *type, unsigned int lvl,
        if (*type == LL_COMPR_TYPE_BEST)
                *type = LL_COMPR_TYPE_GZIP;
        else if (*type == LL_COMPR_TYPE_FAST)
-               *type = LL_COMPR_TYPE_LZ4;
+               *type = LL_COMPR_TYPE_LZ4FAST;
 
-       *cc = crypto_alloc_comp(crypto_name_from_type(*type), 0, 0);
+       *cc = crypto_alloc_comp(crypto_name_from_type(*type, lvl), 0, 0);
        if (IS_ERR(*cc)) {
                int ret = PTR_ERR(*cc);
                CERROR("Cannot initialize compressor %i, error %i\n", *type,

diff --git a/lustre/osc/osc_internal.h b/lustre/osc/osc_internal.h
index f5f2e6a..ee068f1 100644 (file)
--- a/lustre/osc/osc_internal.h
+++ b/lustre/osc/osc_internal.h
@@ -224,16 +224,19 @@ static inline void osc_set_io_portal(struct ptlrpc_request *req)
                req->rq_request_portal = OST_IO_PORTAL;
 }
 
-static inline const char *crypto_name_from_type(enum ll_compr_type type)
+static inline const char *crypto_name_from_type(enum ll_compr_type type,
+                                               unsigned int level)
 {
        switch (type) {
        case LL_COMPR_TYPE_NONE:
                return "none";
        case LL_COMPR_TYPE_GZIP:
                return "deflate";
-       case LL_COMPR_TYPE_LZ4:
+       case LL_COMPR_TYPE_LZ4FAST:
                return "lz4";
        case LL_COMPR_TYPE_LZ4HC:
+               if (level < LZ4HC_MIN_CLEVEL - 1)
+                       return "lz4";
                return "lz4hc";
        case LL_COMPR_TYPE_LZO:
                return "lzo";

diff --git a/lustre/ptlrpc/wiretest.c b/lustre/ptlrpc/wiretest.c
index 9d1680f..b5b8c3f 100644 (file)
--- a/lustre/ptlrpc/wiretest.c
+++ b/lustre/ptlrpc/wiretest.c
@@ -828,8 +828,8 @@ void lustre_assert_wire_constants(void)
                 (long long)LL_COMPR_TYPE_BEST);
        LASSERTF(LL_COMPR_TYPE_GZIP == 3, "found %lld\n",
                 (long long)LL_COMPR_TYPE_GZIP);
-       LASSERTF(LL_COMPR_TYPE_LZ4 == 4, "found %lld\n",
-                (long long)LL_COMPR_TYPE_LZ4);
+       LASSERTF(LL_COMPR_TYPE_LZ4FAST == 4, "found %lld\n",
+                (long long)LL_COMPR_TYPE_LZ4FAST);
        LASSERTF(LL_COMPR_TYPE_LZ4HC == 5, "found %lld\n",
                 (long long)LL_COMPR_TYPE_LZ4HC);
        LASSERTF(LL_COMPR_TYPE_LZO == 6, "found %lld\n",
@@ -6362,8 +6362,8 @@ void lustre_assert_wire_constants(void)
                 (long long)LL_COMPR_TYPE_BEST);
        LASSERTF(LL_COMPR_TYPE_GZIP == 3, "found %lld\n",
                 (long long)LL_COMPR_TYPE_GZIP);
-       LASSERTF(LL_COMPR_TYPE_LZ4 == 4, "found %lld\n",
-                (long long)LL_COMPR_TYPE_LZ4);
+       LASSERTF(LL_COMPR_TYPE_LZ4FAST == 4, "found %lld\n",
+                (long long)LL_COMPR_TYPE_LZ4FAST);
        LASSERTF(LL_COMPR_TYPE_LZ4HC == 5, "found %lld\n",
                 (long long)LL_COMPR_TYPE_LZ4HC);
        LASSERTF(LL_COMPR_TYPE_LZO == 6, "found %lld\n",

diff --git a/lustre/tests/sanity-pfl.sh b/lustre/tests/sanity-pfl.sh
index bbf9db7..572b6d3 100644 (file)
--- a/lustre/tests/sanity-pfl.sh
+++ b/lustre/tests/sanity-pfl.sh
@@ -2466,7 +2466,10 @@ test_100b() {
                skip "Need MDS >= 2.14.0.88 for compression support"
 
        local tf=$DIR/$tdir/$tfile
-       local type="gzip lz4 lz4hc"
+       # putting lz4 twice:
+       # - first one will have lvl 1, which corresponds to normal lz4
+       # - second one will have lvl 3, which corresponds to lz4hc
+       local type="lz4 gzip lz4 lz4fast"
        # lzo does not grok a compression level, add here other such algs
        local type_nolvl="lzo"
        local l1=1
@@ -2585,23 +2588,23 @@ test_100c() {
                $LFS getstripe $tf
                error "gzip max compress level $lvl != 9"
        }
-       # lz4 max compress level 27
+       # lz4fast max acceleration factor 26
        rm -f $tf
-       $LFS setstripe -Eeof -Z lz4:28 $tf ||
-               error "failed setting lz4 type, level 28"
+       $LFS setstripe -Eeof -Z lz4fast:27 $tf ||
+               error "failed setting lz4fast type, level 26"
        lvl=$($LFS getstripe --compr-level $tf)
-       (( $lvl == 27 )) || {
+       (( $lvl == 26 )) || {
                $LFS getstripe $tf
-               error "gzip max compress level $lvl != 27"
+               error "lz4fast max acceleration factor $lvl != 26"
        }
-       # lz4hc max compress level 16
+       # lz4 max compress level 16
        rm -f $tf
-       $LFS setstripe -Eeof -Z lz4hc:28 $tf ||
-               error "failed setting lz4hc type, level 28"
+       $LFS setstripe -Eeof -Z lz4:17 $tf ||
+               error "failed setting lz4 type, level 17"
        lvl=$($LFS getstripe --compr-level $tf)
        (( $lvl == 16 )) || {
                $LFS getstripe $tf
-               error "gzip max compress level $lvl != 16"
+               error "lz4 max compress level $lvl != 16"
        }
 
        # test compress chunk size
@@ -2644,7 +2647,10 @@ test_100d() {
                skip "Need MDS >= 2.14.0.88 for compression support"
 
        local tf=$DIR/$tdir/$tfile
-       local type="gzip lz4 lz4hc"
+       # putting lz4 twice:
+       # - first one will have lvl 1, which corresponds to normal lz4
+       # - second one will have lvl 3, which corresponds to lz4hc
+       local type="lz4 gzip lz4 lz4fast"
        local type_num=$(wc -w <<< $type)
        # lzo does not grok a compression level, add here other such algs
        local type_nolvl="lzo"
@@ -2668,12 +2674,13 @@ test_100d() {
 
                [[ "$type_nolvl" =~ "$tp" ]] || has_level="y"
                $LFS setstripe -Eeof -Z $tp${has_level:+":$lvl"} \
-                       --compress-chunk=${cs} ${tf}_${tp} ||
-               error "set a compress component in $tf_${tp} failed"
-               $LFS setstripe -Eeof --comp-flags=nocompr ${tf}_nocompr_${tp} ||
-                       error "set a nocompr component in $tf_nocompr_${tp} failed"
-               $LFS setstripe -Eeof ${tf}_not_${tp} ||
-                       error "set a plain file ${tf}_not_${tp} failed"
+                       --compress-chunk=${cs} ${tf}_${tp}_${lvl}_${cs} ||
+               error "can't set compress component in ${tf}_${tp}_${lvl}_${cs}"
+               $LFS setstripe -Eeof --comp-flags=nocompr \
+                       ${tf}_nocompr_${tp}_${lvl}_${cs} ||
+               error "can't set nocompr in ${tf}_nocompr_${tp}_${lvl}_${cs}"
+               $LFS setstripe -Eeof ${tf}_not_${tp}_${lvl}_${cs} ||
+                       error "cant set plain file ${tf}_not_${tp}_${lvl}_${cs}"
 
                lvl=$((lvl + 1))
                cs=$((cs * 2))
@@ -2702,8 +2709,15 @@ test_100d() {
 
        for tp in $type $type_nolvl; do
                flg_opts="--compr-type=$tp"
+               # lz4 appears twice in the type list
+               if [ "$tp" == "lz4" ]; then
+                       expect=2
+               else
+                       expect=1
+               fi
                found=$($LFS find $flg_opts $DIR/$tdir | wc -l)
-               (( found == 1 )) || error "found $found $tp compress file != 1"
+               (( found == expect )) ||
+                       error "found $found $tp compress file != $expect"
        done
        flg_opts="--compr-level=0"
        found=$($LFS find $flg_opts $DIR/$tdir | wc -l)
@@ -2730,15 +2744,15 @@ test_100d() {
        done
 
        # test combination of compr type and compr level
-       flg_opts="--compr-type=gzip --compr-level=1"
+       flg_opts="--compr-type=gzip --compr-level=2"
        found=$($LFS find $flg_opts $DIR/$tdir | wc -l)
        (( $found == 1 )) ||
                error "found $found compress type gzip level 1 files != 1"
-       flg_opts="--compr-type=gzip ! --compr-level=1"
+       flg_opts="--compr-type=gzip ! --compr-level=2"
        found=$($LFS find $flg_opts $DIR/$tdir | wc -l)
        (( $found == 0 )) ||
                error "found $found compress type gzip ! level 1 files != 0"
-       flg_opts="--compr-type=gzip --compr-level=2"
+       flg_opts="--compr-type=gzip --compr-level=1"
        found=$($LFS find $flg_opts $DIR/$tdir | wc -l)
        (( $found == 0 )) ||
                error "found $found compress type gzip level 2 files != 0"

diff --git a/lustre/tests/sanity.sh b/lustre/tests/sanity.sh
index 27e53fd..c0fdbc7 100755 (executable)
--- a/lustre/tests/sanity.sh
+++ b/lustre/tests/sanity.sh
@@ -27601,8 +27601,8 @@ compress_type() {
 compress_content() {
        t1="lzo" l1=5 c1="64k" compress_type $1
        t1="lzo" l1=5 c1="128k" compress_type $1
-       t1="lz4" l1=5 c1="64k" compress_type $1
-       t1="lz4hc" l1=5 c1="64k" compress_type $1
+       t1="lz4" l1=1 c1="64k" compress_type $1
+       t1="lz4" l1=9 c1="64k" compress_type $1
        t1="gzip" l1=5 c1="64k" compress_type $1
 }
 

diff --git a/lustre/utils/liblustreapi.c b/lustre/utils/liblustreapi.c
index 7575ba8..3fa86f4 100644 (file)
--- a/lustre/utils/liblustreapi.c
+++ b/lustre/utils/liblustreapi.c
@@ -3158,10 +3158,10 @@ static int mapback_compress_level(enum ll_compr_type type, __u8 level)
 
        for (i = 0; i < ARRAY_SIZE(compr_type_table); i++) {
                if (compr_type_table[i].ctn_compr_type == type) {
-                      known_type = true;
-                      if (compr_type_table[i].ctn_to_compr_level != NULL)
-                              return compr_type_table[i].ctn_to_compr_level(
-                                                                       level);
+                       known_type = true;
+                       if (compr_type_table[i].ctn_to_compr_level != NULL)
+                               return compr_type_table[i].ctn_to_compr_level(
+                                                                        level);
                }
        }
 

diff --git a/lustre/utils/liblustreapi_layout.c b/lustre/utils/liblustreapi_layout.c
index 4db23c1..a5eeb0b 100644 (file)
--- a/lustre/utils/liblustreapi_layout.c
+++ b/lustre/utils/liblustreapi_layout.c
@@ -1516,9 +1516,16 @@ int llapi_parse_compress_type(const char *optarg, unsigned int *type,
        if (!found)
                return -1;
 
+       if (!level)
+               return 0;
+
        if (ptr)
                *level = strtoul(ptr, NULL, 0);
 
+       if (*type == LL_COMPR_TYPE_LZ4HC && *level < LZ4HC_MIN_CLEVEL)
+               /* if not an HC level, just use standard lz4 default */
+               *level = LZ4_ACCELERATION_DEFAULT;
+
        return 0;
 }
 

diff --git a/lustre/utils/wirecheck.c b/lustre/utils/wirecheck.c
index 61511d4..30af5c6 100644 (file)
--- a/lustre/utils/wirecheck.c
+++ b/lustre/utils/wirecheck.c
@@ -425,7 +425,7 @@ check_ll_compr_hdr(void)
        CHECK_VALUE(LL_COMPR_TYPE_FAST);
        CHECK_VALUE(LL_COMPR_TYPE_BEST);
        CHECK_VALUE(LL_COMPR_TYPE_GZIP);
-       CHECK_VALUE(LL_COMPR_TYPE_LZ4);
+       CHECK_VALUE(LL_COMPR_TYPE_LZ4FAST);
        CHECK_VALUE(LL_COMPR_TYPE_LZ4HC);
        CHECK_VALUE(LL_COMPR_TYPE_LZO);
 

diff --git a/lustre/utils/wiretest.c b/lustre/utils/wiretest.c
index 4384042..d0b9154 100644 (file)
--- a/lustre/utils/wiretest.c
+++ b/lustre/utils/wiretest.c
@@ -864,8 +864,8 @@ void lustre_assert_wire_constants(void)
                 (long long)LL_COMPR_TYPE_BEST);
        LASSERTF(LL_COMPR_TYPE_GZIP == 3, "found %lld\n",
                 (long long)LL_COMPR_TYPE_GZIP);
-       LASSERTF(LL_COMPR_TYPE_LZ4 == 4, "found %lld\n",
-                (long long)LL_COMPR_TYPE_LZ4);
+       LASSERTF(LL_COMPR_TYPE_LZ4FAST == 4, "found %lld\n",
+                (long long)LL_COMPR_TYPE_LZ4FAST);
        LASSERTF(LL_COMPR_TYPE_LZ4HC == 5, "found %lld\n",
                 (long long)LL_COMPR_TYPE_LZ4HC);
        LASSERTF(LL_COMPR_TYPE_LZO == 6, "found %lld\n",
@@ -6397,8 +6397,8 @@ void lustre_assert_wire_constants(void)
                 (long long)LL_COMPR_TYPE_BEST);
        LASSERTF(LL_COMPR_TYPE_GZIP == 3, "found %lld\n",
                 (long long)LL_COMPR_TYPE_GZIP);
-       LASSERTF(LL_COMPR_TYPE_LZ4 == 4, "found %lld\n",
-                (long long)LL_COMPR_TYPE_LZ4);
+       LASSERTF(LL_COMPR_TYPE_LZ4FAST == 4, "found %lld\n",
+                (long long)LL_COMPR_TYPE_LZ4FAST);
        LASSERTF(LL_COMPR_TYPE_LZ4HC == 5, "found %lld\n",
                 (long long)LL_COMPR_TYPE_LZ4HC);
        LASSERTF(LL_COMPR_TYPE_LZO == 6, "found %lld\n",