[\fB--stripe-count\fR|\fB-c\fR]
[\fB--stripe-index\fR|\fB-i\fR]
[\fB--stripe-size\fR|\fB-S\fR] [\fB--mirror-count\fR|\fB-N\fR]
+ [\fB--compress-type\fR|\fB--compr-type\fR]
+[\fB--compress-level\fR|\fB--compr-level\fR]
+ [\fB--compress-chunk\fR|\fB--compr-chunk\fR]
[[\fB!\fR] \fB--mirror-index\fR=[\fB+-\fR]\fI<index>\fR | [\fB!\fR] \fB--mirror-id\fR=[\fB+-\fR]\fI<id>\fR]
[\fB--verbose\fR|\fB-v\fR]
[\fB--yaml\fR|\fB-y\fR]
.BR --component-count ,
.BR --component-start ,
.BR --component-end ,
-.BR --pool
-or
+.BR --pool,
+.BR --compress-type,
+.BR --compress-level,
+.BR --compress-chunk,
.BR --mirror-index
or
.BR --mirror-id
Print the extension size in bytes. For composite files this is the extension
size of the first extension component.
.TP
+.BR --compress-type | --compr-type
+Print only the compress type of components in the file's layout if available.
+.TP
+.BR --compress-level | --compr-level
+Print only the compress level of components in the file's layout if available.
+.TP
+.BR --compress-chunk | --compr-chunk
+Print only the compress chunk size in KiB of components in the file's layout if available.
+.TP
.BR --verbose | -v
Also print the layout magic, FID sequence, FID object ID, and FID, in
addition to the normally-printed attributes.
.BR G ibibytes.
The
.I stripe_size
-must be a multiple of 64KiB. Values below 4096 are assumed to be in KiB units.
+must be a multiple of 64KiB and a multiple of the
+.BR compress-chunk
+size, if set. Values below 4096 are assumed to be in KiB units.
.TP
.B -i\fR, \fB--stripe-index \fR<\fIstart_ost_index\fR>
The OST index (starting at 0) on which to start striping for this file. A
for details.
.RE
.TP
+.B -Z\fR, \fB--compress \fR<\fItype\fR>[:<\fIlevel\fR>]
+Used to specify the compression algorithm <\fItype\fR> and compress
+<\fIlevel\fR> to be used in the component. The compress level showed in
+\fBlfs getstripe\fR could be not exactly the one input here.
+.TP
+.B --compress-chunk\fR=<\fIsize\fR>
+Used to specify the data chunk size in KiB used by compression algorithm,
+the value will be adjusted to power-of-two multiples of the base 64KiB.
+The
+.B stripe-size
+must be a multiple of the
+.B compress-chunk
+size.
+.TP
.B -o\fR, \fB--ost \fR<\fIost_indices\fR>
Used to specify the exact stripe layout on the file system. \fIost_indices\fR
is a list of OSTs referenced by their indices, which are specified in decimal
VERBOSE_EXT_SIZE = 0x40000,
VERBOSE_INHERIT = 0x80000,
VERBOSE_INHERIT_RR = 0x100000,
+ VERBOSE_COMPRESS_TYPE = 0x200000,
+ VERBOSE_COMPRESS_LEVEL = 0x400000,
+ VERBOSE_COMPRESS_CHUNK = 0x800000,
VERBOSE_DEFAULT = VERBOSE_STRIPE_COUNT | VERBOSE_STRIPE_SIZE |
VERBOSE_STRIPE_OFFSET | VERBOSE_POOL |
VERBOSE_OBJID | VERBOSE_GENERATION |
VERBOSE_COMP_START | VERBOSE_COMP_END |
VERBOSE_COMP_ID | VERBOSE_MIRROR_COUNT |
VERBOSE_MIRROR_ID | VERBOSE_EXT_SIZE |
- VERBOSE_INHERIT | VERBOSE_INHERIT_RR
+ VERBOSE_INHERIT | VERBOSE_INHERIT_RR |
+ VERBOSE_COMPRESS_TYPE |
+ VERBOSE_COMPRESS_LEVEL |
+ VERBOSE_COMPRESS_CHUNK,
};
/* Compatibility with original names */
#define VERBOSE_SIZE VERBOSE_STRIPE_SIZE
int llapi_get_obd_count(char *mnt, int *count, int is_mdt);
int llapi_parse_size(const char *optarg, unsigned long long *size,
unsigned long long *size_units, int bytes_spec);
+int llapi_parse_compress_type(const char *optarg, unsigned int *type,
+ int *level);
int llapi_search_mounts(const char *pathname, int index, char *mntdir,
char *fsname);
int llapi_search_fsname(const char *pathname, char *fsname);
#define LLAPI_LAYOUT_MDT 2ULL
#define LLAPI_LAYOUT_OVERSTRIPING 4ULL
#define LLAPI_LAYOUT_FOREIGN 8ULL
+#define LLAPI_LAYOUT_COMPRESS 0x10ULL
/**
* The layout includes a specific set of OSTs on which to allocate.
int llapi_layout_stripe_size_set(struct llapi_layout *layout, uint64_t size);
+/**
+ * Set the compress parameters of \a layout.
+ *
+ * \retval 0 Success.
+ * \retval -1 Invalid argument.
+ */
+int llapi_layout_compress_set(struct llapi_layout *layout,
+ enum ll_compr_type type, int level,
+ __u32 chunk_size, unsigned long long stripe_size);
+
/******************** Extension Size ********************/
/**
{ LCME_FL_NOCOMPR, "nocompr" },
};
+
+/* map compression algorithm compress level to 4bits value */
+#ifndef LZ4_ACCELERATION_DEFAULT
+#define LZ4_ACCELERATION_DEFAULT 1
+
+/* Level as stored in the Lustre file layout is limited to 4 bits, i.e.
+ * between 0 and 15. But for lz4, acceleration factor can be fine tuned
+ * with each successive value providing roughly +~3% to speed.
+ * So we map the provided level to the lz4 acceleration factor by keeping 1-9
+ * as-is, and then going by steps of 3 so 12 15 18 21 24 27.
+ *
+ */
+static inline __u8 to_lz4_level(__u8 level)
+{
+ if (level == LZ4_ACCELERATION_DEFAULT)
+ return level;
+ if (level < 10)
+ return level;
+ return (level - 9) * 3 + 9;
+}
+
+#endif
+
+/* if user does not set level (-1), it would be set to the default level */
+static inline __u8 from_lz4_level(int level)
+{
+ if (level == -1 )
+ return LZ4_ACCELERATION_DEFAULT;
+ if (level < 10)
+ return level;
+ if ((level - 9) / 3 + 9 > 0xf)
+ return 0xf;
+ return (level - 9) / 3 + 9;
+}
+
+#ifndef LZ4HC_DEFAULT_CLEVEL
+#define LZ4HC_DEFAULT_CLEVEL 9
+
+/* Level as stored in the Lustre file layout is limited to 4 bits, i.e.
+ * between 0 and 15.
+ * So we map the provided level to the lz4hc compression level
+ * simply by adding 1.
+ */
+static inline __u8 to_lz4hc_level(__u8 level)
+{
+ if (level == LZ4HC_DEFAULT_CLEVEL)
+ return level;
+ return level + 1;
+}
+
+#endif
+
+/* if user does not set level (-1), it would be set to the default level */
+static inline __u8 from_lz4hc_level(int level)
+{
+ if (level == -1)
+ return LZ4HC_DEFAULT_CLEVEL;
+ if (level == 0)
+ return 0;
+ if ((level - 1) > 0xf)
+ return 0xf;
+ return level - 1;
+}
+
+static inline __u8 from_gzip_level(int level)
+{
+ if (level == -1)
+ return 6;
+ if (level == 0)
+ return 1;
+ if (level > 9)
+ return 9;
+ return level;
+}
+
+typedef __u8 (*from_compress_level)(int level);
+typedef __u8 (*to_compress_level)(__u8 level);
+
+static const struct compr_type_name {
+ enum ll_compr_type ctn_compr_type;
+ const char *ctn_name;
+ from_compress_level ctn_from_compr_level;
+ to_compress_level ctn_to_compr_level;
+} compr_type_table[] = {
+ { LL_COMPR_TYPE_NONE, "none", NULL, NULL },
+ { LL_COMPR_TYPE_FAST, "fast", NULL, NULL },
+ { LL_COMPR_TYPE_BEST, "best", NULL, NULL },
+ { LL_COMPR_TYPE_GZIP, "gzip", from_gzip_level, NULL },
+ { LL_COMPR_TYPE_LZ4, "lz4", from_lz4_level, to_lz4_level },
+ { LL_COMPR_TYPE_LZ4HC, "lz4hc", from_lz4hc_level, to_lz4hc_level },
+ { LL_COMPR_TYPE_LZO, "lzo", NULL, NULL },
+};
+
/* HSM component flags table */
static const struct hsm_flag_name {
enum hsm_states hfn_flag;
};
#define OBD_MAX_FIDS_IN_ARRAY 4096
+/**
+ * component compress algorithm type
+ */
+enum ll_compr_type {
+ LL_COMPR_TYPE_NONE = 0,
+ LL_COMPR_TYPE_FAST = 1,
+ LL_COMPR_TYPE_BEST = 2,
+ LL_COMPR_TYPE_GZIP = 3,
+ LL_COMPR_TYPE_LZ4 = 4,
+ LL_COMPR_TYPE_LZ4HC = 5,
+ LL_COMPR_TYPE_LZO = 6,
+ LL_COMPR_TYPE_MAX
+};
+
#if defined(__cplusplus)
}
#endif
*stripe_count = 1;
}
+/**
+ * limit 2^log_bits to less than stripe_size and given stripe_size is already
+ * a multiple fo 64KiB, stripe_size must be a multiple of chunk size.
+ */
+static inline void
+lod_adjust_compr_chunk_size(__u16 *log_bits, __u32 stripe_size)
+{
+ while ((1 << *log_bits) > stripe_size ||
+ stripe_size & ((1 << *log_bits) - 1))
+ (*log_bits)--;
+}
+
void lod_fix_desc(struct lov_desc *desc);
void lod_fix_desc_qos_maxage(__u32 *val);
void lod_fix_desc_pattern(__u32 *val);
lcme->lcme_offset = cpu_to_le32(offset);
if (lod_comp->llc_pattern & LOV_PATTERN_COMPRESS) {
+ lcme->lcme_flags |= cpu_to_le32(LCME_FL_COMPRESS);
lcme->lcme_compr_type = lod_comp->llc_compr_type;
lcme->lcme_compr_lvl = lod_comp->llc_compr_lvl;
lcme->lcme_compr_chunk_log_bits =
lod_comp->llc_stripe_count = le16_to_cpu(lmm->lmm_stripe_count);
lod_comp->llc_layout_gen = le16_to_cpu(lmm->lmm_layout_gen);
+ if (lo->ldo_is_composite && (pattern & LOV_PATTERN_COMPRESS)) {
+ lod_comp->llc_flags |= LCME_FL_COMPRESS;
+ lod_comp->llc_compr_type =
+ comp_v1->lcm_entries[i].lcme_compr_type;
+ lod_comp->llc_compr_lvl =
+ comp_v1->lcm_entries[i].lcme_compr_lvl;
+ lod_comp->llc_compr_chunk_log_bits =
+ comp_v1->lcm_entries[i].lcme_compr_chunk_log_bits +
+ 16;
+ }
+
if (lmm->lmm_magic == cpu_to_le32(LOV_MAGIC_V3)) {
struct lov_mds_md_v3 *v3 = (struct lov_mds_md_v3 *)lmm;
lod_comp->llc_stripe_size = v1->lmm_stripe_size;
lod_adjust_stripe_info(lod_comp, desc, 0);
+ lod_comp->llc_compr_type =
+ comp_v1->lcm_entries[i].lcme_compr_type;
+ lod_comp->llc_compr_lvl =
+ comp_v1->lcm_entries[i].lcme_compr_lvl;
+ lod_comp->llc_compr_chunk_log_bits =
+ comp_v1->lcm_entries[i].lcme_compr_chunk_log_bits + 16;
+
+ lod_adjust_compr_chunk_size(&lod_comp->llc_compr_chunk_log_bits,
+ lod_comp->llc_stripe_size);
+
if (v1->lmm_magic == LOV_USER_MAGIC_V3) {
v3 = (struct lov_user_md_v3 *) v1;
if (v3->lmm_pool_name[0] != '\0') {
RETURN(-EUCLEAN);
}
if (flags & LCME_FL_NOCOMPR &&
- lod_comp->llc_compr_type != 0) {
+ lod_comp->llc_compr_type !=
+ LL_COMPR_TYPE_NONE) {
mutex_unlock(
&lo->ldo_layout_mutex);
RETURN(-EINVAL);
for (i = 0; i < entry_count; i++) {
struct lod_layout_component *llc = &lds->lds_def_comp_entries[i];
+ struct lov_comp_md_entry_v1 *lcme = NULL;
const char *pool;
/*
v1 = (struct lov_user_md *)((char *)lcm +
lcm->lcm_entries[i].lcme_offset);
+ lcme = &lcm->lcm_entries[i];
if (want_composite) {
- llc->llc_extent = lcm->lcm_entries[i].lcme_extent;
+ llc->llc_extent = lcme->lcme_extent;
/* We only inherit certain flags from the layout */
- llc->llc_flags = lcm->lcm_entries[i].lcme_flags &
- LCME_TEMPLATE_FLAGS;
+ llc->llc_flags = lcme->lcme_flags &
+ LCME_TEMPLATE_FLAGS;
}
}
llc->llc_stripe_size = v1->lmm_stripe_size;
llc->llc_stripe_offset = v1->lmm_stripe_offset;
llc->llc_pattern = v1->lmm_pattern;
+ if (lcme) {
+ llc->llc_compr_type = lcme->lcme_compr_type;
+ llc->llc_compr_lvl = lcme->lcme_compr_lvl;
+ llc->llc_compr_chunk_log_bits =
+ lcme->lcme_compr_chunk_log_bits + 16;
+ }
if (append_stripe_count != 0 || append_pool != NULL)
llc->llc_pattern = LOV_PATTERN_RAID0;
lod_comp->llc_stripe_size = le32_to_cpu(v1->lmm_stripe_size);
lod_comp->llc_stripe_count = le16_to_cpu(v1->lmm_stripe_count);
lod_comp->llc_layout_gen = le16_to_cpu(v1->lmm_layout_gen);
+
+ if (mo->ldo_is_composite &&
+ lov_pattern(lod_comp->llc_pattern) & LOV_PATTERN_COMPRESS) {
+ lod_comp->llc_flags |= LCME_FL_COMPRESS;
+ lod_comp->llc_compr_type =
+ comp_v1->lcm_entries[i].lcme_compr_type;
+ lod_comp->llc_compr_lvl =
+ comp_v1->lcm_entries[i].lcme_compr_lvl;
+ lod_comp->llc_compr_chunk_log_bits =
+ comp_v1->lcm_entries[i].lcme_compr_chunk_log_bits +
+ 16;
+ }
/**
* The stripe_offset of an uninit-ed component is stored in
* the lmm_layout_gen
}
if (lov_pattern(lod_comp->llc_pattern) & LOV_PATTERN_COMPRESS) {
+ lod_comp->llc_flags |= LCME_FL_COMPRESS;
lod_comp->llc_compr_type = lcme->lcme_compr_type;
lod_comp->llc_compr_lvl = lcme->lcme_compr_lvl;
lod_comp->llc_compr_chunk_log_bits =
lcme->lcme_compr_chunk_log_bits + 16;
+ lod_adjust_compr_chunk_size(
+ &lod_comp->llc_compr_chunk_log_bits,
+ lod_comp->llc_stripe_size);
}
lod_comp->llc_stripe_offset = v1->lmm_stripe_offset;
(lov_pattern(lsme->lsme_pattern) == LOV_PATTERN_MDT) ||
(lov_pattern(lsme->lsme_pattern) == LOV_PATTERN_FOREIGN))
return lov_pattern(lsme->lsme_pattern &
- ~LOV_PATTERN_OVERSTRIPING);
+ ~(LOV_PATTERN_OVERSTRIPING | LOV_PATTERN_COMPRESS));
return 0;
}
lse->lsme_uuid);
} else {
CDEBUG(level, DEXT ": id: %u, flags: %x, "
- "magic 0x%08X, layout_gen %u, "
- "stripe count %u, sstripe size %u, "
+ "magic 0x%08X, pattern 0x%X, layout_gen %u, "
+ "stripe_count %u, stripe_size %u, "
"pool: ["LOV_POOLNAMEF"]\n",
PEXT(&lse->lsme_extent), lse->lsme_id,
lse->lsme_flags, lse->lsme_magic,
+ lse->lsme_pattern,
lse->lsme_layout_gen, lse->lsme_stripe_count,
lse->lsme_stripe_size, lse->lsme_pool_name);
+ if (lse->lsme_compr_type > 0) {
+ CDEBUG(level, " compr_type %u, compr_lvl %u, "
+ "compr_chunk_log_bits %u\n",
+ lse->lsme_compr_type,
+ lse->lsme_compr_lvl,
+ lse->lsme_compr_chunk_log_bits);
+ }
if (!lsme_inited(lse) ||
lse->lsme_pattern & LOV_PATTERN_F_RELEASED)
continue;
lle->lle_comp_ops = NULL;
break;
default:
- CERROR("%s: unknown composite layout entry type %i\n",
+ CERROR("%s: unknown composite layout entry type 0x%x\n",
lov2obd(dev->ld_lov)->obd_name,
lsm->lsm_entries[i]->lsme_pattern);
dump_lsm(D_ERROR, lsm);
req->rq_request_portal = OST_IO_PORTAL;
}
+static inline const char *crypto_name_from_type(enum ll_compr_type type)
+{
+ switch (type) {
+ case LL_COMPR_TYPE_NONE:
+ return "none";
+ case LL_COMPR_TYPE_GZIP:
+ return "deflate";
+ case LL_COMPR_TYPE_LZ4:
+ return "lz4";
+ case LL_COMPR_TYPE_LZ4HC:
+ return "lz4hc";
+ case LL_COMPR_TYPE_LZO:
+ return "lzo";
+ default:
+ return "unknown";
+ }
+}
+
#endif /* OSC_INTERNAL_H */
(long long)PORTALS_CFG_TYPE);
LASSERTF(LUSTRE_CFG_TYPE == 123, "found %lld\n",
(long long)LUSTRE_CFG_TYPE);
+ LASSERTF(LL_COMPR_TYPE_NONE == 0, "found %lld\n",
+ (long long)LL_COMPR_TYPE_NONE);
+ LASSERTF(LL_COMPR_TYPE_FAST == 1, "found %lld\n",
+ (long long)LL_COMPR_TYPE_FAST);
+ LASSERTF(LL_COMPR_TYPE_BEST == 2, "found %lld\n",
+ (long long)LL_COMPR_TYPE_BEST);
+ LASSERTF(LL_COMPR_TYPE_GZIP == 3, "found %lld\n",
+ (long long)LL_COMPR_TYPE_GZIP);
+ LASSERTF(LL_COMPR_TYPE_LZ4 == 4, "found %lld\n",
+ (long long)LL_COMPR_TYPE_LZ4);
+ LASSERTF(LL_COMPR_TYPE_LZ4HC == 5, "found %lld\n",
+ (long long)LL_COMPR_TYPE_LZ4HC);
+ LASSERTF(LL_COMPR_TYPE_LZO == 6, "found %lld\n",
+ (long long)LL_COMPR_TYPE_LZO);
+ LASSERTF(LL_COMPR_TYPE_MAX == 7, "found %lld\n",
+ (long long)LL_COMPR_TYPE_MAX);
}
[[ "$comp_flags" =~ "nocompr" ]] || error "$comp_flags do not have nocompr"
}
-run_test 100a "set stripe coponent with nocompr"
+run_test 100a "set stripe component with nocompr"
+
+test_100b() {
+ (( $MDS1_VERSION >= $(version_code 2.14.0.88) )) ||
+ skip "Need MDS >= 2.14.0.88 for compression support"
+
+ local tf=$DIR/$tdir/$tfile
+ local type="gzip lz4 lz4hc lzo"
+ local l1=1
+ local c1="64"
+ local p="$TMP/$TESTSUITE-$TESTNAME.parameters"
+
+ save_lustre_params client "llite.*.enable_compression" > $p
+ stack_trap "rm -rf $DIR/$tdir; restore_lustre_params < $p" EXIT
+ $LCTL set_param llite.*.enable_compression=1
+
+ test_mkdir $DIR/$tdir
+
+ # none compress type
+ rm -f $tf
+ $LFS setstripe -Eeof -S4M -Z none:$l1 --compress-chunk=${c1} $tf ||
+ error "set a none compress component in $tf failed"
+
+ local t2=$($LFS getstripe --compr-type $tf)
+ local l2=$($LFS getstripe --compr-level $tf)
+ local c2=$($LFS getstripe --compr-chunk $tf)
+ local patn=$($LFS getstripe -L $tf)
+
+ [[ -z "$t2" ]] || {
+ $LFS getstripe $tf
+ error "compress type $t2 should be empty"
+ }
+ [[ -z "$l2" ]] || {
+ $LFS getstripe $tf
+ error "compress level $l2 should be empty"
+ }
+ [[ -z "$c2" ]] || {
+ $LFS getstripe $tf
+ error "compress chunk size $c2 should be empty"
+ }
+ [[ "$patn" =~ "compress" ]] && {
+ $LFS getstripe $tf
+ error "pattern $patn should not contain compress flag"
+ }
+
+ # normal compress type
+ for t1 in $type; do
+ rm -f $tf
+
+ $LFS setstripe -Eeof -S4M -Z $t1:$l1 --compress-chunk=${c1} \
+ $tf || error "set a compress component in $tf failed"
+
+ local t2=$($LFS getstripe --compr-type $tf)
+ local l2=$($LFS getstripe --compr-level $tf)
+ local c2=$($LFS getstripe --compr-chunk $tf)
+ local patn=$($LFS getstripe -L $tf)
+
+ [[ "$t1" == "$t2" ]] || {
+ $LFS getstripe $tf
+ error "compress type $t1 != $t2"
+ }
+ (( $l1 == $l2 )) || {
+ $LFS getstripe $tf
+ error "compress level $l1 != $l2"
+ }
+ (( $c1 == $c2 )) || {
+ $LFS getstripe $tf
+ error "compress chunk size $c1 != $c2"
+ }
+ [[ "$patn" =~ "compress" ]] || {
+ $LFS getstripe $tf
+ error "pattern $patn does not contain compress flag"
+ }
+
+ l1=$((l1 + 1))
+ c1=$((c1 * 2))
+ (( $c1 < 4096 )) || $c1=4096
+ done
+}
+run_test 100b "create with compress component"
+
+test_100c() {
+ (( $MDS1_VERSION >= $(version_code 2.14.0.88) )) ||
+ skip "Need MDS >= 2.14.0.88 for compression support"
+
+ local tf=$DIR/$tdir/$tfile
+ local d1=$DIR/$tdir/dir-1
+ local p="$TMP/$TESTSUITE-$TESTNAME.parameters"
+
+ save_lustre_params client "llite.*.enable_compression" > $p
+ stack_trap "rm -rf $DIR/$tdir; restore_lustre_params < $p" EXIT
+ $LCTL set_param llite.*.enable_compression=1
+
+ test_mkdir $DIR/$tdir
+
+ # unknown compress type
+ $LFS setstripe -Eeof -Z bad $tf 2>/dev/null &&
+ error "should fail with unknown compress type"
+
+ # none compress type
+ test_mkdir $d1
+ $LFS setstripe -Eeof -Z gzip $DIR/$tdir/dir-1 ||
+ error "failed to set compress dir"
+ touch $d1/$tfile || error "failed to create $d1/$tfile"
+ type=$($LFS getstripe --compr-type $d1/$tfile)
+ [[ "gzip" == $type ]] || error "file compress type $type is not gzip"
+ $LFS setstripe -Eeof -Z none $d1/$tfile-1 ||
+ error "failed to set none compress type for $d1/$tfile-1"
+ type=$($LFS getstripe --compr-type $d1/$tfile-1)
+ [[ -z $type ]] || error "non compress file have compress type $type"
+
+ # compress level
+ # gzip max compress level 9
+ $LFS setstripe -Eeof -Z gzip:15 $tf ||
+ error "failed setting gzip type, level 15"
+ lvl=$($LFS getstripe --compr-level $tf)
+ (( $lvl == 9 )) || {
+ $LFS getstripe $tf
+ error "gzip max compress level $lvl != 9"
+ }
+ # lz4 max compress level 27
+ rm -f $tf
+ $LFS setstripe -Eeof -Z lz4:28 $tf ||
+ error "failed setting lz4 type, level 28"
+ lvl=$($LFS getstripe --compr-level $tf)
+ (( $lvl == 27 )) || {
+ $LFS getstripe $tf
+ error "gzip max compress level $lvl != 27"
+ }
+ # lz4hc max compress level 16
+ rm -f $tf
+ $LFS setstripe -Eeof -Z lz4hc:28 $tf ||
+ error "failed setting lz4hc type, level 28"
+ lvl=$($LFS getstripe --compr-level $tf)
+ (( $lvl == 16 )) || {
+ $LFS getstripe $tf
+ error "gzip max compress level $lvl != 16"
+ }
+
+ # test compress chunk size
+ # chunk size should at least be 64KiB
+ rm -f $tf
+ $LFS setstripe -Eeof -S1M -Z lz4 --compress-chunk=4 $tf ||
+ error "failed setting 4K chunk size"
+ c2=$($LFS getstripe --compr-chunk $tf)
+ (( $c2 == 64 )) || {
+ $LFS getstripe $tf
+ error "compress chunk size ($c2) < 64 KiB"
+ }
+
+ # should not bigger than stripe size,
+ rm -f $tf
+ $LFS setstripe -Eeof -S1M -Z lz4 --compress-chunk=4M $tf ||
+ error "failed setting 4M chunk size"
+ c2=$($LFS getstripe --compr-chunk $tf)
+ (( $c2 == 1024 )) || {
+ $LFS getstripe $tf
+ error "compress chunk size ($c2) > stripe_size (1024)"
+ }
+
+ # stripe size should be a multiple of chunk size
+ rm -f $tf
+ ss=$((4 * 64 * 1024)) # 4*64KiB
+ cc=$((3 * 64)) # 3*64KiB, and it should be rounded to 2*64KiB
+ $LFS setstripe -Eeof -S $ss -Z lz4 --compress-chunk=$cc $tf ||
+ error "failed setting $cc chunk size"
+ c2=$($LFS getstripe --compr-chunk $tf)
+ (( $c2 == 2 * 64 )) || {
+ $LFS getstripe $tf
+ error "stripe_size (254) should be a multiple of chunk size ($c2)"
+ }
+}
+run_test 100c "create compress file with improper compress arguments"
complete $SECONDS
check_and_cleanup_lustre
" [--overstripe-count|-C <stripe_count>]\n" \
" [--stripe-index|-i <start_ost_idx>]\n" \
" [--stripe-size|-S <stripe_size>]\n" \
+ " [--compress|-Z <type>[:<level>]\n" \
+ " [--compress-chunk=<size>]\n" \
" [--extension-size|--ext-size|-z]\n" \
" [--layout|-L <pattern>]\n" \
" [--mirror-count|-N[mirror_count]]\n" \
"\tstripe_size: Number of bytes on each OST (0=fs default)\n" \
"\t Optional K, M, or G suffix (for KB, MB, GB\n" \
"\t respectively). Must be a multiple of 64KiB.\n" \
+ "\tcompress:\n" \
+ "\t Compression algorithm <type> and compress <level>.\n" \
+ "\t The compress level showed in getstripe could be \n" \
+ "\t not exactly the one input here.\n" \
+ "\tcompress-chunk:\n" \
+ "\t Compression data chunk size in KiB, the value\n" \
+ "\t will be adjusted to power-of-two multiples of the\n"\
+ "\t base 64KiB.\n" \
"\textension_size:\n" \
"\t Number of bytes the previous component is extended\n" \
"\t each time. Optional K, M, or G suffix (for KB,\n" \
" [--extension-size|--ext-size|-z]\n"
" [--component-start[=[+-]comp_start]]\n"
" [--component-end[=[+-]comp_end]|-E[[+-]comp_end]]\n"
+ " [--compress-type|--compr-type]\n"
+ " [--compress-level|--compr-level]\n"
+ " [--compress-chunk|--compr-chunk]\n"
" [[!] --mirror-index=[+-]<index> |\n"
" [!] --mirror-id=[+-]<id>] [--mirror-count|-N]\n"
" <directory|filename> ..."},
int lsa_nr_tgts;
bool lsa_first_comp;
bool lsa_extension_comp;
+ enum ll_compr_type lsa_compr_type;
+ int lsa_compr_lvl;
+ __u32 lsa_compr_chunk_size;
__u32 *lsa_tgts;
char *lsa_pool_name;
};
lsa->lsa_pattern = LLAPI_LAYOUT_RAID0;
lsa->lsa_pool_name = NULL;
+ lsa->lsa_compr_type = LL_COMPR_TYPE_NONE;
+ lsa->lsa_compr_lvl = -1;
+
lsa->lsa_mirror_count = mirror_count;
lsa->lsa_first_comp = first_comp;
}
return rc;
}
+ /* set compress parameters */
+ rc = llapi_layout_compress_set(layout, lsa->lsa_compr_type,
+ lsa->lsa_compr_lvl,
+ lsa->lsa_compr_chunk_size,
+ lsa->lsa_stripe_size);
+ if (rc) {
+ fprintf(stderr, "Set compress parameters type %d level %d chunk_size %d stripe_size %llu failed: %s\n",
+ lsa->lsa_compr_type, lsa->lsa_compr_lvl,
+ lsa->lsa_compr_chunk_size, lsa->lsa_stripe_size,
+ strerror(errno));
+ return rc;
+ }
+
if (lsa->lsa_pool_name) {
rc = llapi_layout_pool_name_set(layout, lsa->lsa_pool_name);
if (rc) {
LFS_FIND_PERM,
LFS_PRINTF_OPT,
LFS_STATS_OPT,
- LFS_STATS_INTERVAL_OPT
+ LFS_STATS_INTERVAL_OPT,
+ LFS_COMPRESS_TYPE_OPT,
+ LFS_COMPRESS_LEVEL_OPT,
+ LFS_COMPRESS_CHUNK_OPT,
};
/* functions */
{ .val = LFS_STATS_INTERVAL_OPT,
.name = "stats-interval",
.has_arg = required_argument},
+ { .val = LFS_COMPRESS_CHUNK_OPT,
+ .name = "compress-chunk", .has_arg = required_argument},
{ .val = 'c', .name = "stripe-count", .has_arg = required_argument},
{ .val = 'c', .name = "stripe_count", .has_arg = required_argument},
{ .val = 'c', .name = "mdt-count", .has_arg = required_argument},
{ .val = 'y', .name = "yaml", .has_arg = required_argument },
{ .val = 'z', .name = "ext-size", .has_arg = required_argument},
{ .val = 'z', .name = "extension-size", .has_arg = required_argument},
+ { .val = 'Z', .name = "compress", .has_arg = required_argument},
+ { .val = 'Z', .name = "compr", .has_arg = required_argument},
{ .name = NULL } };
setstripe_args_init(&lsa);
snprintf(cmd, sizeof(cmd), "%s %s", progname, argv[0]);
progname = cmd;
while ((c = getopt_long(argc, argv,
- "bc:C:dDE:f:H:i:I:m:N::no:p:L:s:S:vx:W:y:z:",
+ "bc:C:dDE:f:H:i:I:m:N::no:p:L:s:S:vx:W:y:z:Z:",
long_opts, NULL)) >= 0) {
size_units = 1;
switch (c) {
lsa.lsa_extension_comp = true;
break;
+ case 'Z':
+ if (layout == NULL && mirror_list == NULL &&
+ lsa.lsa_comp_end == 0)
+ fprintf(stderr,
+ "WARNING: (-Z) option specified, however no layout defined (-E), compression feature will not be enabled.\n");
+
+ result = llapi_parse_compress_type(optarg,
+ &lsa.lsa_compr_type,
+ &lsa.lsa_compr_lvl);
+ if (result) {
+ fprintf(stderr,
+ "%s %s: invalid compression type/level '%s'\n",
+ progname, argv[0], optarg);
+ goto usage_error;
+ }
+ break;
+ case LFS_COMPRESS_CHUNK_OPT: {
+ unsigned long long chunk_size;
+
+ size_units = 1024; /* default in KiB */
+ result = llapi_parse_size(optarg, &chunk_size,
+ &size_units, 1);
+ if (result) {
+ fprintf(stderr,
+ "%s %s: invalid compress chunk size '%s'\n",
+ progname, argv[0], optarg);
+ goto usage_error;
+ }
+ lsa.lsa_compr_chunk_size = chunk_size;
+ }
+ break;
default:
fprintf(stderr, "%s %s: unrecognized option '%s'\n",
progname, argv[0], argv[optind - 1]);
.name = "mirror-index", .has_arg = required_argument },
{ .val = LFS_MIRROR_ID_OPT,
.name = "mirror-id", .has_arg = required_argument },
+ { .val = LFS_COMPRESS_TYPE_OPT,
+ .name ="compress-type", .has_arg = no_argument },
+ { .val = LFS_COMPRESS_TYPE_OPT,
+ .name ="compr-type", .has_arg = no_argument },
+ { .val = LFS_COMPRESS_LEVEL_OPT,
+ .name ="compress-level", .has_arg = no_argument },
+ { .val = LFS_COMPRESS_LEVEL_OPT,
+ .name ="compr-level", .has_arg = no_argument },
+ { .val = LFS_COMPRESS_CHUNK_OPT,
+ .name ="compress-chunk", .has_arg = no_argument },
+ { .val = LFS_COMPRESS_CHUNK_OPT,
+ .name ="compr-chunk", .has_arg = no_argument },
{ .val = 'c', .name = "stripe-count", .has_arg = no_argument },
{ .val = 'c', .name = "stripe_count", .has_arg = no_argument },
/* find { .val = 'C', .name = "ctime", .has_arg = required_argument }*/
param->fp_exclude_mirror_id = !!neg_opt;
break;
}
+ case LFS_COMPRESS_TYPE_OPT: {
+ if (!(param->fp_verbose & VERBOSE_DETAIL)) {
+ param->fp_verbose |= VERBOSE_COMPRESS_TYPE;
+ param->fp_max_depth = 0;
+ }
+ break;
+ }
+ case LFS_COMPRESS_LEVEL_OPT: {
+ if (!(param->fp_verbose & VERBOSE_DETAIL)) {
+ param->fp_verbose |= VERBOSE_COMPRESS_LEVEL;
+ param->fp_max_depth = 0;
+ }
+ break;
+ }
+ case LFS_COMPRESS_CHUNK_OPT: {
+ if (!(param->fp_verbose & VERBOSE_DETAIL)) {
+ param->fp_verbose |= VERBOSE_COMPRESS_CHUNK;
+ param->fp_max_depth = 0;
+ }
+ break;
+ }
case 'd':
param->fp_max_depth = 0;
break;
}
/**
- * size_units is to be initialized (or zeroed) by caller.
+ * \a size_units is to be initialized (or zeroed) by caller.
+ * if \a bytes_spec is true, then input "xxxb" means xxx bytes, otherwise
+ * it means xxx blocks, i.e. (xxx * 512) bytes.
*/
int llapi_parse_size(const char *optarg, unsigned long long *size,
unsigned long long *size_units, int bytes_spec)
{
if (layout_pattern & LOV_PATTERN_F_RELEASED)
return "released";
- else if (layout_pattern & LOV_PATTERN_FOREIGN)
+
+ if (layout_pattern & LOV_PATTERN_FOREIGN)
return "foreign";
- else if (layout_pattern == LOV_PATTERN_MDT)
+
+ switch (layout_pattern) {
+ case LOV_PATTERN_MDT:
return "mdt";
- else if (layout_pattern == LOV_PATTERN_RAID0)
+ case (LOV_PATTERN_MDT | LOV_PATTERN_COMPRESS):
+ return "mdt,compress";
+ case LOV_PATTERN_RAID0:
return "raid0";
- else if (layout_pattern ==
- (LOV_PATTERN_RAID0 | LOV_PATTERN_OVERSTRIPING))
+ case (LOV_PATTERN_RAID0 | LOV_PATTERN_OVERSTRIPING):
return "raid0,overstriped";
- else
+ case (LOV_PATTERN_RAID0 | LOV_PATTERN_COMPRESS):
+ return "raid0,compress";
+ case (LOV_PATTERN_RAID0 | LOV_PATTERN_OVERSTRIPING |
+ LOV_PATTERN_COMPRESS):
+ return "raid0,overstriped,compress";
+ default:
return "unknown";
+ }
}
enum lov_dump_flags {
llapi_printf(LLAPI_MSG_NORMAL, "components:\n");
}
+static const char *compress_type_2str(__u8 compr_type)
+{
+ bool found = false;
+ int i = 0;
+
+ for (i = 0; i < ARRAY_SIZE(compr_type_table); i++) {
+ if (compr_type == compr_type_table[i].ctn_compr_type) {
+ found = true;
+ break;
+ }
+ }
+ if (found)
+ return compr_type_table[i].ctn_name;
+ else
+ return("unknown");
+}
+
+static int mapback_compress_level(enum ll_compr_type type, __u8 level)
+{
+ int i;
+ bool known_type = false;
+
+ for (i = 0; i < ARRAY_SIZE(compr_type_table); i++) {
+ if (compr_type_table[i].ctn_compr_type == type) {
+ known_type = true;
+ if (compr_type_table[i].ctn_to_compr_level != NULL)
+ return compr_type_table[i].ctn_to_compr_level(
+ level);
+ }
+ }
+
+ if (!known_type)
+ return 0;
+
+ return level;
+}
+
static void lcme_flags2str(__u32 comp_flags)
{
bool found = false;
{
struct lov_comp_md_v1 *comp_v1 = (void *)¶m->fp_lmd->lmd_lmm;
struct lov_comp_md_entry_v1 *entry;
+ struct lov_user_md *lmm;
char *separator = "";
enum llapi_layout_verbose verbose = param->fp_verbose;
bool yaml = flags & LDF_YAML;
entry = &comp_v1->lcm_entries[index];
+ lmm = (struct lov_user_md *)((char *)comp_v1 + entry->lcme_offset);
if (yaml)
llapi_printf(LLAPI_MSG_NORMAL, "%2scomponent%d:\n", " ", index);
separator = "\n";
}
+ if (lmm->lmm_pattern & LOV_PATTERN_COMPRESS) {
+ if (verbose & VERBOSE_COMPRESS_TYPE) {
+ llapi_printf(LLAPI_MSG_NORMAL, "%s", separator);
+ if (verbose & ~VERBOSE_COMPRESS_TYPE)
+ llapi_printf(LLAPI_MSG_NORMAL,
+ "%4slcme_compr_type: ", " ");
+ llapi_printf(LLAPI_MSG_NORMAL, "%s",
+ compress_type_2str(entry->lcme_compr_type));
+ separator = "\n";
+ }
+ if (verbose & VERBOSE_COMPRESS_LEVEL) {
+ llapi_printf(LLAPI_MSG_NORMAL, "%s", separator);
+ if (verbose & ~VERBOSE_COMPRESS_LEVEL)
+ llapi_printf(LLAPI_MSG_NORMAL,
+ "%4slcme_compr_lvl: ", " ");
+ llapi_printf(LLAPI_MSG_NORMAL, "%d",
+ mapback_compress_level(
+ entry->lcme_compr_type,
+ entry->lcme_compr_lvl));
+ separator = "\n";
+ }
+ if (verbose & VERBOSE_COMPRESS_CHUNK) {
+ llapi_printf(LLAPI_MSG_NORMAL, "%s", separator);
+ if (verbose & ~VERBOSE_COMPRESS_CHUNK)
+ llapi_printf(LLAPI_MSG_NORMAL,
+ "%4slcme_compr_chunk_kb: ",
+ " ");
+ /* chunk_size_in_KiB = 2^chunk_log_bits * 64 */
+ llapi_printf(LLAPI_MSG_NORMAL, "%d",
+ 1 << (entry->lcme_compr_chunk_log_bits + 6));
+ separator = "\n";
+ }
+ }
+
if (yaml) {
llapi_printf(LLAPI_MSG_NORMAL, "%s", separator);
llapi_printf(LLAPI_MSG_NORMAL, "%4ssub_layout:\n", " ");
#include <libcfs/util/list.h>
#include <lustre/lustreapi.h>
#include "lustreapi_internal.h"
+#include "lstddef.h"
/**
* Layout component, which contains all attributes of a plain
* initialized.
*/
uint32_t llc_objects_count;
+ uint8_t llc_compr_type;
+ uint8_t llc_compr_lvl;
+ uint8_t llc_compr_chunk_log_bits;
struct lov_user_ost_data_v1 *llc_objects;
};
struct { /* For FOREIGN/HSM layout. */
ent->lcme_timestamp = comp->llc_timestamp;
ent->lcme_extent.e_start = comp->llc_extent.e_start;
ent->lcme_extent.e_end = comp->llc_extent.e_end;
+ if (comp->llc_compr_type != LL_COMPR_TYPE_NONE) {
+ ent->lcme_compr_type = comp->llc_compr_type;
+ ent->lcme_compr_lvl = comp->llc_compr_lvl;
+ ent->lcme_compr_chunk_log_bits =
+ comp->llc_compr_chunk_log_bits;
+ blob->lmm_pattern |= LOV_PATTERN_COMPRESS;
+ }
ent->lcme_size = blob_size;
ent->lcme_offset = offset;
offset += blob_size;
return layout_stripe_size_set(layout, size, true);
}
+/* return the index of the most significant bit of x */
+static inline int msb_index(__u32 x)
+{
+ int n = 0;
+
+ while (x) {
+ n++;
+ x >>= 1;
+ }
+
+ return n - 1;
+}
+
+/**
+ * Extract compress algorithm type and compress level value
+ *
+ * compress argument examples could be:
+ * -Z lz4
+ * -Z gzip:9
+ * -Z fast
+ * -Z best
+ */
+int llapi_parse_compress_type(const char *optarg, unsigned int *type,
+ int *level)
+{
+ char *ptr;
+ char *argbuf = (char *)optarg;
+ int i;
+ bool found = false;
+
+ ptr = strchr(argbuf, ':');
+ if (ptr) {
+ *ptr = '\0';
+ ptr++;
+ }
+
+ for (i = 0; i < ARRAY_SIZE(compr_type_table); i++) {
+ if (strcmp(argbuf, compr_type_table[i].ctn_name) == 0) {
+ *type = compr_type_table[i].ctn_compr_type;
+ found = true;
+ break;
+ }
+ }
+
+ if (!found)
+ return -1;
+
+ if (ptr)
+ *level = strtoul(ptr, NULL, 0);
+
+ return 0;
+}
+
+/**
+ * Set the compress parameters of \a layout.
+ *
+ * \param[in] layout layout to set compress parameters
+ * \param[in] type compress type
+ * \param[in] level compress level
+ * \param[in] chunk_size compress chunk size, the log bits is
+ * (chunksize = 2^(chunk_log_bits + 16))
+ * \param[in] stripe_size stripe size, it must be a multiple of chunk size
+ *
+ * \retval 0 on success
+ * \retval -1 if arguments are invalid
+ */
+int llapi_layout_compress_set(struct llapi_layout *layout,
+ enum ll_compr_type type, int level,
+ __u32 chunk_size, unsigned long long stripe_size)
+{
+ struct llapi_layout_comp *comp;
+ int i;
+ bool compr_level_set = false;
+ int log_bits;
+
+ comp = __llapi_layout_cur_comp(layout);
+ if (comp == NULL)
+ return -1;
+
+ /* type 0 means this is not a compress component */
+ comp->llc_compr_type = type;
+ if (type == LL_COMPR_TYPE_NONE)
+ return 0;
+
+ for (i = 0; i < ARRAY_SIZE(compr_type_table); i++) {
+ if (compr_type_table[i].ctn_compr_type == type) {
+ if (compr_type_table[i].ctn_from_compr_level != NULL) {
+ comp->llc_compr_lvl = compr_type_table[i].
+ ctn_from_compr_level(level);
+ compr_level_set = true;
+ }
+ break;
+ }
+ }
+ if (!compr_level_set) {
+ if (level == -1)
+ comp->llc_compr_lvl = 0;
+ else
+ comp->llc_compr_lvl = level & 0xf;
+ }
+
+ /**
+ * adjust chunk_size to be power-of-two multiples of 64KiB,
+ * llc_chunk_log_bits is limited to 4bits unsigned value.
+ */
+ if (chunk_size <= (__u32)(1 << 16))
+ comp->llc_compr_chunk_log_bits = 0;
+ else if (chunk_size >= (__u32)(1 << 31))
+ comp->llc_compr_chunk_log_bits = 15;
+ else
+ comp->llc_compr_chunk_log_bits = msb_index(chunk_size) - 16;
+
+ /* stripe size must be a multiple of chunk size */
+ if (stripe_size != LLAPI_LAYOUT_DEFAULT) {
+ log_bits = comp->llc_compr_chunk_log_bits + 16;
+
+ while ((1 << log_bits) > stripe_size ||
+ stripe_size & ((1 << log_bits) - 1))
+ log_bits--;
+
+ if (log_bits < 16 || log_bits > 31)
+ return -1;
+ else
+ comp->llc_compr_chunk_log_bits = log_bits - 16;
+ }
+
+ return 0;
+}
+
/**
* Get the RAID pattern of \a layout.
*
new->llc_extent.e_end = comp->llc_extent.e_end;
new->llc_id = comp->llc_id;
new->llc_flags = comp->llc_flags;
+ new->llc_compr_type = comp->llc_compr_type;
+ new->llc_compr_lvl = comp->llc_compr_lvl;
+ new->llc_compr_chunk_log_bits = comp->llc_compr_chunk_log_bits;
list_add_tail(&new->llc_list, &new_layout->llot_comp_list);
new_layout->llot_cur_comp = new;
CHECK_VALUE(LUSTRE_CFG_TYPE);
}
+static void
+check_ll_compr_type()
+{
+ CHECK_VALUE(LL_COMPR_TYPE_NONE);
+ CHECK_VALUE(LL_COMPR_TYPE_FAST);
+ CHECK_VALUE(LL_COMPR_TYPE_BEST);
+ CHECK_VALUE(LL_COMPR_TYPE_GZIP);
+ CHECK_VALUE(LL_COMPR_TYPE_LZ4);
+ CHECK_VALUE(LL_COMPR_TYPE_LZ4HC);
+ CHECK_VALUE(LL_COMPR_TYPE_LZO);
+ CHECK_VALUE(LL_COMPR_TYPE_MAX);
+}
+
int
main(int argc, char **argv)
{
#endif /* !HAVE_NATIVE_LINUX_CLIENT */
check_lustre_cfg();
+ check_ll_compr_type();
+
printf("}\n");
return 0;
(long long)PORTALS_CFG_TYPE);
LASSERTF(LUSTRE_CFG_TYPE == 123, "found %lld\n",
(long long)LUSTRE_CFG_TYPE);
+ LASSERTF(LL_COMPR_TYPE_NONE == 0, "found %lld\n",
+ (long long)LL_COMPR_TYPE_NONE);
+ LASSERTF(LL_COMPR_TYPE_FAST == 1, "found %lld\n",
+ (long long)LL_COMPR_TYPE_FAST);
+ LASSERTF(LL_COMPR_TYPE_BEST == 2, "found %lld\n",
+ (long long)LL_COMPR_TYPE_BEST);
+ LASSERTF(LL_COMPR_TYPE_GZIP == 3, "found %lld\n",
+ (long long)LL_COMPR_TYPE_GZIP);
+ LASSERTF(LL_COMPR_TYPE_LZ4 == 4, "found %lld\n",
+ (long long)LL_COMPR_TYPE_LZ4);
+ LASSERTF(LL_COMPR_TYPE_LZ4HC == 5, "found %lld\n",
+ (long long)LL_COMPR_TYPE_LZ4HC);
+ LASSERTF(LL_COMPR_TYPE_LZO == 6, "found %lld\n",
+ (long long)LL_COMPR_TYPE_LZO);
+ LASSERTF(LL_COMPR_TYPE_MAX == 7, "found %lld\n",
+ (long long)LL_COMPR_TYPE_MAX);
}
git://git.whamcloud.com / fs / lustre-release.git / commitdiff