From: Bobi Jam Date: Thu, 17 Aug 2023 07:13:49 +0000 (+0800) Subject: EX-7806 csdc: set DoM compression component X-Git-Url: https://git.whamcloud.com/?a=commitdiff_plain;h=b0b262fe098ddaead239e277b095e8dfaafaafd9;p=fs%2Flustre-release.git EX-7806 csdc: set DoM compression component Allow to create file with compression DoM component. Signed-off-by: Bobi Jam Change-Id: I8b5c4d5fd18033e796b1c7555c37251e4f7f9a49 Reviewed-on: https://review.whamcloud.com/c/ex/lustre-release/+/51968 Tested-by: jenkins Tested-by: Maloo Reviewed-by: Andreas Dilger --- diff --git a/lipe/src/lamigo.c b/lipe/src/lamigo.c index 8c45c27..b82dc2b 100644 --- a/lipe/src/lamigo.c +++ b/lipe/src/lamigo.c @@ -1131,7 +1131,7 @@ static int lamigo_striping_is_in_sync(struct lov_user_md *lum, /* do not reset onsrc as any object matters */ } - if (opt.o_include_dom && v3->lmm_pattern == LOV_PATTERN_MDT) { + if (opt.o_include_dom && (v3->lmm_pattern & LOV_PATTERN_MDT)) { LX_DEBUG("DoM component\n"); onsrc++; continue; diff --git a/lipe/src/lpurge.c b/lipe/src/lpurge.c index c6deae1..6bf150a 100644 --- a/lipe/src/lpurge.c +++ b/lipe/src/lpurge.c @@ -616,7 +616,7 @@ static int lpurge_check_mdt_object(struct lpurge_slot *ls, v3 = (struct lov_user_md_v3 *)lov_comp_entry(comp_v1, i); - if (v3->lmm_pattern == LOV_PATTERN_MDT) { + if (v3->lmm_pattern & LOV_PATTERN_MDT) { src_comp_id = mirror_id_of(entry->lcme_id); } else { tgt_comp_id = mirror_id_of(entry->lcme_id); diff --git a/lustre/include/uapi/linux/lustre/lustre_user.h b/lustre/include/uapi/linux/lustre/lustre_user.h index 7e92a32..ac611d4 100644 --- a/lustre/include/uapi/linux/lustre/lustre_user.h +++ b/lustre/include/uapi/linux/lustre/lustre_user.h @@ -740,7 +740,8 @@ static inline bool lov_pattern_supported(__u32 pattern) return pattern_base == LOV_PATTERN_RAID0 || pattern_base == (LOV_PATTERN_RAID0 | LOV_PATTERN_OVERSTRIPING) || pattern_base == (LOV_PATTERN_RAID0 | LOV_PATTERN_COMPRESS) || - pattern_base == LOV_PATTERN_MDT; + pattern_base == LOV_PATTERN_MDT || + pattern_base == (LOV_PATTERN_MDT | LOV_PATTERN_COMPRESS); } /* RELEASED and MDT patterns are not valid in many places, so rather than diff --git a/lustre/llite/file.c b/lustre/llite/file.c index 204072d..29ab6b3 100644 --- a/lustre/llite/file.c +++ b/lustre/llite/file.c @@ -2593,7 +2593,7 @@ int ll_lov_getstripe_ea_info(struct inode *inode, const char *filename, * second one is always that applications which are * doing large IOs. */ - if (lmm->lmm_pattern == LOV_PATTERN_MDT) + if (lmm->lmm_pattern & LOV_PATTERN_MDT) i = comp_v1->lcm_entry_count > 1 ? 1 : 0; else i = comp_v1->lcm_entry_count > 1 ? diff --git a/lustre/lod/lod_lov.c b/lustre/lod/lod_lov.c index f146b31..22d5c41 100644 --- a/lustre/lod/lod_lov.c +++ b/lustre/lod/lod_lov.c @@ -1778,7 +1778,7 @@ static int lod_verify_v1v3(struct lod_device *d, const struct lu_buf *buf, stripe_offset = le16_to_cpu(lum->lmm_stripe_offset); if (!is_from_disk && stripe_offset != LOV_OFFSET_DEFAULT && - lov_pattern(le32_to_cpu(lum->lmm_pattern)) != LOV_PATTERN_MDT) { + !(lov_pattern(le32_to_cpu(lum->lmm_pattern)) & LOV_PATTERN_MDT)) { /* if offset is not within valid range [0, osts_size) */ if (stripe_offset >= d->lod_ost_descs.ltd_tgts_size) { CDEBUG(D_LAYOUT, "stripe offset %u >= bitmap size %u\n", @@ -2135,7 +2135,7 @@ int lod_verify_striping(const struct lu_env *env, struct lod_device *d, case LOV_USER_MAGIC_V1: case LOV_USER_MAGIC_V3: case LOV_USER_MAGIC_SPECIFIC: - if (lov_pattern(le32_to_cpu(lum->lmm_pattern)) == + if (lov_pattern(le32_to_cpu(lum->lmm_pattern)) & LOV_PATTERN_MDT) { /* DoM must use composite layout */ CDEBUG(D_LAYOUT, "DoM without composite layout\n"); @@ -2288,7 +2288,7 @@ recheck: } /* Check DoM entry is always the first one */ - if (lov_pattern(le32_to_cpu(lum->lmm_pattern)) == + if (lov_pattern(le32_to_cpu(lum->lmm_pattern)) & LOV_PATTERN_MDT) { /* DoM component must be the first in a mirror */ if (le64_to_cpu(ext->e_start) > 0) { diff --git a/lustre/lod/lod_object.c b/lustre/lod/lod_object.c index a67ab0e..53d9e94 100644 --- a/lustre/lod/lod_object.c +++ b/lustre/lod/lod_object.c @@ -1158,7 +1158,7 @@ static inline void lod_adjust_stripe_info(struct lod_layout_component *comp, struct lov_desc *desc, int append_stripes) { - if (comp->llc_pattern != LOV_PATTERN_MDT) { + if (!(comp->llc_pattern & LOV_PATTERN_MDT)) { if (append_stripes) { comp->llc_stripe_count = append_stripes; } else if (!comp->llc_stripe_count) { @@ -3425,7 +3425,7 @@ static int lod_declare_layout_merge(const struct lu_env *env, /* check if first entry in new layout is DOM */ lmm = (struct lov_mds_md_v1 *)((char *)merge_lcm + merge_lcm->lcm_entries[0].lcme_offset); - merge_has_dom = lov_pattern(le32_to_cpu(lmm->lmm_pattern)) == + merge_has_dom = lov_pattern(le32_to_cpu(lmm->lmm_pattern)) & LOV_PATTERN_MDT; for (i = 0; i < merge_entry_count; i++) { @@ -5966,7 +5966,7 @@ int lod_striped_create(const struct lu_env *env, struct dt_object *dt, if (lod_comp->llc_pattern & LOV_PATTERN_F_RELEASED) lod_comp_set_init(lod_comp); - if (lov_pattern(lod_comp->llc_pattern) == LOV_PATTERN_MDT) + if (lov_pattern(lod_comp->llc_pattern) & LOV_PATTERN_MDT) lod_comp_set_init(lod_comp); if (lod_comp->llc_stripe == NULL) @@ -6017,7 +6017,7 @@ static inline bool lod_obj_is_dom(struct dt_object *dt) if (!lo->ldo_comp_cnt) return false; - return (lov_pattern(lo->ldo_comp_entries[0].llc_pattern) == + return (lov_pattern(lo->ldo_comp_entries[0].llc_pattern) & LOV_PATTERN_MDT); } @@ -7045,7 +7045,7 @@ static bool lod_is_instantiation_needed(struct lod_layout_component *comp) if (comp->llc_magic == LOV_MAGIC_FOREIGN) return false; - return !(((lov_pattern(comp->llc_pattern) == LOV_PATTERN_MDT) && + return !(((lov_pattern(comp->llc_pattern) & LOV_PATTERN_MDT) && lod_comp_inited(comp)) || comp->llc_stripe); } diff --git a/lustre/lod/lod_qos.c b/lustre/lod/lod_qos.c index 9dbdea5..655a8b9 100644 --- a/lustre/lod/lod_qos.c +++ b/lustre/lod/lod_qos.c @@ -2456,12 +2456,7 @@ int lod_qos_parse_config(const struct lu_env *env, struct lod_object *lo, if (v1->lmm_pattern == 0) v1->lmm_pattern = LOV_PATTERN_RAID0; - if (lov_pattern(v1->lmm_pattern) != LOV_PATTERN_RAID0 && - lov_pattern(v1->lmm_pattern) != LOV_PATTERN_MDT && - lov_pattern(v1->lmm_pattern) != - (LOV_PATTERN_RAID0 | LOV_PATTERN_COMPRESS) && - lov_pattern(v1->lmm_pattern) != - (LOV_PATTERN_RAID0 | LOV_PATTERN_OVERSTRIPING)) { + if (!lov_pattern_supported(lov_pattern(v1->lmm_pattern))) { CDEBUG(D_LAYOUT, "%s: invalid pattern: %x\n", lod2obd(d)->obd_name, v1->lmm_pattern); GOTO(free_comp, rc = -EINVAL); @@ -2473,10 +2468,10 @@ int lod_qos_parse_config(const struct lu_env *env, struct lod_object *lo, lod_comp->llc_stripe_count = desc->ld_default_stripe_count; if (v1->lmm_stripe_count || - lov_pattern(v1->lmm_pattern) == LOV_PATTERN_MDT) + (lov_pattern(v1->lmm_pattern) & LOV_PATTERN_MDT)) lod_comp->llc_stripe_count = v1->lmm_stripe_count; - if (lov_pattern(lod_comp->llc_pattern) == LOV_PATTERN_MDT && + if ((lov_pattern(lod_comp->llc_pattern) & LOV_PATTERN_MDT) && lod_comp->llc_stripe_count != 0) { CDEBUG(D_LAYOUT, "%s: invalid stripe count: %u\n", lod2obd(d)->obd_name, @@ -2741,7 +2736,7 @@ int lod_qos_prep_create(const struct lu_env *env, struct lod_object *lo, RETURN(0); /* A Data-on-MDT component is being created */ - if (lov_pattern(lod_comp->llc_pattern) == LOV_PATTERN_MDT) + if (lov_pattern(lod_comp->llc_pattern) & LOV_PATTERN_MDT) RETURN(0); if (lod_comp->llc_pool) diff --git a/lustre/lov/lov_cl_internal.h b/lustre/lov/lov_cl_internal.h index 5201db4..5bee437 100644 --- a/lustre/lov/lov_cl_internal.h +++ b/lustre/lov/lov_cl_internal.h @@ -151,7 +151,7 @@ static inline char *llt2str(enum lov_layout_type llt) static inline __u32 lov_entry_type(struct lov_stripe_md_entry *lsme) { if ((lov_pattern(lsme->lsme_pattern) & LOV_PATTERN_RAID0) || - (lov_pattern(lsme->lsme_pattern) == LOV_PATTERN_MDT) || + (lov_pattern(lsme->lsme_pattern) & LOV_PATTERN_MDT) || (lov_pattern(lsme->lsme_pattern) == LOV_PATTERN_FOREIGN)) return lov_pattern(lsme->lsme_pattern & ~(LOV_PATTERN_OVERSTRIPING | LOV_PATTERN_COMPRESS)); diff --git a/lustre/lov/lov_ea.c b/lustre/lov/lov_ea.c index 29777ee..aa9dfa5 100644 --- a/lustre/lov/lov_ea.c +++ b/lustre/lov/lov_ea.c @@ -498,7 +498,7 @@ lsme_unpack_comp(struct lov_obd *lov, struct lov_mds_md *lmm, if (magic != LOV_MAGIC_FOREIGN && le16_to_cpu(lmm->lmm_stripe_count) == 0 && - lov_pattern(le32_to_cpu(lmm->lmm_pattern)) != LOV_PATTERN_MDT) + !(lov_pattern(le32_to_cpu(lmm->lmm_pattern)) & LOV_PATTERN_MDT)) RETURN(ERR_PTR(-EINVAL)); if (magic == LOV_MAGIC_FOREIGN) { diff --git a/lustre/lov/lov_internal.h b/lustre/lov/lov_internal.h index 43a280b..5571657 100644 --- a/lustre/lov/lov_internal.h +++ b/lustre/lov/lov_internal.h @@ -81,7 +81,7 @@ struct lov_stripe_md_entry { static inline bool lsme_is_dom(struct lov_stripe_md_entry *lsme) { - return (lov_pattern(lsme->lsme_pattern) == LOV_PATTERN_MDT); + return (lov_pattern(lsme->lsme_pattern) & LOV_PATTERN_MDT); } static inline void copy_lsm_entry(struct lov_stripe_md_entry *dst, diff --git a/lustre/mdd/mdd_object.c b/lustre/mdd/mdd_object.c index 8843b04..3da4924 100644 --- a/lustre/mdd/mdd_object.c +++ b/lustre/mdd/mdd_object.c @@ -1217,7 +1217,7 @@ static inline __u64 mdd_lmm_dom_size(void *buf) v1 = (struct lov_mds_md *)((char *)comp_v1 + off); /* DoM entry is the first entry always */ - if (lov_pattern(le32_to_cpu(v1->lmm_pattern)) == LOV_PATTERN_MDT) + if (lov_pattern(le32_to_cpu(v1->lmm_pattern)) & LOV_PATTERN_MDT) return le64_to_cpu(comp_v1->lcm_entries[0].lcme_extent.e_end); return 0; diff --git a/lustre/mdt/mdt_handler.c b/lustre/mdt/mdt_handler.c index cbdb3c7..e2ce2c7 100644 --- a/lustre/mdt/mdt_handler.c +++ b/lustre/mdt/mdt_handler.c @@ -601,7 +601,7 @@ __u32 mdt_lmm_dom_entry_check(struct lov_mds_md *lmm, int *is_dom_only) /* Fast check for DoM entry with no mirroring, should be the first */ if (le16_to_cpu(comp_v1->lcm_mirror_count) == 0 && - lov_pattern(le32_to_cpu(v1->lmm_pattern)) != LOV_PATTERN_MDT) + !(lov_pattern(le32_to_cpu(v1->lmm_pattern)) & LOV_PATTERN_MDT)) RETURN(0); /* check all entries otherwise */ @@ -615,7 +615,7 @@ __u32 mdt_lmm_dom_entry_check(struct lov_mds_md *lmm, int *is_dom_only) off = le32_to_cpu(lcme->lcme_offset); v1 = (struct lov_mds_md *)((char *)comp_v1 + off); - if (lov_pattern(le32_to_cpu(v1->lmm_pattern)) == + if (lov_pattern(le32_to_cpu(v1->lmm_pattern)) & LOV_PATTERN_MDT) dom_stripesize = le32_to_cpu(v1->lmm_stripe_size); else diff --git a/lustre/tests/sanity-pfl.sh b/lustre/tests/sanity-pfl.sh index 8147ce7..7a9e521 100644 --- a/lustre/tests/sanity-pfl.sh +++ b/lustre/tests/sanity-pfl.sh @@ -2967,6 +2967,30 @@ test_100j() { } run_test 100j "basic debug test for compression" +test_100k() { + (( $MDS1_VERSION >= $(version_code 2.14.0.92) )) || + skip "Need MDS >= 2.14.0.92 for compression support" + + local tf=$DIR/$tdir/$tfile + local p="$TMP/$TESTSUITE-$TESTNAME.parameters" + + save_lustre_params client "llite.*.enable_compression" > $p + stack_trap "rm -rf $DIR/$tdir; restore_lustre_params < $p" EXIT + $LCTL set_param llite.*.enable_compression=1 + + test_mkdir $DIR/$tdir + + $LFS setstripe -E1M -L mdt -Z lz4 -E4M -Eeof -Z none $tf || + error "set DoM compress component in $tf failed" + + verify_comp_attr lcme_flags $tf 0x1 init,compress + verify_comp_attr compr-type $tf 0x1 lz4 + verify_comp_attr pattern $tf 0x1 mdt,compress + verify_comp_attr pattern $tf 0x2 raid0,compress + verify_comp_attr pattern $tf 0x3 raid0 +} +run_test 100k "create with DoM compression component" + export LFS_SETSTRIPE_COMPR_OK="" complete_test $SECONDS diff --git a/lustre/tests/test-framework.sh b/lustre/tests/test-framework.sh index 09e4100..f67c2d3 100755 --- a/lustre/tests/test-framework.sh +++ b/lustre/tests/test-framework.sh @@ -10434,7 +10434,11 @@ verify_comp_attr() { pool) cmd+=" -p $tf" ;; comp-start) cmd+=" --component-start $tf" ;; comp-end) cmd+=" --component-end $tf" ;; + compr-type) cmd+=" --compr-type $tf" ;; + compr-level) cmd+=" --compr-level $tf" ;; + compr-chunk) cmd+= " --compr-chunk $tf" ;; lcme_flags) cmd+=" $tf | awk '/lcme_flags:/ { print \$2 }'" ;; + pattern) cmd+=" $tf | awk '/lmm_pattern:/ { print \$2 }'" ;; *) error "invalid attribute $attr";; esac diff --git a/lustre/utils/liblustreapi.c b/lustre/utils/liblustreapi.c index e50df41..dc50a42 100644 --- a/lustre/utils/liblustreapi.c +++ b/lustre/utils/liblustreapi.c @@ -463,7 +463,7 @@ static int llapi_stripe_param_verify(const struct llapi_stripe_param *param, } count = param->lsp_stripe_count; - if (param->lsp_stripe_pattern == LOV_PATTERN_MDT) { + if (param->lsp_stripe_pattern & LOV_PATTERN_MDT) { rc = -EINVAL; llapi_error(LLAPI_MSG_ERROR, rc, "Invalid pattern: '-L mdt', must be specified " @@ -2622,7 +2622,7 @@ static void lov_dump_user_lmm_header(struct lov_user_md *lum, char *path, space, prefix); if (is_dir) { if (!is_raw && lum->lmm_stripe_count == 0 && - lov_pattern(lum->lmm_pattern) != LOV_PATTERN_MDT) { + !(lov_pattern(lum->lmm_pattern) & LOV_PATTERN_MDT)){ unsigned int scount; rc = sattr_cache_get_defaults(NULL, path, @@ -2715,7 +2715,7 @@ static void lov_dump_user_lmm_header(struct lov_user_md *lum, char *path, lum->lmm_stripe_offset == (typeof(lum->lmm_stripe_offset))(-1) ? -1 : lum->lmm_stripe_offset); - else if (lov_pattern(lum->lmm_pattern) == LOV_PATTERN_MDT) + else if (lov_pattern(lum->lmm_pattern) & LOV_PATTERN_MDT) llapi_printf(LLAPI_MSG_NORMAL, "0"); else llapi_printf(LLAPI_MSG_NORMAL, "%u", @@ -2772,7 +2772,7 @@ void lov_dump_user_lmm_v1v3(struct lov_user_md *lum, char *pool_name, if (!is_dir && !skip_objs && (verbose & VERBOSE_OBJID) && !(lum->lmm_pattern & LOV_PATTERN_F_RELEASED || - lov_pattern(lum->lmm_pattern) == LOV_PATTERN_MDT)) { + (lov_pattern(lum->lmm_pattern) & LOV_PATTERN_MDT))) { char *space = " - "; if (indent) diff --git a/lustre/utils/liblustreapi_layout.c b/lustre/utils/liblustreapi_layout.c index 4f7f27c..e185787 100644 --- a/lustre/utils/liblustreapi_layout.c +++ b/lustre/utils/liblustreapi_layout.c @@ -675,7 +675,7 @@ struct llapi_layout *llapi_layout_get_by_xattr(void *lov_xattr, else if (v1->lmm_pattern == (LOV_PATTERN_RAID0 | LOV_PATTERN_OVERSTRIPING)) comp->llc_pattern = LLAPI_LAYOUT_OVERSTRIPING; - else if (v1->lmm_pattern == LOV_PATTERN_MDT) + else if (v1->lmm_pattern & LOV_PATTERN_MDT) comp->llc_pattern = LLAPI_LAYOUT_MDT; else /* Lustre only supports RAID0, overstripping, @@ -761,6 +761,12 @@ __u32 llapi_pattern_to_lov(uint64_t pattern) case LLAPI_LAYOUT_OVERSTRIPING: lov_pattern = LOV_PATTERN_OVERSTRIPING | LOV_PATTERN_RAID0; break; + case LLAPI_LAYOUT_RAID0 | LLAPI_LAYOUT_COMPRESS: + lov_pattern = LOV_PATTERN_RAID0 | LOV_PATTERN_COMPRESS; + break; + case LLAPI_LAYOUT_MDT | LLAPI_LAYOUT_COMPRESS: + lov_pattern = LOV_PATTERN_MDT | LOV_PATTERN_COMPRESS; + break; default: lov_pattern = EINVAL; } @@ -3819,8 +3825,8 @@ static int llapi_layout_sanity_cb(struct llapi_layout *layout, goto out_err; /* DoM sanity checks */ - if (comp->llc_pattern == LLAPI_LAYOUT_MDT || - comp->llc_pattern == LOV_PATTERN_MDT) { + if (!(comp->llc_pattern & LLAPI_LAYOUT_INVALID) && + (comp->llc_pattern & (LLAPI_LAYOUT_MDT | LOV_PATTERN_MDT))) { /* DoM components can't be extension components */ if (comp->llc_flags & LCME_FL_EXTENSION) { args->lsa_rc = LSE_DOM_EXTENSION; @@ -4017,7 +4023,8 @@ int llapi_layout_dom_size(struct llapi_layout *layout, uint64_t *size) if (rc) return -errno; - if (pattern != LOV_PATTERN_MDT && pattern != LLAPI_LAYOUT_MDT) { + if (pattern & LLAPI_LAYOUT_INVALID || + !(pattern & (LOV_PATTERN_MDT | LLAPI_LAYOUT_MDT))) { *size = 0; return 0; }