From 62f64a1077468e01a41df35212b5e5b1fe6b31d5 Mon Sep 17 00:00:00 2001 From: Bobi Jam Date: Fri, 12 May 2017 10:13:57 +0800 Subject: [PATCH] LU-9489 lod: keep minimum LOVEA size For a PFL file, some of its component could be un-instantiated, and their lov_ost_data_v1 array is not needed, we should keep its LOVEA as small as possible. An unstantiated component's stripe offset should be set. Signed-off-by: Bobi Jam Change-Id: I1cd70399446f063dca4de5b61d7e1dc2c8dde37c Reviewed-on: https://review.whamcloud.com/27089 Tested-by: Jenkins Reviewed-by: Niu Yawei Tested-by: Maloo Reviewed-by: Andreas Dilger --- lustre/lod/lod_internal.h | 20 ++++++++++++++++++ lustre/lod/lod_lov.c | 50 ++++++++++++++++++++++++++++++-------------- lustre/lod/lod_object.c | 4 ++++ lustre/lov/lov_internal.h | 38 +++++++++++++++++++-------------- lustre/tests/conf-sanity.sh | 3 ++- lustre/tests/sanity-lfsck.sh | 4 ++-- lustre/tests/sanity-pfl.sh | 48 +++++++++++++++++++++++++++++++++++++++--- 7 files changed, 129 insertions(+), 38 deletions(-) diff --git a/lustre/lod/lod_internal.h b/lustre/lod/lod_internal.h index 03c19cc..b388347 100644 --- a/lustre/lod/lod_internal.h +++ b/lustre/lod/lod_internal.h @@ -560,6 +560,26 @@ lod_comp_inited(const struct lod_layout_component *entry) return entry->llc_flags & LCME_FL_INIT; } +/** + * For a PFL file, some of its component could be un-instantiated, so + * that their lov_ost_data_v1 array is not needed, we'd use this function + * to reduce the LOVEA buffer size. + * + * Note: if llc_ostlist contains value, we'd need lov_ost_data_v1 array to + * save the specified OST index list. + */ +static inline void +lod_comp_shrink_stripecount(struct lod_layout_component *lod_comp, + __u16 *stripe_count) +{ + /** + * Need one lov_ost_data_v1 to store invalid ost_idx, please refer to + * lod_parse_striping() + */ + if (!lod_comp_inited(lod_comp) && lod_comp->llc_ostlist.op_count == 0) + *stripe_count = 1; +} + void lod_fix_desc(struct lov_desc *desc); void lod_fix_desc_qos_maxage(__u32 *val); void lod_fix_desc_pattern(__u32 *val); diff --git a/lustre/lod/lod_lov.c b/lustre/lod/lod_lov.c index 6b01a87..671b768 100644 --- a/lustre/lod/lod_lov.c +++ b/lustre/lod/lod_lov.c @@ -789,6 +789,8 @@ static int lod_gen_component_ea(const struct lu_env *env, objs = &v3->lmm_objects[0]; } stripecnt = lod_comp_entry_stripecnt(lo, lod_comp, is_dir); + if (!is_dir && lo->ldo_is_composite) + lod_comp_shrink_stripecount(lod_comp, &stripecnt); if (is_dir || lod_comp->llc_pattern & LOV_PATTERN_F_RELEASED) GOTO(done, rc = 0); @@ -1273,26 +1275,42 @@ int lod_parse_striping(const struct lu_env *env, struct lod_object *lo, /** * If uninstantiated template component has valid l_ost_idx, - * then use has specified ost list for this component. + * then user has specified ost list for this component. */ - if (!lod_comp_inited(lod_comp) && - objs[0].l_ost_idx != (__u32)-1UL) { - /** - * load the user specified ost list, when this - * component is instantiated later, it will be used - * in lod_alloc_ost_list(). - */ - lod_comp->llc_ostlist.op_count = lod_comp->llc_stripenr; - lod_comp->llc_ostlist.op_size = + if (!lod_comp_inited(lod_comp)) { + if (objs[0].l_ost_idx != (__u32)-1UL) { + /** + * load the user specified ost list, when this + * component is instantiated later, it will be + * used in lod_alloc_ost_list(). + */ + lod_comp->llc_ostlist.op_count = + lod_comp->llc_stripenr; + lod_comp->llc_ostlist.op_size = lod_comp->llc_stripenr * sizeof(__u32); - OBD_ALLOC(lod_comp->llc_ostlist.op_array, - lod_comp->llc_ostlist.op_size); - if (!lod_comp->llc_ostlist.op_array) - GOTO(out, rc = -ENOMEM); + OBD_ALLOC(lod_comp->llc_ostlist.op_array, + lod_comp->llc_ostlist.op_size); + if (!lod_comp->llc_ostlist.op_array) + GOTO(out, rc = -ENOMEM); - for (j = 0; j < lod_comp->llc_stripenr; j++) - lod_comp->llc_ostlist.op_array[j] = + for (j = 0; j < lod_comp->llc_stripenr; j++) + lod_comp->llc_ostlist.op_array[j] = le32_to_cpu(objs[j].l_ost_idx); + + /** + * this component OST objects starts from the + * first ost_idx, lod_alloc_ost_list() will + * check this. + */ + lod_comp->llc_stripe_offset = objs[0].l_ost_idx; + } else { + /** + * for uninstantiated component, + * lmm_layout_gen stores default stripe offset. + */ + lod_comp->llc_stripe_offset = + lmm->lmm_layout_gen; + } } /* skip un-instantiated component object initialization */ diff --git a/lustre/lod/lod_object.c b/lustre/lod/lod_object.c index 2318bdb..f4a86a9 100644 --- a/lustre/lod/lod_object.c +++ b/lustre/lod/lod_object.c @@ -2190,6 +2190,10 @@ static int lod_comp_md_size(struct lod_object *lo, bool is_dir) magic = comp_entries[i].llc_pool ? LOV_MAGIC_V3 : LOV_MAGIC_V1; stripenr = lod_comp_entry_stripecnt(lo, &comp_entries[i], is_dir); + if (!is_dir && is_composite) + lod_comp_shrink_stripecount(&comp_entries[i], + &stripenr); + size += lov_user_md_size(stripenr, magic); LASSERT(size % sizeof(__u64) == 0); } diff --git a/lustre/lov/lov_internal.h b/lustre/lov/lov_internal.h index b9743b6..6f74868 100644 --- a/lustre/lov/lov_internal.h +++ b/lustre/lov/lov_internal.h @@ -80,6 +80,21 @@ struct lov_stripe_md { struct lov_stripe_md_entry *lsm_entries[]; }; +static inline bool lsme_inited(const struct lov_stripe_md_entry *lsme) +{ + return lsme->lsme_flags & LCME_FL_INIT; +} + +static inline bool lsm_entry_inited(const struct lov_stripe_md *lsm, int index) +{ + return lsme_inited(lsm->lsm_entries[index]); +} + +static inline bool lsm_is_composite(__u32 magic) +{ + return !!(magic & LOV_MAGIC_COMP_V1); +} + static inline size_t lov_comp_md_size(const struct lov_stripe_md *lsm) { struct lov_stripe_md_entry *lsme; @@ -94,8 +109,15 @@ static inline size_t lov_comp_md_size(const struct lov_stripe_md *lsm) size = sizeof(struct lov_comp_md_v1); for (entry = 0; entry < lsm->lsm_entry_count; entry++) { + u16 stripe_count; + lsme = lsm->lsm_entries[entry]; + if (lsme_inited(lsme)) + stripe_count = lsme->lsme_stripe_count; + else + stripe_count = 0; + size += sizeof(*lsme); size += lov_mds_md_size(lsme->lsme_stripe_count, lsme->lsme_magic); @@ -338,20 +360,4 @@ static inline void lov_lsm2layout(struct lov_stripe_md *lsm, ol->ol_comp_id = 0; } } - -static inline bool lsme_inited(const struct lov_stripe_md_entry *lsme) -{ - return lsme->lsme_flags & LCME_FL_INIT; -} - -static inline bool lsm_entry_inited(const struct lov_stripe_md *lsm, int index) -{ - return lsme_inited(lsm->lsm_entries[index]); -} - -static inline bool lsm_is_composite(__u32 magic) -{ - return !!(magic & LOV_MAGIC_COMP_V1); -} - #endif diff --git a/lustre/tests/conf-sanity.sh b/lustre/tests/conf-sanity.sh index 525d436..1fdde25 100755 --- a/lustre/tests/conf-sanity.sh +++ b/lustre/tests/conf-sanity.sh @@ -6158,7 +6158,8 @@ test_87() { #LU-6544 #set xattr $SETSTRIPE -E 1M -c 1 -E 64M -c 1 -E -1 -c -1 $file || error "Create file with 3 components failed" - i=$($GETSTRIPE -I 3 -c $file) + $TRUNCATE $file $((1024*1024*64+1)) || error "truncate file failed" + i=$($GETSTRIPE -I3 -c $file) || error "get 3rd stripe count failed" if [ $i -ne $OSTCOUNT ]; then left_size=$(expr $left_size + $(expr $OSTCOUNT - $i) \* 24) echo -n "Since only $i out $OSTCOUNT OSTs are used, " diff --git a/lustre/tests/sanity-lfsck.sh b/lustre/tests/sanity-lfsck.sh index a4d68b2..79e130e 100644 --- a/lustre/tests/sanity-lfsck.sh +++ b/lustre/tests/sanity-lfsck.sh @@ -2508,8 +2508,8 @@ test_18e() { sleep 2 echo "Write new data to f2/f4 to modify the new created OST-object." - echo "dummy" >> $DIR/$tdir/a1/f2 - echo "dummy" >> $DIR/$tdir/a1/f4 + echo "dummy" >> $DIR/$tdir/a1/f2 || error "write a1/f2 failed" + echo "dummy" >> $DIR/$tdir/a1/f4 || error "write a1/f4 failed" do_facet $SINGLEMDS $LCTL set_param fail_val=0 fail_loc=0 diff --git a/lustre/tests/sanity-pfl.sh b/lustre/tests/sanity-pfl.sh index 3c83c37..cc4d01e 100644 --- a/lustre/tests/sanity-pfl.sh +++ b/lustre/tests/sanity-pfl.sh @@ -361,7 +361,7 @@ test_9() { local f2=$($LFS getstripe -I2 $comp_file | awk '/l_fid:/ {print $7}') echo "after MDS recovery, the ost fid of 2nd component is $f2" - [ $f1 == $f2 ] || error "$f1 != $f2" + [ "x$f1" == "x$f2" ] || error "$f1 != $f2" } run_test 9 "Replay layout extend object instantiation" @@ -479,8 +479,14 @@ test_12() { # specify ost list for component $LFS setstripe -E1m -c2 -o0,1 -E2m -c2 -o1,2 -E3m -c2 -o2,1 \ - -E4m -c2 -o2,0 -E-1 $file || + -E4m -c1 -i2 -E-1 $file || error "Create $file failed" + + # clear lod component cache + stop $SINGLEMDS || error "stop MDS" + local MDT_DEV=$(mdsdevname ${SINGLEMDS//mds/}) + start $SINGLEMDS $MDT_DEV $MDS_MOUNT_OPTS || error "start MDS" + # instantiate all components $TRUNCATE $file $((1024*1024*4+1)) @@ -499,7 +505,7 @@ test_12() { local o4=$($LFS getstripe -I4 $file | awk '/l_ost_idx:/ {printf("%d",$5)}') - [[ $o4 != "20" ]] && error "$o4 is not 20" + [[ $o4 != "2" ]] && error "$o4 is not 2" return 0 } @@ -630,6 +636,42 @@ test_15() { } run_test 15 "Verify component options for lfs find" +test_17() { + [ $OSTCOUNT -lt 2 ] && skip "needs >= 2 OSTs" && return + local file=$DIR/$tdir/$tfile + test_mkdir -p $DIR/$tdir + rm -f $file + + $LFS setstripe -E1m -E2m -c2 -E-1 -c-1 $file || + error "Create $file failed" + + local s1=$($LFS getstripe -I1 -v $file | awk '/lcme_size:/{print $2}') + local s2=$($LFS getstripe -I2 -v $file | awk '/lcme_size:/{print $2}') + local s3=$($LFS getstripe -I3 -v $file | awk '/lcme_size:/{print $2}') + echo "1st init: comp size 1:$s1 2:$s2 3:$s3" + + # init 2nd component + $TRUNCATE $file $((1024*1024+1)) + local s1n=$($LFS getstripe -I1 -v $file | awk '/lcme_size:/{print $2}') + local s2n=$($LFS getstripe -I2 -v $file | awk '/lcme_size:/{print $2}') + echo "2nd init: comp size 1:$s1n 2:$s2n 3:$s3" + + [ $s1 -eq $s1n ] || error "1st comp size $s1 should == $s1n" + [ $s2 -lt $s2n ] || error "2nd comp size $s2 should < $s2n" + + # init 3rd component + $TRUNCATE $file $((1024*1024*2+1)) + s1n=$($LFS getstripe -I1 -v $file | awk '/lcme_size:/{print $2}') + s2n=$($LFS getstripe -I2 -v $file | awk '/lcme_size:/{print $2}') + local s3n=$($LFS getstripe -I3 -v $file | awk '/lcme_size:/{print $2}') + echo "3rd init: comp size 1:$s1n 2:$s2n 3:$s3n" + + [ $s1 -eq $s1n ] || error "1st comp size $s1 should == $s1n" + [ $s2 -lt $s2n ] || error "2nd comp size $s2 should < $s2n" + [ $s3 -lt $s3n ] || error "3rd comp size $s3 should < $s3n" +} +run_test 17 "Verify LOVEA grows with more component inited" + complete $SECONDS check_and_cleanup_lustre exit_status -- 1.8.3.1