From 6d8358544e2456c6c8d4942fe3c26af96539d121 Mon Sep 17 00:00:00 2001 From: Bobi Jam Date: Mon, 13 May 2024 11:06:37 -0700 Subject: [PATCH] LU-16831 lfs: limit stripe count for component size If stripe count is larger than component_size/stripe_size, some allocated OST objects are created but inaccessible. This patch reduces the number of stripes in that case to avoid this. Lustre-change: https://review.whamcloud.com/51143 Lustre-commit: a250ecb959a98c2ec0a01bbca9d943a19b8fa078 Signed-off-by: Bobi Jam Change-Id: I117ed8a7696c6c6adcdd0c2c6531a958cc53bd51 Reviewed-by: Andreas Dilger Reviewed-by: Jian Yu Reviewed-on: https://review.whamcloud.com/c/fs/lustre-release/+/53775 Tested-by: jenkins Tested-by: Maloo Reviewed-by: Oleg Drokin --- lustre/lod/lod_object.c | 23 ++++++++++++- lustre/lod/lod_qos.c | 45 +++++++++++++++++++++++++ lustre/tests/sanity-lfsck.sh | 10 +++--- lustre/tests/sanity-pfl.sh | 80 +++++++++++++++++++++++++++++++++++++++----- 4 files changed, 143 insertions(+), 15 deletions(-) diff --git a/lustre/lod/lod_object.c b/lustre/lod/lod_object.c index de4b6fa..cd8f0cb 100644 --- a/lustre/lod/lod_object.c +++ b/lustre/lod/lod_object.c @@ -2781,8 +2781,29 @@ static int lod_declare_layout_add(const struct lu_env *env, lod_comp->llc_stripe_offset = v1->lmm_stripe_offset; lod_comp->llc_flags = comp_v1->lcm_entries[i].lcme_flags; - lod_comp->llc_stripe_count = v1->lmm_stripe_count; lod_comp->llc_stripe_size = v1->lmm_stripe_size; + lod_comp->llc_stripe_count = v1->lmm_stripe_count; + /** + * limit stripe count so that it's less than/equal to + * extent_size / stripe_size. + * + * Note: extension size reused llc_stripe_size field and + * uninstantiated component could be defined with + * extent_start == extent_end as extension component will + * expand it later. + */ + if (!(lod_comp->llc_flags & LCME_FL_EXTENSION) && + (lod_comp_inited(lod_comp) || + lod_comp->llc_extent.e_start < + lod_comp->llc_extent.e_end) && + lod_comp->llc_stripe_count != (__u16)-1 && + ext->e_end != (__u64)-1 && + (__u64)(lod_comp->llc_stripe_count * + lod_comp->llc_stripe_size) > + (ext->e_end - ext->e_start)) + lod_comp->llc_stripe_count = + DIV_ROUND_UP(ext->e_end - ext->e_start, + lod_comp->llc_stripe_size); lod_adjust_stripe_info(lod_comp, desc, 0); if (v1->lmm_magic == LOV_USER_MAGIC_V3) { diff --git a/lustre/lod/lod_qos.c b/lustre/lod/lod_qos.c index 4d32877..b470764 100644 --- a/lustre/lod/lod_qos.c +++ b/lustre/lod/lod_qos.c @@ -2111,6 +2111,29 @@ int lod_use_defined_striping(const struct lu_env *env, lod_comp->llc_pattern = le32_to_cpu(v1->lmm_pattern); lod_comp->llc_stripe_size = le32_to_cpu(v1->lmm_stripe_size); lod_comp->llc_stripe_count = le16_to_cpu(v1->lmm_stripe_count); + /** + * limit stripe count so that it's less than/equal to + * extent_size / stripe_size. + * + * Note: extension size reused llc_stripe_size field and + * uninstantiated component could be defined with + * extent_start == extent_end as extension component will + * expand it later. + */ + if (mo->ldo_is_composite && + !(lod_comp->llc_flags & LCME_FL_EXTENSION) && + (lod_comp_inited(lod_comp) || + lod_comp->llc_extent.e_start < + lod_comp->llc_extent.e_end) && + lod_comp->llc_stripe_count != (__u16)-1 && + lod_comp->llc_extent.e_end != (__u64)-1 && + (__u64)lod_comp->llc_stripe_count * + lod_comp->llc_stripe_size > + (lod_comp->llc_extent.e_end - lod_comp->llc_extent.e_start)) + lod_comp->llc_stripe_count = + DIV_ROUND_UP(lod_comp->llc_extent.e_end - + lod_comp->llc_extent.e_start, + lod_comp->llc_stripe_size); lod_comp->llc_layout_gen = le16_to_cpu(v1->lmm_layout_gen); /** * The stripe_offset of an uninit-ed component is stored in @@ -2396,6 +2419,28 @@ int lod_qos_parse_config(const struct lu_env *env, struct lod_object *lo, lod_comp->llc_stripe_count); GOTO(free_comp, rc = -EINVAL); } + /** + * limit stripe count so that it's less than/equal to + * extent_size / stripe_size. + * + * Note: extension size reused llc_stripe_size field and + * uninstantiated component could be defined with + * extent_start == extent_end as extension component will + * expand it later. + */ + if (lo->ldo_is_composite && + !(lod_comp->llc_flags & LCME_FL_EXTENSION) && + lod_comp->llc_stripe_count != (__u16)-1 && + (lod_comp_inited(lod_comp) || + lod_comp->llc_extent.e_start < + lod_comp->llc_extent.e_end) && + lod_comp->llc_extent.e_end != (__u64)-1 && + lod_comp->llc_stripe_count * lod_comp->llc_stripe_size > + (lod_comp->llc_extent.e_end - lod_comp->llc_extent.e_start)) + lod_comp->llc_stripe_count = + DIV_ROUND_UP(lod_comp->llc_extent.e_end - + lod_comp->llc_extent.e_start, + lod_comp->llc_stripe_size); lod_comp->llc_stripe_offset = v1->lmm_stripe_offset; lod_qos_set_pool(lo, i, pool_name, v1); diff --git a/lustre/tests/sanity-lfsck.sh b/lustre/tests/sanity-lfsck.sh index dbc8cc7..116e872 100644 --- a/lustre/tests/sanity-lfsck.sh +++ b/lustre/tests/sanity-lfsck.sh @@ -5481,13 +5481,13 @@ test_36a() { lctl get_param osc.*.*grant* stack_trap "lfs df $DIR; lfs df -i $DIR; lctl get_param osc.*.*grant*" - $LFS setstripe -N -E 1M -o 0,1 -E -1 -o 2 -N -E 2M -o 1,2 -E -1 -o 0 \ + $LFS setstripe -N -E 2M -o 0,1 -E -1 -o 2 -N -E 2M -o 1,2 -E -1 -o 0 \ -N -E 3M -o 2,0 -E -1 -o 1 $DIR/$tdir/f0 || error "(0) Fail to create mirror file $DIR/$tdir/f0" - $LFS setstripe -N -E 1M -o 0,1 -E -1 -o 2 -N -E 2M -o 1,2 -E -1 -o 0 \ + $LFS setstripe -N -E 2M -o 0,1 -E -1 -o 2 -N -E 2M -o 1,2 -E -1 -o 0 \ -N -E 3M -o 2,0 -E -1 -o 1 $DIR/$tdir/f1 || error "(1) Fail to create mirror file $DIR/$tdir/f1" - $LFS setstripe -N -E 1M -o 0,1 -E -1 -o 2 -N -E 2M -o 1,2 -E -1 -o 0 \ + $LFS setstripe -N -E 2M -o 0,1 -E -1 -o 2 -N -E 2M -o 1,2 -E -1 -o 0 \ -N -E 3M -o 2,0 -E -1 -o 1 $DIR/$tdir/f2 || error "(2) Fail to create mirror file $DIR/$tdir/f2" @@ -5609,7 +5609,7 @@ test_36b() { check_mount_and_prep - $LFS setstripe -N -E 1M -o 0,1 -E -1 -o 2 -N -E 2M -o 1,2 -E -1 -o 0 \ + $LFS setstripe -N -E 2M -o 0,1 -E -1 -o 2 -N -E 2M -o 1,2 -E -1 -o 0 \ -N -E 3M -o 2,0 -E -1 -o 1 $DIR/$tdir/f0 || error "(0) Fail to create mirror file $DIR/$tdir/f0" @@ -5698,7 +5698,7 @@ test_36c() { check_mount_and_prep - $LFS setstripe -N -E 1M -o 0,1 -E -1 -o 2 -N -E 2M -o 1,2 -E -1 -o 0 \ + $LFS setstripe -N -E 2M -o 0,1 -E -1 -o 2 -N -E 2M -o 1,2 -E -1 -o 0 \ $DIR/$tdir/f0 || error "(0) Fail to create mirror file $DIR/$tdir/f0" diff --git a/lustre/tests/sanity-pfl.sh b/lustre/tests/sanity-pfl.sh index 81d25da..ec827df 100644 --- a/lustre/tests/sanity-pfl.sh +++ b/lustre/tests/sanity-pfl.sh @@ -119,8 +119,10 @@ test_0c() { test_mkdir $DIR/$tdir - $LFS setstripe -E -1 -C $LOV_MAX_STRIPE_COUNT -z 128M $comp_file || - error "Create $comp_file failed" + # extension size aligned to 64M and limit to 1G + $LFS setstripe -E-1 -S64K -C $LOV_MAX_STRIPE_COUNT \ + -z $((64*1024*((LOV_MAX_STRIPE_COUNT+1023)>>10<<10))) \ + $comp_file || error "Create $comp_file failed" local count=$($LFS getstripe -I1 -c $comp_file) [ $count -eq $LOV_MAX_STRIPE_COUNT ] || @@ -186,6 +188,66 @@ test_0d() { } run_test 0d "Verify comp end and stripe size" +test_0e() { + (( OSTCOUNT >= 2 )) || skip "needs >= 2 OSTs" + + (( MDS1_VERSION >= $(version_code 2.15.4.1) )) || + skip "need MDS >= 2.15.4.1 for stripe limits by size" + + local td=$DIR/$tdir + local tf=$td/$tfile + + test_mkdir $td + + # component size 1M can only utilize 1 stripe + $LFS setstripe -E1M -S1M -c2 -Eeof -S1M -c2 $tf || + error "failed to setstripe $tf" + + local sc=$($LFS getstripe -I1 -c $tf) + (( sc == 1 )) || { + $LFS getstripe $tf + error "expect 1 stripe count instread of $sc" + } + sc=$($LFS getstripe -I2 -c $tf) + (( sc == 2 )) || { + $LFS getstripe $tf + error "expect 2 stripe count instread of $sc" + } + + (( OSTCOUNT >= 3 )) || skip "needs >= 3 OSTs" + + rm -f $tf + # component size 2M can only utilize 2 stripes + $LFS setstripe -E2M -S1M -c3 -Eeof -S1M -c3 $tf || + error "failed to setstripe $tf" + sc=$($LFS getstripe -I1 -c $tf) + (( sc == 2 )) || { + $LFS getstripe $tf + error "expect 2 stripe count instread of $sc" + } + sc=$($LFS getstripe -I2 -c $tf) + (( sc == 3 )) || { + $LFS getstripe $tf + error "expect 3 stripe count instread of $sc" + } + + rm -f $tf + # 2nd component size 11M and stripe_size 4M should cross 3 stripes + $LFS setstripe -E1M -S1M -c3 -E12M -S4M -c3 $tf || + error "failed to setstripe $tf" + sc=$($LFS getstripe -I1 -c $tf) + (( sc == 1 )) || { + $LFS getstripe $tf + error "expect 1 stripe count instread of $sc" + } + sc=$($LFS getstripe -I2 -c $tf) + (( sc == 3 )) || { + $LFS getstripe $tf + error "expect 3 stripe count instread of $sc" + } +} +run_test 0e "lfs setstripe should limite stripe count for component size" + test_1a() { local comp_file=$DIR/$tdir/$tfile local rw_len=$((3 * 1024 * 1024)) # 3M @@ -222,7 +284,7 @@ test_1b() { test_mkdir $DIR/$tdir - $LFS setstripe -E 1m -S 1m -o 0,0 -E -1 -o 1,1,0,0 $comp_file || + $LFS setstripe -E 2m -S 1m -o 0,0 -E -1 -o 1,1,0,0 $comp_file || error "Create $comp_file failed" #instantiate all components, so that objs are allocted @@ -258,7 +320,7 @@ test_1c() { test_mkdir $DIR/$tdir - $LFS setstripe -E 1m -C 10 -E 10M -C 100 -E -1 \ + $LFS setstripe -E 10m -C 10 -E 110M -C 100 -E -1 \ -C $LOV_MAX_STRIPE_COUNT $comp_file || error "Create $comp_file failed" @@ -711,8 +773,8 @@ test_12() { rm -f $file # specify ost list for component - $LFS setstripe -E 1M -S 1M -c 2 -o 0,1 -E 2M -c 2 -o 1,2 \ - -E 3M -c 2 -o 2,1 -E 4M -c 1 -i 2 -E -1 $file || + $LFS setstripe -E 2M -S 1M -c 2 -o 0,1 -E 4M -c 2 -o 1,2 \ + -E 6M -c 2 -o 2,1 -E 8M -c 1 -i 2 -E -1 $file || error "Create $file failed" # clear lod component cache @@ -721,7 +783,7 @@ test_12() { start $SINGLEMDS $MDT_DEV $MDS_MOUNT_OPTS || error "start MDS" # instantiate all components - $TRUNCATE $file $((1024*1024*4+1)) + $TRUNCATE $file $((1024*1024*8+1)) #verify object alloc order local o1=$($LFS getstripe -I1 $file | @@ -1057,7 +1119,7 @@ test_17() { test_mkdir -p $DIR/$tdir rm -f $file - $LFS setstripe -E 1M -S 1M -E 2M -c 2 -E -1 -c -1 $file || + $LFS setstripe -E 1M -S 1M -E 3M -c 2 -E -1 -c -1 $file || error "Create $file failed" local s1=$($LFS getstripe -I1 -v $file | awk '/lcme_size:/{print $2}') @@ -1075,7 +1137,7 @@ test_17() { [ $s2 -lt $s2n ] || error "2nd comp size $s2 should < $s2n" # init 3rd component - $TRUNCATE $file $((1024*1024*2+1)) + $TRUNCATE $file $((1024*1024*3+1)) s1n=$($LFS getstripe -I1 -v $file | awk '/lcme_size:/{print $2}') s2n=$($LFS getstripe -I2 -v $file | awk '/lcme_size:/{print $2}') local s3n=$($LFS getstripe -I3 -v $file | awk '/lcme_size:/{print $2}') -- 1.8.3.1