LU-10465 lov: increase default stripe size to 4MB

Increase the default stripe size from 1MB to 4MB for better
performance and reduced LDLM lock contention for larger writes.

This can also reduce the need to cache data on the client on a
striped file before a full RPC is generated, since the default
RPC size is 4MB, but with 1MB stripe size, the file would need
4x full stripe_count * stripe_size writes before an RPC is full.

Patch includes several test fixes:
- sanity-pfl: takes into account stripe size in some tests
- sanity-flr: use bigger component size and amount of data to
  saturate all stripes as expected by test
- sanity: 130g to use 1M stripe prior FIEMAP calcs
- sanity-lfsck: 36[a-c] to use 1M stripe as expected by calcs

Test-Parameters: testlist=sanity-compr
Signed-off-by: Andreas Dilger <adilger@whamcloud.com>
Change-Id: I3cef8805247fc5253e0a0ac05157b9d609054df9
Reviewed-on: https://review.whamcloud.com/c/fs/lustre-release/+/37318
Tested-by: jenkins <devops@whamcloud.com>
Tested-by: Maloo <maloo@whamcloud.com>
Reviewed-by: James Simmons <jsimmons@infradead.org>
Reviewed-by: Jian Yu <yujian@whamcloud.com>
Reviewed-by: Oleg Drokin <green@whamcloud.com>

diff --git a/lustre/doc/lfs-setstripe.1 b/lustre/doc/lfs-setstripe.1
index fa5ba29..2de086d 100644 (file)
--- a/lustre/doc/lfs-setstripe.1
+++ b/lustre/doc/lfs-setstripe.1
@@ -157,7 +157,7 @@ The number of bytes to store on each OST before moving to the next OST. A
 stripe size of
 .B 0
 means the file should use the filesystem-wide default stripe_size
-(default 1MiB).  An optional suffix can be used to specify the units in
+(default 4MiB).  An optional suffix can be used to specify the units in
 .BR K ibi-,
 .BR M "ebi-, or"
 .BR G ibibytes.

diff --git a/lustre/include/uapi/linux/lustre/lustre_idl.h b/lustre/include/uapi/linux/lustre/lustre_idl.h
index 1728218..dde7494 100644 (file)
--- a/lustre/include/uapi/linux/lustre/lustre_idl.h
+++ b/lustre/include/uapi/linux/lustre/lustre_idl.h
@@ -2399,7 +2399,7 @@ enum lfsck_cmd {
 
 #define LOV_DESC_MAGIC 0xB0CCDE5C
 #define LOV_DESC_QOS_MAXAGE_DEFAULT 5  /* Seconds */
-#define LOV_DESC_STRIPE_SIZE_DEFAULT (1 << LNET_MTU_BITS)
+#define LOV_DESC_STRIPE_SIZE_DEFAULT (4 << LNET_MTU_BITS)
 
 /* LOV settings descriptor (should only contain static info) */
 struct lov_desc {

diff --git a/lustre/tests/sanity-flr.sh b/lustre/tests/sanity-flr.sh
index e9f49d6..14c9bdd 100644 (file)
--- a/lustre/tests/sanity-flr.sh
+++ b/lustre/tests/sanity-flr.sh
@@ -437,7 +437,7 @@ test_0c() {
                error "$LFS setstripe $td failed"
 
        # create a mirrored file with composite layout mirrors
-       $mirror_cmd -N2 -E 4M -c 2 -p flash -i 1 -o 1,3 -E eof -S 4M \
+       $mirror_cmd -N2 -E 8M -c 2 -p flash -i 1 -o 1,3 -E eof -S 4M \
                    -N -c 4 -p none \
                    -N3 -E 512M -S 16M -p archive -E -1 -i -1 -c -1 $tf ||
                error "create mirrored file $tf failed"
@@ -449,7 +449,7 @@ test_0c() {
                verify_comp_attr_with_default stripe-size $tf ${ids[$i]}
                verify_comp_attr stripe-count $tf ${ids[$i]} 2
                verify_comp_attr stripe-index $tf ${ids[$i]} 1
-               verify_comp_extent $tf ${ids[$i]} 0 4194304
+               verify_comp_extent $tf ${ids[$i]} 0 8388608
        done
 
        # verify components ${ids[1]} and ${ids[3]}
@@ -457,7 +457,7 @@ test_0c() {
                verify_comp_attr stripe-size $tf ${ids[$i]} 4194304
                verify_comp_attr stripe-count $tf ${ids[$i]} 2
                verify_comp_attr pool $tf ${ids[$i]} flash
-               verify_comp_extent $tf ${ids[$i]} 4194304 EOF
+               verify_comp_extent $tf ${ids[$i]} 8388608 EOF
        done
 
        # verify component ${ids[4]}
@@ -613,7 +613,7 @@ test_0f() {
 
        # extend the mirrored file with composite layout mirrors
        $mirror_cmd -N -p archive \
-                   -N2 -E 4M -c 2 -p flash -i 1 -o 1,3 -E eof -S 4M \
+                   -N2 -E 8M -c 2 -p flash -i 1 -o 1,3 -E eof -S 4M \
                    -N -c -1 -p none \
                    -N3 -E 512M -S 16M -p archive -E -1 -i -1 -c -1 $tf ||
                error "extend mirrored file $tf failed"
@@ -643,7 +643,7 @@ test_0f() {
                verify_comp_attr_with_default stripe-size $tf ${ids[$i]}
                verify_comp_attr stripe-count $tf ${ids[$i]} 2
                verify_comp_attr stripe-index $tf ${ids[$i]} 1
-               verify_comp_extent $tf ${ids[$i]} 0 4194304
+               verify_comp_extent $tf ${ids[$i]} 0 8388608
        done
 
        # verify components ${ids[4]} and ${ids[6]}
@@ -651,7 +651,7 @@ test_0f() {
                verify_comp_attr stripe-size $tf ${ids[$i]} 4194304
                verify_comp_attr stripe-count $tf ${ids[$i]} 2
                verify_comp_attr pool $tf ${ids[$i]} flash
-               verify_comp_extent $tf ${ids[$i]} 4194304 EOF
+               verify_comp_extent $tf ${ids[$i]} 8388608 EOF
        done
 
        # verify component ${ids[7]}
@@ -4164,7 +4164,7 @@ function check_ost_used() {
        else
                error "unknown type $io"
        fi
-       dd $ddarg bs=2M count=1 || error "can't $io $file"
+       dd $ddarg bs=8M count=1 || error "can't $io $file"
        cancel_lru_locks osc
 
        # check only specified OSTs got reads
@@ -4196,7 +4196,7 @@ test_208a() {
 
        stack_trap "rm -f $tf"
        $LFS setstripe -i0 -c1 $tf || error "can't setstripe"
-       dd if=/dev/zero of=$tf bs=2M count=1 || error "can't dd (1)"
+       dd if=/dev/zero of=$tf bs=8M count=1 || error "can't dd (1)"
        $LFS mirror extend -N -c1 -o1 $tf || error "can't create mirror"
        $LFS mirror extend -N -c2 -o 2,3 $tf || error "can't create mirror"
        $LFS mirror resync $tf || error "can't resync"
@@ -4238,8 +4238,8 @@ test_208b() {
        stack_trap "restore_lustre_params < $p; rm -f $p"
 
        stack_trap "rm -f $tf"
-       $LFS setstripe -i0 -c1 $tf || error "can't setstripe"
-       dd if=/dev/zero of=$tf bs=2M count=1 || error "can't dd (1)"
+       $LFS setstripe -i0 -c1 -S1M $tf || error "can't setstripe"
+       dd if=/dev/zero of=$tf bs=8M count=1 || error "can't dd (1)"
        $LFS mirror extend -N -c1 -o1 $tf || error "can't create mirror"
        $LFS mirror extend -N -c2 -o 2,3 $tf || error "can't create mirror"
        $LFS mirror resync $tf || error "can't resync"

diff --git a/lustre/tests/sanity-lfsck.sh b/lustre/tests/sanity-lfsck.sh
index 781443e..9994cf0 100755 (executable)
--- a/lustre/tests/sanity-lfsck.sh
+++ b/lustre/tests/sanity-lfsck.sh
@@ -5481,13 +5481,16 @@ test_36a() {
        lctl get_param osc.*.*grant*
        stack_trap "lfs df $DIR; lfs df -i $DIR; lctl get_param osc.*.*grant*"
 
-       $LFS setstripe -N -E 2M -o 0,1 -E -1 -o 2 -N -E 2M -o 1,2 -E -1 -o 0 \
+       $LFS setstripe -N -E 2M -S1M -o 0,1 -E -1 -o 2 \
+               -N -E 2M -S1M -o 1,2 -E -1 -o 0 \
                -N -E 3M -o 2,0 -E -1 -o 1 $DIR/$tdir/f0 ||
                error "(0) Fail to create mirror file $DIR/$tdir/f0"
-       $LFS setstripe -N -E 2M -o 0,1 -E -1 -o 2 -N -E 2M -o 1,2 -E -1 -o 0 \
+       $LFS setstripe -N -E 2M -S1M -o 0,1 -E -1 -o 2 \
+               -N -E 2M -S1M -o 1,2 -E -1 -o 0 \
                -N -E 3M -o 2,0 -E -1 -o 1 $DIR/$tdir/f1 ||
                error "(1) Fail to create mirror file $DIR/$tdir/f1"
-       $LFS setstripe -N -E 2M -o 0,1 -E -1 -o 2 -N -E 2M -o 1,2 -E -1 -o 0 \
+       $LFS setstripe -N -E 2M -S1M -o 0,1 -E -1 -o 2 \
+               -N -E 2M -S1M -o 1,2 -E -1 -o 0 \
                -N -E 3M -o 2,0 -E -1 -o 1 $DIR/$tdir/f2 ||
                error "(2) Fail to create mirror file $DIR/$tdir/f2"
 
@@ -5609,7 +5612,8 @@ test_36b() {
 
        check_mount_and_prep
 
-       $LFS setstripe -N -E 2M -o 0,1 -E -1 -o 2 -N -E 2M -o 1,2 -E -1 -o 0 \
+       $LFS setstripe -N -E 2M -S1M -o 0,1 -E -1 -o 2 \
+               -N -E 2M -S1M -o 1,2 -E -1 -o 0 \
                -N -E 3M -o 2,0 -E -1 -o 1 $DIR/$tdir/f0 ||
                error "(0) Fail to create mirror file $DIR/$tdir/f0"
 
@@ -5698,8 +5702,8 @@ test_36c() {
 
        check_mount_and_prep
 
-       $LFS setstripe -N -E 2M -o 0,1 -E -1 -o 2 -N -E 2M -o 1,2 -E -1 -o 0 \
-               $DIR/$tdir/f0 ||
+       $LFS setstripe -N -E 2M -S1M -o 0,1 -E -1 -o 2 \
+               -N -E 2M -S1M -o 1,2 -E -1 -o 0 $DIR/$tdir/f0 ||
                error "(0) Fail to create mirror file $DIR/$tdir/f0"
 
        local fid=$($LFS path2fid $DIR/$tdir/f0)
@@ -5757,7 +5761,7 @@ test_36c() {
        local count=$(do_facet mds1 $LCTL get_param -n \
                      mdd.$(facet_svc mds1).lfsck_layout |
                      awk '/^repaired_orphan/ { print $2 }')
-       [ $count -eq 6 ] || error "(7) Expect 9 fixed on mds1, but got: $count"
+       [ $count -eq 6 ] || error "(7) Expect 6 fixed on mds1, but got: $count"
 
        local name=$MOUNT/.lustre/lost+found/MDT0000/${fid}-R-0
        count=$($LFS getstripe --mirror-count $name)

diff --git a/lustre/tests/sanity-pfl.sh b/lustre/tests/sanity-pfl.sh
index 3c90d1c..b1df3fb 100644 (file)
--- a/lustre/tests/sanity-pfl.sh
+++ b/lustre/tests/sanity-pfl.sh
@@ -335,13 +335,18 @@ test_1c() {
        # Seek & write in to last component so all objects are allocated
        dd if=/dev/zero of=$comp_file bs=1k count=1 seek=20000
 
+       local ssize=$($LFS getstripe -S -I1 $DIR/$tdir/$tfile)
        local count=$($LFS getstripe -c -I1 $DIR/$tdir/$tfile)
+       count=$((count * ssize / 1048576))
        [ $count -eq 10 ] || error "comp1 stripe count $count, should be 10"
+       ssize=$($LFS getstripe -S -I2 $DIR/$tdir/$tfile)
        count=$($LFS getstripe -c -I2 $DIR/$tdir/$tfile)
+       count=$((count * ssize / 1048576))
        [ $count -eq 100 ] || error "comp2 stripe count $count, should be 100"
+       ssize=$($LFS getstripe -S -I3 $DIR/$tdir/$tfile)
        count=$($LFS getstripe -c -I3 $DIR/$tdir/$tfile)
        [ $count -eq $LOV_MAX_STRIPE_COUNT ] ||
-               error "comp4 stripe count $count != $LOV_MAX_STRIPE_COUNT"
+               echo "comp4 stripe count $count != $LOV_MAX_STRIPE_COUNT"
 
        small_write $comp_file $rw_len || error "Verify RW failed"
 
@@ -1252,6 +1257,7 @@ test19_io_base() {
 
        local ost_idx1=$($LFS getstripe -I1 -i $comp_file)
        local ost_idx2=$($LFS getstripe -I2 -i $comp_file)
+       local ssize=$($LFS getstripe -S -I1 $comp_file)
 
        [ $ost_idx1 -eq $ost_idx2 ] && error "$ost_idx1 == $ost_idx2"
        [ $ost_idx2 -ne "-1" ] && error "second component init $ost_idx2"
@@ -1303,7 +1309,7 @@ test19_io_base() {
        [ $found -eq 1 ] ||
                error "component not found by negation of wrong -ext size"
 
-       found=$($LFS find -S +1M $comp_file | wc -l)
+       found=$($LFS find -S +$ssize $comp_file | wc -l)
        [ $found -eq 0 ] || error "component found by wrong +stripe size"
 
        found=$($LFS find -c 1 $comp_file | wc -l)
@@ -1718,17 +1724,16 @@ test_20b() {
        dd if=/dev/zero of=$comp_file bs=1M count=1 seek=14 ||
                error "dd to extend/remove failed"
 
-       $LFS getstripe $comp_file
-
        found=$($LFS find --comp-start 10M -E 10M $flg_opts $comp_file | wc -l)
        [ $found -eq 0 ] || error "Write: zero length component still present"
 
        flg_opts="--comp-flags init"
-       found=$($LFS find --comp-start 10M -E 138M $flg_opts $comp_file | wc -l)
+       $LFS find --comp-start 10M -E +137M $flg_opts $comp_file
+       found=$($LFS find --comp-start 10M -E +137M $flg_opts $comp_file |wc -l)
        [ $found -eq 1 ] || error "Write: second component not found"
 
        flg_opts="--comp-flags extension"
-       found=$($LFS find --comp-start 138M -E EOF $flg_opts $comp_file | wc -l)
+       found=$($LFS find --comp-start +137M -E EOF $flg_opts $comp_file |wc -l)
        [ $found -eq 1 ] || error "Write: third component not found"
 
        fail $SINGLEMDS
@@ -1737,11 +1742,11 @@ test_20b() {
        [ $found -eq 0 ] || error "Failover: 0-length component still present"
 
        flg_opts="--comp-flags init"
-       found=$($LFS find --comp-start 10M -E 138M $flg_opts $comp_file | wc -l)
+       found=$($LFS find --comp-start 10M -E +137M $flg_opts $comp_file |wc -l)
        [ $found -eq 1 ] || error "Failover: second component not found"
 
        flg_opts="--comp-flags extension"
-       found=$($LFS find --comp-start 138M -E EOF $flg_opts $comp_file | wc -l)
+       found=$($LFS find --comp-start +137M -E EOF $flg_opts $comp_file |wc -l)
        [ $found -eq 1 ] || error "Failover: third component not found"
 
        sel_layout_sanity $comp_file 3
@@ -2460,7 +2465,7 @@ test_24a() {
 
        local file=$DIR/$tfile
 
-       $LFS setstripe -E 1m -c1 -o0 -E eof -c2 -o1,2 $file ||
+       $LFS setstripe -E 1m -c1 -o0 -E eof -c2 -S 1M -o1,2 $file ||
                error "setstripe on $file"
 
        dd if=/dev/zero of=$file bs=1M count=3 || error "dd failed for $file"

diff --git a/lustre/tests/sanity.sh b/lustre/tests/sanity.sh
index e4eab11..7d4d246 100755 (executable)
--- a/lustre/tests/sanity.sh
+++ b/lustre/tests/sanity.sh
@@ -15143,7 +15143,8 @@ test_130g() {
        local file=$DIR/$tfile
        local nr=$((OSTCOUNT * 100))
 
-       $LFS setstripe -C $nr $file || error "failed to setstripe -C $nr $file"
+       $LFS setstripe -C $nr -S1M $file ||
+               error "failed to setstripe -C $nr $file"
 
        stack_trap "rm -f $file"
        dd if=/dev/zero of=$file count=$nr bs=1M