Increase the default stripe size from 1MB to 4MB for better
performance and reduced LDLM lock contention for larger writes.
This can also reduce the need to cache data on the client on a
striped file before a full RPC is generated, since the default
RPC size is 4MB, but with 1MB stripe size, the file would need
4x full stripe_count * stripe_size writes before an RPC is full.
Patch includes several test fixes:
- sanity-pfl: takes into account stripe size in some tests
- sanity-flr: use bigger component size and amount of data to
saturate all stripes as expected by test
- sanity: 130g to use 1M stripe prior FIEMAP calcs
- sanity-lfsck: 36[a-c] to use 1M stripe as expected by calcs
Test-Parameters: testlist=sanity-compr
Signed-off-by: Andreas Dilger <adilger@whamcloud.com>
Change-Id: I3cef8805247fc5253e0a0ac05157b9d609054df9
Reviewed-on: https://review.whamcloud.com/c/fs/lustre-release/+/37318
Tested-by: jenkins <devops@whamcloud.com>
Tested-by: Maloo <maloo@whamcloud.com>
Reviewed-by: James Simmons <jsimmons@infradead.org>
Reviewed-by: Jian Yu <yujian@whamcloud.com>
Reviewed-by: Oleg Drokin <green@whamcloud.com>
stripe size of
.B 0
means the file should use the filesystem-wide default stripe_size
stripe size of
.B 0
means the file should use the filesystem-wide default stripe_size
-(default 1MiB). An optional suffix can be used to specify the units in
+(default 4MiB). An optional suffix can be used to specify the units in
.BR K ibi-,
.BR M "ebi-, or"
.BR G ibibytes.
.BR K ibi-,
.BR M "ebi-, or"
.BR G ibibytes.
#define LOV_DESC_MAGIC 0xB0CCDE5C
#define LOV_DESC_QOS_MAXAGE_DEFAULT 5 /* Seconds */
#define LOV_DESC_MAGIC 0xB0CCDE5C
#define LOV_DESC_QOS_MAXAGE_DEFAULT 5 /* Seconds */
-#define LOV_DESC_STRIPE_SIZE_DEFAULT (1 << LNET_MTU_BITS)
+#define LOV_DESC_STRIPE_SIZE_DEFAULT (4 << LNET_MTU_BITS)
/* LOV settings descriptor (should only contain static info) */
struct lov_desc {
/* LOV settings descriptor (should only contain static info) */
struct lov_desc {
error "$LFS setstripe $td failed"
# create a mirrored file with composite layout mirrors
error "$LFS setstripe $td failed"
# create a mirrored file with composite layout mirrors
- $mirror_cmd -N2 -E 4M -c 2 -p flash -i 1 -o 1,3 -E eof -S 4M \
+ $mirror_cmd -N2 -E 8M -c 2 -p flash -i 1 -o 1,3 -E eof -S 4M \
-N -c 4 -p none \
-N3 -E 512M -S 16M -p archive -E -1 -i -1 -c -1 $tf ||
error "create mirrored file $tf failed"
-N -c 4 -p none \
-N3 -E 512M -S 16M -p archive -E -1 -i -1 -c -1 $tf ||
error "create mirrored file $tf failed"
verify_comp_attr_with_default stripe-size $tf ${ids[$i]}
verify_comp_attr stripe-count $tf ${ids[$i]} 2
verify_comp_attr stripe-index $tf ${ids[$i]} 1
verify_comp_attr_with_default stripe-size $tf ${ids[$i]}
verify_comp_attr stripe-count $tf ${ids[$i]} 2
verify_comp_attr stripe-index $tf ${ids[$i]} 1
- verify_comp_extent $tf ${ids[$i]} 0 4194304
+ verify_comp_extent $tf ${ids[$i]} 0 8388608
done
# verify components ${ids[1]} and ${ids[3]}
done
# verify components ${ids[1]} and ${ids[3]}
verify_comp_attr stripe-size $tf ${ids[$i]} 4194304
verify_comp_attr stripe-count $tf ${ids[$i]} 2
verify_comp_attr pool $tf ${ids[$i]} flash
verify_comp_attr stripe-size $tf ${ids[$i]} 4194304
verify_comp_attr stripe-count $tf ${ids[$i]} 2
verify_comp_attr pool $tf ${ids[$i]} flash
- verify_comp_extent $tf ${ids[$i]} 4194304 EOF
+ verify_comp_extent $tf ${ids[$i]} 8388608 EOF
done
# verify component ${ids[4]}
done
# verify component ${ids[4]}
# extend the mirrored file with composite layout mirrors
$mirror_cmd -N -p archive \
# extend the mirrored file with composite layout mirrors
$mirror_cmd -N -p archive \
- -N2 -E 4M -c 2 -p flash -i 1 -o 1,3 -E eof -S 4M \
+ -N2 -E 8M -c 2 -p flash -i 1 -o 1,3 -E eof -S 4M \
-N -c -1 -p none \
-N3 -E 512M -S 16M -p archive -E -1 -i -1 -c -1 $tf ||
error "extend mirrored file $tf failed"
-N -c -1 -p none \
-N3 -E 512M -S 16M -p archive -E -1 -i -1 -c -1 $tf ||
error "extend mirrored file $tf failed"
verify_comp_attr_with_default stripe-size $tf ${ids[$i]}
verify_comp_attr stripe-count $tf ${ids[$i]} 2
verify_comp_attr stripe-index $tf ${ids[$i]} 1
verify_comp_attr_with_default stripe-size $tf ${ids[$i]}
verify_comp_attr stripe-count $tf ${ids[$i]} 2
verify_comp_attr stripe-index $tf ${ids[$i]} 1
- verify_comp_extent $tf ${ids[$i]} 0 4194304
+ verify_comp_extent $tf ${ids[$i]} 0 8388608
done
# verify components ${ids[4]} and ${ids[6]}
done
# verify components ${ids[4]} and ${ids[6]}
verify_comp_attr stripe-size $tf ${ids[$i]} 4194304
verify_comp_attr stripe-count $tf ${ids[$i]} 2
verify_comp_attr pool $tf ${ids[$i]} flash
verify_comp_attr stripe-size $tf ${ids[$i]} 4194304
verify_comp_attr stripe-count $tf ${ids[$i]} 2
verify_comp_attr pool $tf ${ids[$i]} flash
- verify_comp_extent $tf ${ids[$i]} 4194304 EOF
+ verify_comp_extent $tf ${ids[$i]} 8388608 EOF
done
# verify component ${ids[7]}
done
# verify component ${ids[7]}
else
error "unknown type $io"
fi
else
error "unknown type $io"
fi
- dd $ddarg bs=2M count=1 || error "can't $io $file"
+ dd $ddarg bs=8M count=1 || error "can't $io $file"
cancel_lru_locks osc
# check only specified OSTs got reads
cancel_lru_locks osc
# check only specified OSTs got reads
stack_trap "rm -f $tf"
$LFS setstripe -i0 -c1 $tf || error "can't setstripe"
stack_trap "rm -f $tf"
$LFS setstripe -i0 -c1 $tf || error "can't setstripe"
- dd if=/dev/zero of=$tf bs=2M count=1 || error "can't dd (1)"
+ dd if=/dev/zero of=$tf bs=8M count=1 || error "can't dd (1)"
$LFS mirror extend -N -c1 -o1 $tf || error "can't create mirror"
$LFS mirror extend -N -c2 -o 2,3 $tf || error "can't create mirror"
$LFS mirror resync $tf || error "can't resync"
$LFS mirror extend -N -c1 -o1 $tf || error "can't create mirror"
$LFS mirror extend -N -c2 -o 2,3 $tf || error "can't create mirror"
$LFS mirror resync $tf || error "can't resync"
stack_trap "restore_lustre_params < $p; rm -f $p"
stack_trap "rm -f $tf"
stack_trap "restore_lustre_params < $p; rm -f $p"
stack_trap "rm -f $tf"
- $LFS setstripe -i0 -c1 $tf || error "can't setstripe"
- dd if=/dev/zero of=$tf bs=2M count=1 || error "can't dd (1)"
+ $LFS setstripe -i0 -c1 -S1M $tf || error "can't setstripe"
+ dd if=/dev/zero of=$tf bs=8M count=1 || error "can't dd (1)"
$LFS mirror extend -N -c1 -o1 $tf || error "can't create mirror"
$LFS mirror extend -N -c2 -o 2,3 $tf || error "can't create mirror"
$LFS mirror resync $tf || error "can't resync"
$LFS mirror extend -N -c1 -o1 $tf || error "can't create mirror"
$LFS mirror extend -N -c2 -o 2,3 $tf || error "can't create mirror"
$LFS mirror resync $tf || error "can't resync"
lctl get_param osc.*.*grant*
stack_trap "lfs df $DIR; lfs df -i $DIR; lctl get_param osc.*.*grant*"
lctl get_param osc.*.*grant*
stack_trap "lfs df $DIR; lfs df -i $DIR; lctl get_param osc.*.*grant*"
- $LFS setstripe -N -E 2M -o 0,1 -E -1 -o 2 -N -E 2M -o 1,2 -E -1 -o 0 \
+ $LFS setstripe -N -E 2M -S1M -o 0,1 -E -1 -o 2 \
+ -N -E 2M -S1M -o 1,2 -E -1 -o 0 \
-N -E 3M -o 2,0 -E -1 -o 1 $DIR/$tdir/f0 ||
error "(0) Fail to create mirror file $DIR/$tdir/f0"
-N -E 3M -o 2,0 -E -1 -o 1 $DIR/$tdir/f0 ||
error "(0) Fail to create mirror file $DIR/$tdir/f0"
- $LFS setstripe -N -E 2M -o 0,1 -E -1 -o 2 -N -E 2M -o 1,2 -E -1 -o 0 \
+ $LFS setstripe -N -E 2M -S1M -o 0,1 -E -1 -o 2 \
+ -N -E 2M -S1M -o 1,2 -E -1 -o 0 \
-N -E 3M -o 2,0 -E -1 -o 1 $DIR/$tdir/f1 ||
error "(1) Fail to create mirror file $DIR/$tdir/f1"
-N -E 3M -o 2,0 -E -1 -o 1 $DIR/$tdir/f1 ||
error "(1) Fail to create mirror file $DIR/$tdir/f1"
- $LFS setstripe -N -E 2M -o 0,1 -E -1 -o 2 -N -E 2M -o 1,2 -E -1 -o 0 \
+ $LFS setstripe -N -E 2M -S1M -o 0,1 -E -1 -o 2 \
+ -N -E 2M -S1M -o 1,2 -E -1 -o 0 \
-N -E 3M -o 2,0 -E -1 -o 1 $DIR/$tdir/f2 ||
error "(2) Fail to create mirror file $DIR/$tdir/f2"
-N -E 3M -o 2,0 -E -1 -o 1 $DIR/$tdir/f2 ||
error "(2) Fail to create mirror file $DIR/$tdir/f2"
- $LFS setstripe -N -E 2M -o 0,1 -E -1 -o 2 -N -E 2M -o 1,2 -E -1 -o 0 \
+ $LFS setstripe -N -E 2M -S1M -o 0,1 -E -1 -o 2 \
+ -N -E 2M -S1M -o 1,2 -E -1 -o 0 \
-N -E 3M -o 2,0 -E -1 -o 1 $DIR/$tdir/f0 ||
error "(0) Fail to create mirror file $DIR/$tdir/f0"
-N -E 3M -o 2,0 -E -1 -o 1 $DIR/$tdir/f0 ||
error "(0) Fail to create mirror file $DIR/$tdir/f0"
- $LFS setstripe -N -E 2M -o 0,1 -E -1 -o 2 -N -E 2M -o 1,2 -E -1 -o 0 \
- $DIR/$tdir/f0 ||
+ $LFS setstripe -N -E 2M -S1M -o 0,1 -E -1 -o 2 \
+ -N -E 2M -S1M -o 1,2 -E -1 -o 0 $DIR/$tdir/f0 ||
error "(0) Fail to create mirror file $DIR/$tdir/f0"
local fid=$($LFS path2fid $DIR/$tdir/f0)
error "(0) Fail to create mirror file $DIR/$tdir/f0"
local fid=$($LFS path2fid $DIR/$tdir/f0)
local count=$(do_facet mds1 $LCTL get_param -n \
mdd.$(facet_svc mds1).lfsck_layout |
awk '/^repaired_orphan/ { print $2 }')
local count=$(do_facet mds1 $LCTL get_param -n \
mdd.$(facet_svc mds1).lfsck_layout |
awk '/^repaired_orphan/ { print $2 }')
- [ $count -eq 6 ] || error "(7) Expect 9 fixed on mds1, but got: $count"
+ [ $count -eq 6 ] || error "(7) Expect 6 fixed on mds1, but got: $count"
local name=$MOUNT/.lustre/lost+found/MDT0000/${fid}-R-0
count=$($LFS getstripe --mirror-count $name)
local name=$MOUNT/.lustre/lost+found/MDT0000/${fid}-R-0
count=$($LFS getstripe --mirror-count $name)
# Seek & write in to last component so all objects are allocated
dd if=/dev/zero of=$comp_file bs=1k count=1 seek=20000
# Seek & write in to last component so all objects are allocated
dd if=/dev/zero of=$comp_file bs=1k count=1 seek=20000
+ local ssize=$($LFS getstripe -S -I1 $DIR/$tdir/$tfile)
local count=$($LFS getstripe -c -I1 $DIR/$tdir/$tfile)
local count=$($LFS getstripe -c -I1 $DIR/$tdir/$tfile)
+ count=$((count * ssize / 1048576))
[ $count -eq 10 ] || error "comp1 stripe count $count, should be 10"
[ $count -eq 10 ] || error "comp1 stripe count $count, should be 10"
+ ssize=$($LFS getstripe -S -I2 $DIR/$tdir/$tfile)
count=$($LFS getstripe -c -I2 $DIR/$tdir/$tfile)
count=$($LFS getstripe -c -I2 $DIR/$tdir/$tfile)
+ count=$((count * ssize / 1048576))
[ $count -eq 100 ] || error "comp2 stripe count $count, should be 100"
[ $count -eq 100 ] || error "comp2 stripe count $count, should be 100"
+ ssize=$($LFS getstripe -S -I3 $DIR/$tdir/$tfile)
count=$($LFS getstripe -c -I3 $DIR/$tdir/$tfile)
[ $count -eq $LOV_MAX_STRIPE_COUNT ] ||
count=$($LFS getstripe -c -I3 $DIR/$tdir/$tfile)
[ $count -eq $LOV_MAX_STRIPE_COUNT ] ||
- error "comp4 stripe count $count != $LOV_MAX_STRIPE_COUNT"
+ echo "comp4 stripe count $count != $LOV_MAX_STRIPE_COUNT"
small_write $comp_file $rw_len || error "Verify RW failed"
small_write $comp_file $rw_len || error "Verify RW failed"
local ost_idx1=$($LFS getstripe -I1 -i $comp_file)
local ost_idx2=$($LFS getstripe -I2 -i $comp_file)
local ost_idx1=$($LFS getstripe -I1 -i $comp_file)
local ost_idx2=$($LFS getstripe -I2 -i $comp_file)
+ local ssize=$($LFS getstripe -S -I1 $comp_file)
[ $ost_idx1 -eq $ost_idx2 ] && error "$ost_idx1 == $ost_idx2"
[ $ost_idx2 -ne "-1" ] && error "second component init $ost_idx2"
[ $ost_idx1 -eq $ost_idx2 ] && error "$ost_idx1 == $ost_idx2"
[ $ost_idx2 -ne "-1" ] && error "second component init $ost_idx2"
[ $found -eq 1 ] ||
error "component not found by negation of wrong -ext size"
[ $found -eq 1 ] ||
error "component not found by negation of wrong -ext size"
- found=$($LFS find -S +1M $comp_file | wc -l)
+ found=$($LFS find -S +$ssize $comp_file | wc -l)
[ $found -eq 0 ] || error "component found by wrong +stripe size"
found=$($LFS find -c 1 $comp_file | wc -l)
[ $found -eq 0 ] || error "component found by wrong +stripe size"
found=$($LFS find -c 1 $comp_file | wc -l)
dd if=/dev/zero of=$comp_file bs=1M count=1 seek=14 ||
error "dd to extend/remove failed"
dd if=/dev/zero of=$comp_file bs=1M count=1 seek=14 ||
error "dd to extend/remove failed"
- $LFS getstripe $comp_file
-
found=$($LFS find --comp-start 10M -E 10M $flg_opts $comp_file | wc -l)
[ $found -eq 0 ] || error "Write: zero length component still present"
flg_opts="--comp-flags init"
found=$($LFS find --comp-start 10M -E 10M $flg_opts $comp_file | wc -l)
[ $found -eq 0 ] || error "Write: zero length component still present"
flg_opts="--comp-flags init"
- found=$($LFS find --comp-start 10M -E 138M $flg_opts $comp_file | wc -l)
+ $LFS find --comp-start 10M -E +137M $flg_opts $comp_file
+ found=$($LFS find --comp-start 10M -E +137M $flg_opts $comp_file |wc -l)
[ $found -eq 1 ] || error "Write: second component not found"
flg_opts="--comp-flags extension"
[ $found -eq 1 ] || error "Write: second component not found"
flg_opts="--comp-flags extension"
- found=$($LFS find --comp-start 138M -E EOF $flg_opts $comp_file | wc -l)
+ found=$($LFS find --comp-start +137M -E EOF $flg_opts $comp_file |wc -l)
[ $found -eq 1 ] || error "Write: third component not found"
fail $SINGLEMDS
[ $found -eq 1 ] || error "Write: third component not found"
fail $SINGLEMDS
[ $found -eq 0 ] || error "Failover: 0-length component still present"
flg_opts="--comp-flags init"
[ $found -eq 0 ] || error "Failover: 0-length component still present"
flg_opts="--comp-flags init"
- found=$($LFS find --comp-start 10M -E 138M $flg_opts $comp_file | wc -l)
+ found=$($LFS find --comp-start 10M -E +137M $flg_opts $comp_file |wc -l)
[ $found -eq 1 ] || error "Failover: second component not found"
flg_opts="--comp-flags extension"
[ $found -eq 1 ] || error "Failover: second component not found"
flg_opts="--comp-flags extension"
- found=$($LFS find --comp-start 138M -E EOF $flg_opts $comp_file | wc -l)
+ found=$($LFS find --comp-start +137M -E EOF $flg_opts $comp_file |wc -l)
[ $found -eq 1 ] || error "Failover: third component not found"
sel_layout_sanity $comp_file 3
[ $found -eq 1 ] || error "Failover: third component not found"
sel_layout_sanity $comp_file 3
- $LFS setstripe -E 1m -c1 -o0 -E eof -c2 -o1,2 $file ||
+ $LFS setstripe -E 1m -c1 -o0 -E eof -c2 -S 1M -o1,2 $file ||
error "setstripe on $file"
dd if=/dev/zero of=$file bs=1M count=3 || error "dd failed for $file"
error "setstripe on $file"
dd if=/dev/zero of=$file bs=1M count=3 || error "dd failed for $file"
local file=$DIR/$tfile
local nr=$((OSTCOUNT * 100))
local file=$DIR/$tfile
local nr=$((OSTCOUNT * 100))
- $LFS setstripe -C $nr $file || error "failed to setstripe -C $nr $file"
+ $LFS setstripe -C $nr -S1M $file ||
+ error "failed to setstripe -C $nr $file"
stack_trap "rm -f $file"
dd if=/dev/zero of=$file count=$nr bs=1M
stack_trap "rm -f $file"
dd if=/dev/zero of=$file count=$nr bs=1M