From: Andreas Dilger Date: Wed, 17 May 2017 06:42:23 +0000 (-0700) Subject: LU-9090 ofd: increase default OST BRW size to 4MB X-Git-Tag: 2.9.59~75 X-Git-Url: https://git.whamcloud.com/?p=fs%2Flustre-release.git;a=commitdiff_plain;h=8c9c1f59d99c924e16317495269b8270cb0be921 LU-9090 ofd: increase default OST BRW size to 4MB Increase the default maximum bulk RPC size reported by OSTs to the clients to 4MB. This has shown to improve performance for tested workloads under a variety of conditions, and is already the defacto standard for new installations. Clients will still negotiate their preferred and supported BRW size with the OSTs at connection time. Signed-off-by: Andreas Dilger Change-Id: Id213e598bde14ed53b78ec230c7ae9aa5f3ebbe5 Reviewed-on: https://review.whamcloud.com/25336 Reviewed-by: Niu Yawei Reviewed-by: Jinshan Xiong Tested-by: Jenkins Tested-by: Maloo Reviewed-by: Oleg Drokin --- diff --git a/lustre/include/lustre_net.h b/lustre/include/lustre_net.h index 926253f..14da6e2 100644 --- a/lustre/include/lustre_net.h +++ b/lustre/include/lustre_net.h @@ -104,6 +104,7 @@ #define MD_MAX_BRW_SIZE (1U << LNET_MTU_BITS) #define MD_MAX_BRW_PAGES (MD_MAX_BRW_SIZE >> PAGE_SHIFT) #define DT_MAX_BRW_SIZE PTLRPC_MAX_BRW_SIZE +#define DT_DEF_BRW_SIZE (4 * ONE_MB_BRW_SIZE) #define DT_MAX_BRW_PAGES (DT_MAX_BRW_SIZE >> PAGE_SHIFT) #define OFD_MAX_BRW_SIZE (1U << LNET_MTU_BITS) diff --git a/lustre/ofd/ofd_dev.c b/lustre/ofd/ofd_dev.c index 31c17f9..5b8b56f 100644 --- a/lustre/ofd/ofd_dev.c +++ b/lustre/ofd/ofd_dev.c @@ -3018,10 +3018,10 @@ static int ofd_init0(const struct lu_env *env, struct ofd_device *m, } tgd->tgd_blockbits = fls(osfs->os_bsize) - 1; - if (ONE_MB_BRW_SIZE < (1U << tgd->tgd_blockbits)) + if (DT_DEF_BRW_SIZE < (1U << tgd->tgd_blockbits)) m->ofd_brw_size = 1U << tgd->tgd_blockbits; else - m->ofd_brw_size = ONE_MB_BRW_SIZE; + m->ofd_brw_size = DT_DEF_BRW_SIZE; m->ofd_cksum_types_supported = cksum_types_supported_server(); m->ofd_precreate_batch = OFD_PRECREATE_BATCH_DEFAULT; diff --git a/lustre/tests/sanity-quota.sh b/lustre/tests/sanity-quota.sh index 56f4a92..8723809 100755 --- a/lustre/tests/sanity-quota.sh +++ b/lustre/tests/sanity-quota.sh @@ -498,6 +498,7 @@ test_1() { $RUNAS $DD of=$TESTFILE count=$((LIMIT/2)) seek=$((LIMIT/2)) || true # flush cache, ensure noquota flag is set on client cancel_lru_locks osc + sync; sync_all_data || true $RUNAS $DD of=$TESTFILE count=1 seek=$LIMIT && quota_error u $TSTUSR "user write success, but expect EDQUOT" @@ -530,6 +531,7 @@ test_1() { # this time maybe cache write, ignore it's failure $RUNAS $DD of=$TESTFILE count=$((LIMIT/2)) seek=$((LIMIT/2)) || true cancel_lru_locks osc + sync; sync_all_data || true $RUNAS $DD of=$TESTFILE count=10 seek=$LIMIT && quota_error g $TSTUSR "Group write success, but expect EDQUOT" rm -f $TESTFILE @@ -569,6 +571,7 @@ test_1() { # this time maybe cache write, ignore it's failure $RUNAS $DD of=$TESTFILE count=$((LIMIT/2)) seek=$((LIMIT/2)) || true cancel_lru_locks osc + sync; sync_all_data || true $RUNAS $DD of=$TESTFILE count=10 seek=$LIMIT && quota_error p \ $TSTPRJID "project write success, but expect edquot" diff --git a/lustre/tests/sanity-sec.sh b/lustre/tests/sanity-sec.sh index a8c814d..3a710b0 100755 --- a/lustre/tests/sanity-sec.sh +++ b/lustre/tests/sanity-sec.sh @@ -1090,8 +1090,8 @@ do_fops_quota_test() { local qused_high=$((qused_orig + quota_fuzz)) local qused_low=$((qused_orig - quota_fuzz)) local testfile=$DIR/$tdir/$tfile - $run_u dd if=/dev/zero of=$testfile bs=1M count=1 >& /dev/null || - error "unable to write quota test file" + $run_u dd if=/dev/zero of=$testfile oflag=sync bs=1M count=1 \ + >& /dev/null || error "unable to write quota test file" sync; sync_all_data || true local qused_new=$(nodemap_check_quota "$run_u") diff --git a/lustre/tests/sanity.sh b/lustre/tests/sanity.sh index 7d81fef..a37944f 100755 --- a/lustre/tests/sanity.sh +++ b/lustre/tests/sanity.sh @@ -6897,8 +6897,8 @@ test_101g_brw_size_test() { # calculate number of full-sized read and write RPCs rpcs=($($LCTL get_param -n 'osc.*.rpc_stats' | sed -n '/pages per rpc/,/^$/p' | - awk '/'$pages':/ { reads += $2; writes += $5 };' \ - 'END { print reads,writes }')) + awk '/'$pages':/ { reads += $2; writes += $6 }; \ + END { print reads,writes }')) [ ${rpcs[0]} -ne $count ] && error "${rpcs[0]} != $count read RPCs" && return 5 [ ${rpcs[1]} -ne $count ] && error "${rpcs[1]} != $count write RPCs" && @@ -6920,7 +6920,7 @@ test_101g() { if [ $(lustre_version_code ost1) -ge $(version_code 2.8.52) ]; then [ $(lustre_version_code ost1) -ge $(version_code 2.9.52) ] && suffix="M" - if [[ $orig_mb < 16 ]]; then + if [[ $orig_mb -lt 16 ]]; then save_lustre_params $osts "$brw_size" > $p do_nodes $list $LCTL set_param -n $brw_size=16$suffix || error "set 16MB RPC size failed" @@ -6936,7 +6936,7 @@ test_101g() { test_101g_brw_size_test 4 || error "4MB RPC test failed" - if [[ $orig_mb < 16 ]]; then + if [[ $orig_mb -lt 16 ]]; then restore_lustre_params < $p remount_client $MOUNT || error "remount_client restore failed" fi @@ -13847,8 +13847,12 @@ test_231a() # is the same across all OSCs local max_pages=$($LCTL get_param -n osc.*.max_pages_per_rpc | head -n1) local bulk_size=$((max_pages * 4096)) + local brw_size=$(do_facet ost1 $LCTL get_param -n obdfilter.*.brw_size | + head -n 1) mkdir -p $DIR/$tdir + $LFS setstripe -S ${brw_size}M $DIR/$tdir || + error "failed to set stripe with -S ${brw_size}M option" # clear the OSC stats $LCTL set_param osc.*.stats=0 &>/dev/null