LU-9090 ofd: increase default OST BRW size to 4MB

Increase the default maximum bulk RPC size reported by OSTs to the
clients to 4MB.  This has shown to improve performance for tested
workloads under a variety of conditions, and is already the defacto
standard for new installations.

Clients will still negotiate their preferred and supported BRW size
with the OSTs at connection time.

Signed-off-by: Andreas Dilger <andreas.dilger@intel.com>
Change-Id: Id213e598bde14ed53b78ec230c7ae9aa5f3ebbe5
Reviewed-on: https://review.whamcloud.com/25336
Reviewed-by: Niu Yawei <yawei.niu@intel.com>
Reviewed-by: Jinshan Xiong <jinshan.xiong@intel.com>
Tested-by: Jenkins
Tested-by: Maloo <hpdd-maloo@intel.com>
Reviewed-by: Oleg Drokin <oleg.drokin@intel.com>

diff --git a/lustre/include/lustre_net.h b/lustre/include/lustre_net.h
index 926253f..14da6e2 100644 (file)
--- a/lustre/include/lustre_net.h
+++ b/lustre/include/lustre_net.h
@@ -104,6 +104,7 @@
 #define MD_MAX_BRW_SIZE                (1U << LNET_MTU_BITS)
 #define MD_MAX_BRW_PAGES       (MD_MAX_BRW_SIZE >> PAGE_SHIFT)
 #define DT_MAX_BRW_SIZE                PTLRPC_MAX_BRW_SIZE
+#define DT_DEF_BRW_SIZE                (4 * ONE_MB_BRW_SIZE)
 #define DT_MAX_BRW_PAGES       (DT_MAX_BRW_SIZE >> PAGE_SHIFT)
 #define OFD_MAX_BRW_SIZE       (1U << LNET_MTU_BITS)
 

diff --git a/lustre/ofd/ofd_dev.c b/lustre/ofd/ofd_dev.c
index 31c17f9..5b8b56f 100644 (file)
--- a/lustre/ofd/ofd_dev.c
+++ b/lustre/ofd/ofd_dev.c
@@ -3018,10 +3018,10 @@ static int ofd_init0(const struct lu_env *env, struct ofd_device *m,
        }
        tgd->tgd_blockbits = fls(osfs->os_bsize) - 1;
 
-       if (ONE_MB_BRW_SIZE < (1U << tgd->tgd_blockbits))
+       if (DT_DEF_BRW_SIZE < (1U << tgd->tgd_blockbits))
                m->ofd_brw_size = 1U << tgd->tgd_blockbits;
        else
-               m->ofd_brw_size = ONE_MB_BRW_SIZE;
+               m->ofd_brw_size = DT_DEF_BRW_SIZE;
 
        m->ofd_cksum_types_supported = cksum_types_supported_server();
        m->ofd_precreate_batch = OFD_PRECREATE_BATCH_DEFAULT;

diff --git a/lustre/tests/sanity-quota.sh b/lustre/tests/sanity-quota.sh
index 56f4a92..8723809 100755 (executable)
--- a/lustre/tests/sanity-quota.sh
+++ b/lustre/tests/sanity-quota.sh
@@ -498,6 +498,7 @@ test_1() {
        $RUNAS $DD of=$TESTFILE count=$((LIMIT/2)) seek=$((LIMIT/2)) || true
        # flush cache, ensure noquota flag is set on client
        cancel_lru_locks osc
+       sync; sync_all_data || true
        $RUNAS $DD of=$TESTFILE count=1 seek=$LIMIT &&
                quota_error u $TSTUSR "user write success, but expect EDQUOT"
 
@@ -530,6 +531,7 @@ test_1() {
        # this time maybe cache write, ignore it's failure
        $RUNAS $DD of=$TESTFILE count=$((LIMIT/2)) seek=$((LIMIT/2)) || true
        cancel_lru_locks osc
+       sync; sync_all_data || true
        $RUNAS $DD of=$TESTFILE count=10 seek=$LIMIT &&
                quota_error g $TSTUSR "Group write success, but expect EDQUOT"
        rm -f $TESTFILE
@@ -569,6 +571,7 @@ test_1() {
        # this time maybe cache write, ignore it's failure
        $RUNAS $DD of=$TESTFILE count=$((LIMIT/2)) seek=$((LIMIT/2)) || true
        cancel_lru_locks osc
+       sync; sync_all_data || true
        $RUNAS $DD of=$TESTFILE count=10 seek=$LIMIT && quota_error p \
                $TSTPRJID "project write success, but expect edquot"
 

diff --git a/lustre/tests/sanity-sec.sh b/lustre/tests/sanity-sec.sh
index a8c814d..3a710b0 100755 (executable)
--- a/lustre/tests/sanity-sec.sh
+++ b/lustre/tests/sanity-sec.sh
@@ -1090,8 +1090,8 @@ do_fops_quota_test() {
        local qused_high=$((qused_orig + quota_fuzz))
        local qused_low=$((qused_orig - quota_fuzz))
        local testfile=$DIR/$tdir/$tfile
-       $run_u dd if=/dev/zero of=$testfile bs=1M count=1 >& /dev/null ||
-               error "unable to write quota test file"
+       $run_u dd if=/dev/zero of=$testfile oflag=sync bs=1M count=1 \
+               >& /dev/null || error "unable to write quota test file"
        sync; sync_all_data || true
 
        local qused_new=$(nodemap_check_quota "$run_u")

diff --git a/lustre/tests/sanity.sh b/lustre/tests/sanity.sh
index 7d81fef..a37944f 100755 (executable)
--- a/lustre/tests/sanity.sh
+++ b/lustre/tests/sanity.sh
@@ -6897,8 +6897,8 @@ test_101g_brw_size_test() {
        # calculate number of full-sized read and write RPCs
        rpcs=($($LCTL get_param -n 'osc.*.rpc_stats' |
                sed -n '/pages per rpc/,/^$/p' |
-               awk '/'$pages':/ { reads += $2; writes += $5 };' \
-               'END { print reads,writes }'))
+               awk '/'$pages':/ { reads += $2; writes += $6 }; \
+               END { print reads,writes }'))
        [ ${rpcs[0]} -ne $count ] && error "${rpcs[0]} != $count read RPCs" &&
                return 5
        [ ${rpcs[1]} -ne $count ] && error "${rpcs[1]} != $count write RPCs" &&
@@ -6920,7 +6920,7 @@ test_101g() {
        if [ $(lustre_version_code ost1) -ge $(version_code 2.8.52) ]; then
                [ $(lustre_version_code ost1) -ge $(version_code 2.9.52) ] &&
                        suffix="M"
-               if [[ $orig_mb < 16 ]]; then
+               if [[ $orig_mb -lt 16 ]]; then
                        save_lustre_params $osts "$brw_size" > $p
                        do_nodes $list $LCTL set_param -n $brw_size=16$suffix ||
                                error "set 16MB RPC size failed"
@@ -6936,7 +6936,7 @@ test_101g() {
 
        test_101g_brw_size_test 4 || error "4MB RPC test failed"
 
-       if [[ $orig_mb < 16 ]]; then
+       if [[ $orig_mb -lt 16 ]]; then
                restore_lustre_params < $p
                remount_client $MOUNT || error "remount_client restore failed"
        fi
@@ -13847,8 +13847,12 @@ test_231a()
        # is the same across all OSCs
        local max_pages=$($LCTL get_param -n osc.*.max_pages_per_rpc | head -n1)
        local bulk_size=$((max_pages * 4096))
+       local brw_size=$(do_facet ost1 $LCTL get_param -n obdfilter.*.brw_size |
+                                      head -n 1)
 
        mkdir -p $DIR/$tdir
+       $LFS setstripe -S ${brw_size}M $DIR/$tdir ||
+               error "failed to set stripe with -S ${brw_size}M option"
 
        # clear the OSC stats
        $LCTL set_param osc.*.stats=0 &>/dev/null