From 5868ad233e5748d561a22ffdb31fc6c76af174e2 Mon Sep 17 00:00:00 2001 From: Li Dongyang Date: Tue, 3 Dec 2024 15:46:45 +1100 Subject: [PATCH] LU-18504 llite: fix whole-file readahead RPC limit We try to increase whole-file readahead to match RPC size, by using the connect data from sbi->ll_dt_exp. But that's not the right location, the lov layer could not touch/change the ocd_brw_size in connect data, it stays 64M as the largest possible RPC size, which could be too large for whole-file readahead. Introduce a new key KEY_MAX_PAGES_PER_RPC in lov_get_info, to iterate every osc obd to get max pages per rpc replied from OST. After obd_connect in client_common_fill_super, there are chances that the connect rpc reply hasn't been processed so the optimal ocd_brw_size replied from server hasn't been updated yet, delay checking max pages per rpc to give us a better chance to get the replies from OST. Bump up SBI_DEFAULT_READAHEAD_WHOLE_MAX to 4M, which is the same to DT_DEF_BRW_SIZE from osd for now. Fixes: 627d0133d9 ("LU-7990 llite: increase whole-file readahead to RPC size") Signed-off-by: Li Dongyang Change-Id: I172bf81ac803971d5f0d50fe00ae4fdfb0d26485 Reviewed-on: https://review.whamcloud.com/c/fs/lustre-release/+/57249 Tested-by: jenkins Tested-by: Maloo Reviewed-by: Andreas Dilger Reviewed-by: Qian Yingjin Reviewed-by: Oleg Drokin --- lustre/include/obd.h | 1 + lustre/llite/llite_internal.h | 2 +- lustre/llite/llite_lib.c | 31 ++++++++++++++++++++----------- lustre/lov/lov_obd.c | 28 ++++++++++++++++++++++++++++ 4 files changed, 50 insertions(+), 12 deletions(-) diff --git a/lustre/include/obd.h b/lustre/include/obd.h index 3ea36fa..558e8df 100644 --- a/lustre/include/obd.h +++ b/lustre/include/obd.h @@ -856,6 +856,7 @@ static inline bool obd_mdt_recovery_abort(struct obd_device *obd) #define KEY_CACHE_LRU_SHRINK "cache_lru_shrink" #define KEY_OSP_CONNECTED "osp_connected" #define KEY_FID2IDX "fid2idx" +#define KEY_MAX_PAGES_PER_RPC "max_pages_per_rpc" #define KEY_UNEVICT_CACHE_SHRINK "unevict_cache_shrink" diff --git a/lustre/llite/llite_internal.h b/lustre/llite/llite_internal.h index eae98ef..32bae6d 100644 --- a/lustre/llite/llite_internal.h +++ b/lustre/llite/llite_internal.h @@ -736,7 +736,7 @@ static inline void ll_inode_unlock(struct inode *inode) #define SBI_DEFAULT_READ_AHEAD_PER_FILE_MAX MiB_TO_PAGES(256UL) /* default read-ahead full files smaller than limit on the second read */ -#define SBI_DEFAULT_READ_AHEAD_WHOLE_MAX MiB_TO_PAGES(2UL) +#define SBI_DEFAULT_READ_AHEAD_WHOLE_MAX MiB_TO_PAGES(4UL) /* default range pages */ #define SBI_DEFAULT_RA_RANGE_PAGES MiB_TO_PAGES(1ULL) diff --git a/lustre/llite/llite_lib.c b/lustre/llite/llite_lib.c index 76820db..969700c 100644 --- a/lustre/llite/llite_lib.c +++ b/lustre/llite/llite_lib.c @@ -678,17 +678,6 @@ retry_connect: sbi->ll_dt_exp->exp_connect_data = *data; - /* Don't change value if it was specified in the config log */ - if (sbi->ll_ra_info.ra_max_read_ahead_whole_pages == -1) { - sbi->ll_ra_info.ra_max_read_ahead_whole_pages = - max_t(unsigned long, SBI_DEFAULT_READ_AHEAD_WHOLE_MAX, - (data->ocd_brw_size >> PAGE_SHIFT)); - if (sbi->ll_ra_info.ra_max_read_ahead_whole_pages > - sbi->ll_ra_info.ra_max_pages_per_file) - sbi->ll_ra_info.ra_max_read_ahead_whole_pages = - sbi->ll_ra_info.ra_max_pages_per_file; - } - err = client_fid_init(sbi->ll_dt_exp->exp_obd, sbi->ll_dt_exp, LUSTRE_SEQ_METADATA); if (err) { @@ -833,6 +822,26 @@ retry_connect: OBD_FREE_PTR(data); OBD_FREE_PTR(osfs); + /* Don't change value if it was specified in the config log */ + if (sbi->ll_ra_info.ra_max_read_ahead_whole_pages == -1) { + u32 max_pages_per_rpc; + + size = sizeof(max_pages_per_rpc); + err = obd_get_info(NULL, sbi->ll_dt_exp, + sizeof(KEY_MAX_PAGES_PER_RPC), + KEY_MAX_PAGES_PER_RPC, &size, + &max_pages_per_rpc); + if (err) + max_pages_per_rpc = 0; + sbi->ll_ra_info.ra_max_read_ahead_whole_pages = + max_t(u32, SBI_DEFAULT_READ_AHEAD_WHOLE_MAX, + max_pages_per_rpc); + if (sbi->ll_ra_info.ra_max_read_ahead_whole_pages > + sbi->ll_ra_info.ra_max_pages_per_file) + sbi->ll_ra_info.ra_max_read_ahead_whole_pages = + sbi->ll_ra_info.ra_max_pages_per_file; + } + if (sbi->ll_dt_obd) { err = sysfs_create_link(&sbi->ll_kset.kobj, &sbi->ll_dt_obd->obd_kset.kobj, diff --git a/lustre/lov/lov_obd.c b/lustre/lov/lov_obd.c index 5842c49..f4c8584 100644 --- a/lustre/lov/lov_obd.c +++ b/lustre/lov/lov_obd.c @@ -1134,6 +1134,34 @@ static int lov_get_info(const struct lu_env *env, struct obd_export *exp, *((u32 *)val) = lov_mds_md_size(def_stripe_count, LOV_MAGIC_V3); } else if (KEY_IS(KEY_TGT_COUNT)) { *((int *)val) = lov->desc.ld_tgt_count; + } else if (KEY_IS(KEY_MAX_PAGES_PER_RPC)) { + struct lov_tgt_desc *tgt; + struct client_obd *cli; + struct obd_import *imp; + u32 result = 0; + u32 i; + + for (i = 0; i < lov->desc.ld_tgt_count; i++) { + tgt = lov->lov_tgts[i]; + + /* OST was disconnected */ + if (tgt == NULL || tgt->ltd_exp == NULL) + continue; + + cli = &tgt->ltd_obd->u.cli; + imp = cli->cl_import; + + if (imp == NULL || imp->imp_state != LUSTRE_IMP_FULL) + continue; + + if (result == 0) + result = cli->cl_max_pages_per_rpc; + else + result = min_t(u32, cli->cl_max_pages_per_rpc, + result); + } + + *((u32 *)val) = result; } else { rc = -EINVAL; } -- 1.8.3.1