From b859a51e5fa580797dd833bb8e5ec7d6e41411af Mon Sep 17 00:00:00 2001 From: Liang Zhen Date: Thu, 7 Mar 2013 15:24:39 +0800 Subject: [PATCH 1/1] LU-2756 ost: buffer utilization rate of OST rqbd This patch covered a couple of things: - different request buffer size for different OST services only requests on OST_IO_PORTAL need large buffers - add extra 46K to OST_IO_BUFSIZE (buffer size is increased to 64K) size of OST_IO_MAXREQSIZE is about 18K, if OST_IO_BUFSIZE is only a little bigger than OST_IO_MAXREQSIZE, then each rqbd can only fit in 2-3 request even there are 18K bytes left in this rqbd, memory utilization rate is low. Extra bytes can make buffer utilization rate increase to 72%. Also, it can avoid version compatibility issue in the future if we want to increase request size again. Signed-off-by: Liang Zhen Change-Id: Iaf8b45520b000dd5219a4af2ec3f5b2ea7f53134 Reviewed-on: http://review.whamcloud.com/5626 Tested-by: Hudson Reviewed-by: Fan Yong Tested-by: Maloo Reviewed-by: Andreas Dilger --- lustre/include/lustre_net.h | 36 ++++++++++++++++++++++++++---------- lustre/ost/ost_handler.c | 6 +++--- 2 files changed, 29 insertions(+), 13 deletions(-) diff --git a/lustre/include/lustre_net.h b/lustre/include/lustre_net.h index c500fb9..cd6225a 100644 --- a/lustre/include/lustre_net.h +++ b/lustre/include/lustre_net.h @@ -456,22 +456,38 @@ #define OSS_CR_NTHRS_MAX 64 /** - * OST_MAXREQSIZE ~= - * lustre_msg + obdo + obd_ioobj + DT_MAX_BRW_PAGES * niobuf_remote + * OST_IO_MAXREQSIZE ~= + * lustre_msg + ptlrpc_body + obdo + obd_ioobj + + * DT_MAX_BRW_PAGES * niobuf_remote * * - single object with 16 pages is 512 bytes - * - OST_MAXREQSIZE must be at least 1 page of cookies plus some spillover + * - OST_IO_MAXREQSIZE must be at least 1 page of cookies plus some spillover * - Must be a multiple of 1024 + * - actual size is about 18K */ -#define _OST_MAXREQSIZE_SUM (sizeof(struct lustre_msg) + sizeof(struct obdo) + \ - sizeof(struct obd_ioobj) + DT_MAX_BRW_PAGES * \ - sizeof(struct niobuf_remote)) -#define OST_MAXREQSIZE (((_OST_MAXREQSIZE_SUM - 1) | (1024 - 1)) + 1) +#define _OST_MAXREQSIZE_SUM (sizeof(struct lustre_msg) + \ + sizeof(struct ptlrpc_body) + \ + sizeof(struct obdo) + \ + sizeof(struct obd_ioobj) + \ + sizeof(struct niobuf_remote) * DT_MAX_BRW_PAGES) +/** + * FIEMAP request can be 4K+ for now + */ +#define OST_MAXREQSIZE (5 * 1024) +#define OST_IO_MAXREQSIZE max_t(int, OST_MAXREQSIZE, \ + (((_OST_MAXREQSIZE_SUM - 1) | (1024 - 1)) + 1)) -#define OST_MAXREPSIZE (9 * 1024) +#define OST_MAXREPSIZE (9 * 1024) +#define OST_IO_MAXREPSIZE OST_MAXREPSIZE -#define OST_NBUFS 64 -#define OST_BUFSIZE (OST_MAXREQSIZE + 1024) +#define OST_NBUFS 64 +/** OST_BUFSIZE = max_reqsize + max sptlrpc payload size */ +#define OST_BUFSIZE max_t(int, OST_MAXREQSIZE + 1024, 16 * 1024) +/** + * OST_IO_MAXREQSIZE is 18K, giving extra 46K can increase buffer utilization + * rate of request buffer, please check comment of MDS_LOV_BUFSIZE for details. + */ +#define OST_IO_BUFSIZE max_t(int, OST_IO_MAXREQSIZE + 1024, 64 * 1024) /* Macro to hide a typecast. */ #define ptlrpc_req_async_args(req) ((void *)&req->rq_async_args) diff --git a/lustre/ost/ost_handler.c b/lustre/ost/ost_handler.c index 562210e..873c23e 100644 --- a/lustre/ost/ost_handler.c +++ b/lustre/ost/ost_handler.c @@ -2611,9 +2611,9 @@ static int ost_setup(struct obd_device *obd, struct lustre_cfg* lcfg) .psc_watchdog_factor = OSS_SERVICE_WATCHDOG_FACTOR, .psc_buf = { .bc_nbufs = OST_NBUFS, - .bc_buf_size = OST_BUFSIZE, - .bc_req_max_size = OST_MAXREQSIZE, - .bc_rep_max_size = OST_MAXREPSIZE, + .bc_buf_size = OST_IO_BUFSIZE, + .bc_req_max_size = OST_IO_MAXREQSIZE, + .bc_rep_max_size = OST_IO_MAXREPSIZE, .bc_req_portal = OST_IO_PORTAL, .bc_rep_portal = OSC_REPLY_PORTAL, }, -- 1.8.3.1