LU-2756 ost: buffer utilization rate of OST rqbd

This patch covered a couple of things:
- different request buffer size for different OST services
  only requests on OST_IO_PORTAL need large buffers

- add extra 46K to OST_IO_BUFSIZE (buffer size is increased to 64K)
  size of OST_IO_MAXREQSIZE is about 18K, if OST_IO_BUFSIZE is only
  a little bigger than OST_IO_MAXREQSIZE, then each rqbd can only fit
  in 2-3 request even there are 18K bytes left in this rqbd, memory
  utilization rate is low. Extra bytes can make buffer utilization
  rate increase to 72%. Also, it can avoid version compatibility
  issue in the future if we want to increase request size again.

Signed-off-by: Liang Zhen <liang.zhen@intel.com>
Change-Id: Iaf8b45520b000dd5219a4af2ec3f5b2ea7f53134
Reviewed-on: http://review.whamcloud.com/5626
Tested-by: Hudson
Reviewed-by: Fan Yong <fan.yong@intel.com>
Tested-by: Maloo <whamcloud.maloo@gmail.com>
Reviewed-by: Andreas Dilger <andreas.dilger@intel.com>

diff --git a/lustre/include/lustre_net.h b/lustre/include/lustre_net.h
index c500fb9..cd6225a 100644 (file)
--- a/lustre/include/lustre_net.h
+++ b/lustre/include/lustre_net.h
@@ -456,22 +456,38 @@
 #define OSS_CR_NTHRS_MAX       64
 
 /**
 #define OSS_CR_NTHRS_MAX       64
 
 /**

- * OST_MAXREQSIZE ~=
- * lustre_msg + obdo + obd_ioobj + DT_MAX_BRW_PAGES * niobuf_remote
+ * OST_IO_MAXREQSIZE ~=
+ *     lustre_msg + ptlrpc_body + obdo + obd_ioobj +
+ *     DT_MAX_BRW_PAGES * niobuf_remote

  *
  * - single object with 16 pages is 512 bytes
  *
  * - single object with 16 pages is 512 bytes

- * - OST_MAXREQSIZE must be at least 1 page of cookies plus some spillover
+ * - OST_IO_MAXREQSIZE must be at least 1 page of cookies plus some spillover

  * - Must be a multiple of 1024
  * - Must be a multiple of 1024

+ * - actual size is about 18K

  */
  */

-#define _OST_MAXREQSIZE_SUM (sizeof(struct lustre_msg) + sizeof(struct obdo) + \
-                            sizeof(struct obd_ioobj) + DT_MAX_BRW_PAGES * \
-                            sizeof(struct niobuf_remote))
-#define OST_MAXREQSIZE (((_OST_MAXREQSIZE_SUM - 1) | (1024 - 1)) + 1)
+#define _OST_MAXREQSIZE_SUM (sizeof(struct lustre_msg) + \
+                            sizeof(struct ptlrpc_body) + \
+                            sizeof(struct obdo) + \
+                            sizeof(struct obd_ioobj) + \
+                            sizeof(struct niobuf_remote) * DT_MAX_BRW_PAGES)
+/**
+ * FIEMAP request can be 4K+ for now
+ */
+#define OST_MAXREQSIZE         (5 * 1024)
+#define OST_IO_MAXREQSIZE      max_t(int, OST_MAXREQSIZE, \
+                               (((_OST_MAXREQSIZE_SUM - 1) | (1024 - 1)) + 1))

 
 

-#define OST_MAXREPSIZE  (9 * 1024)
+#define OST_MAXREPSIZE         (9 * 1024)
+#define OST_IO_MAXREPSIZE      OST_MAXREPSIZE

 
 

-#define OST_NBUFS       64
-#define OST_BUFSIZE     (OST_MAXREQSIZE + 1024)
+#define OST_NBUFS              64
+/** OST_BUFSIZE = max_reqsize + max sptlrpc payload size */
+#define OST_BUFSIZE            max_t(int, OST_MAXREQSIZE + 1024, 16 * 1024)
+/**
+ * OST_IO_MAXREQSIZE is 18K, giving extra 46K can increase buffer utilization
+ * rate of request buffer, please check comment of MDS_LOV_BUFSIZE for details.
+ */
+#define OST_IO_BUFSIZE         max_t(int, OST_IO_MAXREQSIZE + 1024, 64 * 1024)

 
 /* Macro to hide a typecast. */
 #define ptlrpc_req_async_args(req) ((void *)&req->rq_async_args)
 
 /* Macro to hide a typecast. */
 #define ptlrpc_req_async_args(req) ((void *)&req->rq_async_args)

diff --git a/lustre/ost/ost_handler.c b/lustre/ost/ost_handler.c
index 562210e..873c23e 100644 (file)
--- a/lustre/ost/ost_handler.c
+++ b/lustre/ost/ost_handler.c
@@ -2611,9 +2611,9 @@ static int ost_setup(struct obd_device *obd, struct lustre_cfg* lcfg)
                .psc_watchdog_factor    = OSS_SERVICE_WATCHDOG_FACTOR,
                .psc_buf                = {
                        .bc_nbufs               = OST_NBUFS,
                .psc_watchdog_factor    = OSS_SERVICE_WATCHDOG_FACTOR,
                .psc_buf                = {
                        .bc_nbufs               = OST_NBUFS,

-                       .bc_buf_size            = OST_BUFSIZE,
-                       .bc_req_max_size        = OST_MAXREQSIZE,
-                       .bc_rep_max_size        = OST_MAXREPSIZE,
+                       .bc_buf_size            = OST_IO_BUFSIZE,
+                       .bc_req_max_size        = OST_IO_MAXREQSIZE,
+                       .bc_rep_max_size        = OST_IO_MAXREPSIZE,

                        .bc_req_portal          = OST_IO_PORTAL,
                        .bc_rep_portal          = OSC_REPLY_PORTAL,
                },
                        .bc_req_portal          = OST_IO_PORTAL,
                        .bc_rep_portal          = OSC_REPLY_PORTAL,
                },