From 27815a0611a2e315a9a7696a20c2f257d48aeb7e Mon Sep 17 00:00:00 2001 From: Prakash Surya Date: Mon, 22 Oct 2012 16:34:29 -0700 Subject: [PATCH] LU-2139 osc: Use SOFT_SYNC to urge server commit This change adds a BRW page flag, OBD_BRW_SOFT_SYNC. This flag is intended to urge a server to commit a client's unstable pages to stable storage. A client will add this flag to any BRW requests while it is in a state where it has "many" unstable pages pinned in its cache. The server, upon receiving a page with this flag set, *should* begin an async commit. The idea being that, with the proactive async commit, the client's pinned unstable pages will transition into a stable state faster than they would have otherwise. Although, the server side portion of this agreement is still unimplemented, so the OBD_BRW_SOFT_SYNC flag will currently fall on deaf ears. Signed-off-by: Prakash Surya Change-Id: I08946a5ec6afab390655643f6d3c8320bf010aea Reviewed-on: http://review.whamcloud.com/4375 Tested-by: Jenkins Tested-by: Maloo Reviewed-by: Jinshan Xiong Reviewed-by: Oleg Drokin --- lustre/include/lustre/lustre_idl.h | 4 ++++ lustre/osc/osc_cache.c | 3 +++ lustre/osc/osc_internal.h | 1 + lustre/osc/osc_page.c | 27 +++++++++++++++++++++++++++ lustre/osc/osc_request.c | 5 +++-- lustre/ptlrpc/wiretest.c | 6 ++++++ lustre/utils/wirecheck.c | 3 +++ lustre/utils/wiretest.c | 6 ++++++ 8 files changed, 53 insertions(+), 2 deletions(-) diff --git a/lustre/include/lustre/lustre_idl.h b/lustre/include/lustre/lustre_idl.h index 9271c00..adf3095 100644 --- a/lustre/include/lustre/lustre_idl.h +++ b/lustre/include/lustre/lustre_idl.h @@ -1784,6 +1784,10 @@ extern void lustre_swab_obd_statfs (struct obd_statfs *os); #define OBD_BRW_MEMALLOC 0x800 /* Client runs in the "kswapd" context */ #define OBD_BRW_OVER_USRQUOTA 0x1000 /* Running out of user quota */ #define OBD_BRW_OVER_GRPQUOTA 0x2000 /* Running out of group quota */ +#define OBD_BRW_SOFT_SYNC 0x4000 /* This flag notifies the server + * that the client is running low on + * space for unstable pages; asking + * it to sync quickly */ #define OBD_OBJECT_EOF 0xffffffffffffffffULL diff --git a/lustre/osc/osc_cache.c b/lustre/osc/osc_cache.c index 1b35afd..4d97b3e 100644 --- a/lustre/osc/osc_cache.c +++ b/lustre/osc/osc_cache.c @@ -2367,6 +2367,9 @@ int osc_queue_async_io(const struct lu_env *env, struct cl_io *io, RETURN(rc); } + if (osc_over_unstable_soft_limit(cli)) + brw_flags |= OBD_BRW_SOFT_SYNC; + oap->oap_cmd = cmd; oap->oap_page_off = ops->ops_from; oap->oap_count = ops->ops_to - ops->ops_from; diff --git a/lustre/osc/osc_internal.h b/lustre/osc/osc_internal.h index d89bad9..40279d4 100644 --- a/lustre/osc/osc_internal.h +++ b/lustre/osc/osc_internal.h @@ -207,4 +207,5 @@ int osc_quota_poll_check(struct obd_export *exp, struct if_quotacheck *qchk); void osc_inc_unstable_pages(struct ptlrpc_request *req); void osc_dec_unstable_pages(struct ptlrpc_request *req); +int osc_over_unstable_soft_limit(struct client_obd *cli); #endif /* OSC_INTERNAL_H */ diff --git a/lustre/osc/osc_page.c b/lustre/osc/osc_page.c index 37b62af..affd0d2 100644 --- a/lustre/osc/osc_page.c +++ b/lustre/osc/osc_page.c @@ -544,6 +544,30 @@ int osc_page_init(const struct lu_env *env, struct cl_object *obj, return result; } +int osc_over_unstable_soft_limit(struct client_obd *cli) +{ + long obd_upages, obd_dpages, osc_upages; + + /* Can't check cli->cl_unstable_count, therefore, no soft limit */ + if (cli == NULL) + return 0; + + obd_upages = cfs_atomic_read(&obd_unstable_pages); + obd_dpages = cfs_atomic_read(&obd_dirty_pages); + + osc_upages = cfs_atomic_read(&cli->cl_unstable_count); + + /* obd_max_dirty_pages is the max number of (dirty + unstable) + * pages allowed at any given time. To simulate an unstable page + * only limit, we subtract the current number of dirty pages + * from this max. This difference is roughly the amount of pages + * currently available for unstable pages. Thus, the soft limit + * is half of that difference. Check osc_upages to ensure we don't + * set SOFT_SYNC for OSCs without any outstanding unstable pages. */ + return osc_upages != 0 && + obd_upages >= (obd_max_dirty_pages - obd_dpages) / 2; +} + /** * Helper function called by osc_io_submit() for every page in an immediate * transfer (i.e., transferred synchronously). @@ -567,6 +591,9 @@ void osc_page_submit(const struct lu_env *env, struct osc_page *opg, oap->oap_count = opg->ops_to - opg->ops_from; oap->oap_brw_flags = OBD_BRW_SYNC | brw_flags; + if (osc_over_unstable_soft_limit(oap->oap_cli)) + oap->oap_brw_flags |= OBD_BRW_SOFT_SYNC; + if (!client_is_remote(osc_export(obj)) && cfs_capable(CFS_CAP_SYS_RESOURCE)) { oap->oap_brw_flags |= OBD_BRW_NOQUOTA; diff --git a/lustre/osc/osc_request.c b/lustre/osc/osc_request.c index c5bca5e..2aa6b30 100644 --- a/lustre/osc/osc_request.c +++ b/lustre/osc/osc_request.c @@ -1172,8 +1172,9 @@ static int check_write_rcs(struct ptlrpc_request *req, static inline int can_merge_pages(struct brw_page *p1, struct brw_page *p2) { if (p1->flag != p2->flag) { - unsigned mask = ~(OBD_BRW_FROM_GRANT| OBD_BRW_NOCACHE| - OBD_BRW_SYNC|OBD_BRW_ASYNC|OBD_BRW_NOQUOTA); + unsigned mask = ~(OBD_BRW_FROM_GRANT | OBD_BRW_NOCACHE | + OBD_BRW_SYNC | OBD_BRW_ASYNC | + OBD_BRW_NOQUOTA | OBD_BRW_SOFT_SYNC); /* warn if we try to combine flags that we don't know to be * safe to combine */ diff --git a/lustre/ptlrpc/wiretest.c b/lustre/ptlrpc/wiretest.c index 4ae2e64..9d6a17b 100644 --- a/lustre/ptlrpc/wiretest.c +++ b/lustre/ptlrpc/wiretest.c @@ -1867,6 +1867,12 @@ void lustre_assert_wire_constants(void) OBD_BRW_ASYNC); LASSERTF(OBD_BRW_MEMALLOC == 0x800, "found 0x%.8x\n", OBD_BRW_MEMALLOC); + LASSERTF(OBD_BRW_OVER_USRQUOTA == 0x1000, "found 0x%.8x\n", + OBD_BRW_OVER_USRQUOTA); + LASSERTF(OBD_BRW_OVER_GRPQUOTA == 0x2000, "found 0x%.8x\n", + OBD_BRW_OVER_GRPQUOTA); + LASSERTF(OBD_BRW_SOFT_SYNC == 0x4000, "found 0x%.8x\n", + OBD_BRW_SOFT_SYNC); /* Checks for struct ost_body */ LASSERTF((int)sizeof(struct ost_body) == 208, "found %lld\n", diff --git a/lustre/utils/wirecheck.c b/lustre/utils/wirecheck.c index 1ac48ab..faa27b4 100644 --- a/lustre/utils/wirecheck.c +++ b/lustre/utils/wirecheck.c @@ -840,6 +840,9 @@ check_niobuf_remote(void) CHECK_DEFINE_X(OBD_BRW_SRVLOCK); CHECK_DEFINE_X(OBD_BRW_ASYNC); CHECK_DEFINE_X(OBD_BRW_MEMALLOC); + CHECK_DEFINE_X(OBD_BRW_OVER_USRQUOTA); + CHECK_DEFINE_X(OBD_BRW_OVER_GRPQUOTA); + CHECK_DEFINE_X(OBD_BRW_SOFT_SYNC); } static void diff --git a/lustre/utils/wiretest.c b/lustre/utils/wiretest.c index 62cbfa1..593b8aa 100644 --- a/lustre/utils/wiretest.c +++ b/lustre/utils/wiretest.c @@ -1875,6 +1875,12 @@ void lustre_assert_wire_constants(void) OBD_BRW_ASYNC); LASSERTF(OBD_BRW_MEMALLOC == 0x800, "found 0x%.8x\n", OBD_BRW_MEMALLOC); + LASSERTF(OBD_BRW_OVER_USRQUOTA == 0x1000, "found 0x%.8x\n", + OBD_BRW_OVER_USRQUOTA); + LASSERTF(OBD_BRW_OVER_GRPQUOTA == 0x2000, "found 0x%.8x\n", + OBD_BRW_OVER_GRPQUOTA); + LASSERTF(OBD_BRW_SOFT_SYNC == 0x4000, "found 0x%.8x\n", + OBD_BRW_SOFT_SYNC); /* Checks for struct ost_body */ LASSERTF((int)sizeof(struct ost_body) == 208, "found %lld\n", -- 1.8.3.1