LU-2139 osc: Use SOFT_SYNC to urge server commit

This change adds a BRW page flag, OBD_BRW_SOFT_SYNC. This flag is
intended to urge a server to commit a client's unstable pages to
stable storage. A client will add this flag to any BRW requests while
it is in a state where it has "many" unstable pages pinned in its cache.

The server, upon receiving a page with this flag set, *should* begin
an async commit. The idea being that, with the proactive async commit,
the client's pinned unstable pages will transition into a stable state
faster than they would have otherwise. Although, the server side portion
of this agreement is still unimplemented, so the OBD_BRW_SOFT_SYNC flag
will currently fall on deaf ears.

Signed-off-by: Prakash Surya <surya1@llnl.gov>
Change-Id: I08946a5ec6afab390655643f6d3c8320bf010aea
Reviewed-on: http://review.whamcloud.com/4375
Tested-by: Jenkins
Tested-by: Maloo <hpdd-maloo@intel.com>
Reviewed-by: Jinshan Xiong <jinshan.xiong@intel.com>
Reviewed-by: Oleg Drokin <oleg.drokin@intel.com>

diff --git a/lustre/include/lustre/lustre_idl.h b/lustre/include/lustre/lustre_idl.h
index 9271c00..adf3095 100644 (file)
--- a/lustre/include/lustre/lustre_idl.h
+++ b/lustre/include/lustre/lustre_idl.h
@@ -1784,6 +1784,10 @@ extern void lustre_swab_obd_statfs (struct obd_statfs *os);
 #define OBD_BRW_MEMALLOC       0x800 /* Client runs in the "kswapd" context */
 #define OBD_BRW_OVER_USRQUOTA 0x1000 /* Running out of user quota */
 #define OBD_BRW_OVER_GRPQUOTA 0x2000 /* Running out of group quota */
+#define OBD_BRW_SOFT_SYNC     0x4000 /* This flag notifies the server
+                                     * that the client is running low on
+                                     * space for unstable pages; asking
+                                     * it to sync quickly */
 
 #define OBD_OBJECT_EOF 0xffffffffffffffffULL
 

diff --git a/lustre/osc/osc_cache.c b/lustre/osc/osc_cache.c
index 1b35afd..4d97b3e 100644 (file)
--- a/lustre/osc/osc_cache.c
+++ b/lustre/osc/osc_cache.c
@@ -2367,6 +2367,9 @@ int osc_queue_async_io(const struct lu_env *env, struct cl_io *io,
                        RETURN(rc);
        }
 
+       if (osc_over_unstable_soft_limit(cli))
+               brw_flags |= OBD_BRW_SOFT_SYNC;
+
        oap->oap_cmd = cmd;
        oap->oap_page_off = ops->ops_from;
        oap->oap_count = ops->ops_to - ops->ops_from;

diff --git a/lustre/osc/osc_internal.h b/lustre/osc/osc_internal.h
index d89bad9..40279d4 100644 (file)
--- a/lustre/osc/osc_internal.h
+++ b/lustre/osc/osc_internal.h
@@ -207,4 +207,5 @@ int osc_quota_poll_check(struct obd_export *exp, struct if_quotacheck *qchk);
 
 void osc_inc_unstable_pages(struct ptlrpc_request *req);
 void osc_dec_unstable_pages(struct ptlrpc_request *req);
+int  osc_over_unstable_soft_limit(struct client_obd *cli);
 #endif /* OSC_INTERNAL_H */

diff --git a/lustre/osc/osc_page.c b/lustre/osc/osc_page.c
index 37b62af..affd0d2 100644 (file)
--- a/lustre/osc/osc_page.c
+++ b/lustre/osc/osc_page.c
@@ -544,6 +544,30 @@ int osc_page_init(const struct lu_env *env, struct cl_object *obj,
        return result;
 }
 
+int osc_over_unstable_soft_limit(struct client_obd *cli)
+{
+       long obd_upages, obd_dpages, osc_upages;
+
+       /* Can't check cli->cl_unstable_count, therefore, no soft limit */
+       if (cli == NULL)
+               return 0;
+
+       obd_upages = cfs_atomic_read(&obd_unstable_pages);
+       obd_dpages = cfs_atomic_read(&obd_dirty_pages);
+
+       osc_upages = cfs_atomic_read(&cli->cl_unstable_count);
+
+       /* obd_max_dirty_pages is the max number of (dirty + unstable)
+        * pages allowed at any given time. To simulate an unstable page
+        * only limit, we subtract the current number of dirty pages
+        * from this max. This difference is roughly the amount of pages
+        * currently available for unstable pages. Thus, the soft limit
+        * is half of that difference. Check osc_upages to ensure we don't
+        * set SOFT_SYNC for OSCs without any outstanding unstable pages. */
+       return osc_upages != 0 &&
+              obd_upages >= (obd_max_dirty_pages - obd_dpages) / 2;
+}
+
 /**
  * Helper function called by osc_io_submit() for every page in an immediate
  * transfer (i.e., transferred synchronously).
@@ -567,6 +591,9 @@ void osc_page_submit(const struct lu_env *env, struct osc_page *opg,
        oap->oap_count     = opg->ops_to - opg->ops_from;
        oap->oap_brw_flags = OBD_BRW_SYNC | brw_flags;
 
+       if (osc_over_unstable_soft_limit(oap->oap_cli))
+               oap->oap_brw_flags |= OBD_BRW_SOFT_SYNC;
+
        if (!client_is_remote(osc_export(obj)) &&
                        cfs_capable(CFS_CAP_SYS_RESOURCE)) {
                oap->oap_brw_flags |= OBD_BRW_NOQUOTA;

diff --git a/lustre/osc/osc_request.c b/lustre/osc/osc_request.c
index c5bca5e..2aa6b30 100644 (file)
--- a/lustre/osc/osc_request.c
+++ b/lustre/osc/osc_request.c
@@ -1172,8 +1172,9 @@ static int check_write_rcs(struct ptlrpc_request *req,
 static inline int can_merge_pages(struct brw_page *p1, struct brw_page *p2)
 {
         if (p1->flag != p2->flag) {
-                unsigned mask = ~(OBD_BRW_FROM_GRANT| OBD_BRW_NOCACHE|
-                                  OBD_BRW_SYNC|OBD_BRW_ASYNC|OBD_BRW_NOQUOTA);
+               unsigned mask = ~(OBD_BRW_FROM_GRANT | OBD_BRW_NOCACHE |
+                                 OBD_BRW_SYNC       | OBD_BRW_ASYNC   |
+                                 OBD_BRW_NOQUOTA    | OBD_BRW_SOFT_SYNC);
 
                 /* warn if we try to combine flags that we don't know to be
                  * safe to combine */

diff --git a/lustre/ptlrpc/wiretest.c b/lustre/ptlrpc/wiretest.c
index 4ae2e64..9d6a17b 100644 (file)
--- a/lustre/ptlrpc/wiretest.c
+++ b/lustre/ptlrpc/wiretest.c
@@ -1867,6 +1867,12 @@ void lustre_assert_wire_constants(void)
                OBD_BRW_ASYNC);
        LASSERTF(OBD_BRW_MEMALLOC == 0x800, "found 0x%.8x\n",
                OBD_BRW_MEMALLOC);
+       LASSERTF(OBD_BRW_OVER_USRQUOTA == 0x1000, "found 0x%.8x\n",
+               OBD_BRW_OVER_USRQUOTA);
+       LASSERTF(OBD_BRW_OVER_GRPQUOTA == 0x2000, "found 0x%.8x\n",
+               OBD_BRW_OVER_GRPQUOTA);
+       LASSERTF(OBD_BRW_SOFT_SYNC == 0x4000, "found 0x%.8x\n",
+               OBD_BRW_SOFT_SYNC);
 
        /* Checks for struct ost_body */
        LASSERTF((int)sizeof(struct ost_body) == 208, "found %lld\n",

diff --git a/lustre/utils/wirecheck.c b/lustre/utils/wirecheck.c
index 1ac48ab..faa27b4 100644 (file)
--- a/lustre/utils/wirecheck.c
+++ b/lustre/utils/wirecheck.c
@@ -840,6 +840,9 @@ check_niobuf_remote(void)
        CHECK_DEFINE_X(OBD_BRW_SRVLOCK);
        CHECK_DEFINE_X(OBD_BRW_ASYNC);
        CHECK_DEFINE_X(OBD_BRW_MEMALLOC);
+       CHECK_DEFINE_X(OBD_BRW_OVER_USRQUOTA);
+       CHECK_DEFINE_X(OBD_BRW_OVER_GRPQUOTA);
+       CHECK_DEFINE_X(OBD_BRW_SOFT_SYNC);
 }
 
 static void

diff --git a/lustre/utils/wiretest.c b/lustre/utils/wiretest.c
index 62cbfa1..593b8aa 100644 (file)
--- a/lustre/utils/wiretest.c
+++ b/lustre/utils/wiretest.c
@@ -1875,6 +1875,12 @@ void lustre_assert_wire_constants(void)
                OBD_BRW_ASYNC);
        LASSERTF(OBD_BRW_MEMALLOC == 0x800, "found 0x%.8x\n",
                OBD_BRW_MEMALLOC);
+       LASSERTF(OBD_BRW_OVER_USRQUOTA == 0x1000, "found 0x%.8x\n",
+               OBD_BRW_OVER_USRQUOTA);
+       LASSERTF(OBD_BRW_OVER_GRPQUOTA == 0x2000, "found 0x%.8x\n",
+               OBD_BRW_OVER_GRPQUOTA);
+       LASSERTF(OBD_BRW_SOFT_SYNC == 0x4000, "found 0x%.8x\n",
+               OBD_BRW_SOFT_SYNC);
 
        /* Checks for struct ost_body */
        LASSERTF((int)sizeof(struct ost_body) == 208, "found %lld\n",