LU-5521 grant: quiet message on grant waiting timeout

Use at_max in osc_enter_cache() to bound how long we wait for grant
space before switching to synchronous I/Os. Do not print a message
on the console when the timeout is hit since such long wait can
be legitimate with flaky network (i.e. BRW is resent multiple times).

Signed-off-by: Johann Lombardi <johann.lombardi@intel.com>
Change-Id: I63b40783381f6133e2f77dbc0f827e13f571ccd2
Reviewed-on: http://review.whamcloud.com/12146
Tested-by: Jenkins
Reviewed-by: Niu Yawei <yawei.niu@intel.com>
Tested-by: Maloo <hpdd-maloo@intel.com>
Reviewed-by: Jinshan Xiong <jinshan.xiong@intel.com>
Reviewed-by: Oleg Drokin <oleg.drokin@intel.com>

diff --git a/lustre/osc/osc_cache.c b/lustre/osc/osc_cache.c
index 52870d5..99a18dc 100644 (file)
--- a/lustre/osc/osc_cache.c
+++ b/lustre/osc/osc_cache.c
@@ -1491,7 +1491,7 @@ static int osc_enter_cache_try(struct client_obd *cli,
 {
        int rc;
 
-       OSC_DUMP_GRANT(D_CACHE, cli, "need:%d.\n", bytes);
+       OSC_DUMP_GRANT(D_CACHE, cli, "need:%d\n", bytes);
 
        rc = osc_reserve_grant(cli, bytes);
        if (rc < 0)
@@ -1532,15 +1532,17 @@ static int ocw_granted(struct client_obd *cli, struct osc_cache_waiter *ocw)
 static int osc_enter_cache(const struct lu_env *env, struct client_obd *cli,
                           struct osc_async_page *oap, int bytes)
 {
-       struct osc_object *osc = oap->oap_obj;
-       struct lov_oinfo  *loi = osc->oo_oinfo;
-       struct osc_cache_waiter ocw;
-       struct l_wait_info lwi = LWI_TIMEOUT_INTR(cfs_time_seconds(600), NULL,
-                                                 LWI_ON_SIGNAL_NOOP, NULL);
-       int rc = -EDQUOT;
+       struct osc_object       *osc = oap->oap_obj;
+       struct lov_oinfo        *loi = osc->oo_oinfo;
+       struct osc_cache_waiter  ocw;
+       struct l_wait_info       lwi;
+       int                      rc = -EDQUOT;
        ENTRY;
 
-       OSC_DUMP_GRANT(D_CACHE, cli, "need:%d.\n", bytes);
+       lwi = LWI_TIMEOUT_INTR(cfs_time_seconds(AT_OFF ? obd_timeout : at_max),
+                              NULL, LWI_ON_SIGNAL_NOOP, NULL);
+
+       OSC_DUMP_GRANT(D_CACHE, cli, "need:%d\n", bytes);
 
        spin_lock(&cli->cl_loi_list_lock);
 
@@ -1548,12 +1550,16 @@ static int osc_enter_cache(const struct lu_env *env, struct client_obd *cli,
         * of queued writes and create a discontiguous rpc stream */
        if (OBD_FAIL_CHECK(OBD_FAIL_OSC_NO_GRANT) ||
            cli->cl_dirty_max_pages == 0 ||
-           cli->cl_ar.ar_force_sync || loi->loi_ar.ar_force_sync)
+           cli->cl_ar.ar_force_sync || loi->loi_ar.ar_force_sync) {
+               OSC_DUMP_GRANT(D_CACHE, cli, "forced sync i/o\n");
                GOTO(out, rc = -EDQUOT);
+       }
 
        /* Hopefully normal case - cache space and write credits available */
-       if (osc_enter_cache_try(cli, oap, bytes, 0))
+       if (osc_enter_cache_try(cli, oap, bytes, 0)) {
+               OSC_DUMP_GRANT(D_CACHE, cli, "granted from cache\n");
                GOTO(out, rc = 0);
+       }
 
        /* We can get here for two reasons: too many dirty pages in cache, or
         * run out of grants. In both cases we should write dirty pages out.
@@ -1578,42 +1584,51 @@ static int osc_enter_cache(const struct lu_env *env, struct client_obd *cli,
 
                spin_lock(&cli->cl_loi_list_lock);
 
-               /* l_wait_event is interrupted by signal, or timed out */
                if (rc < 0) {
-                       switch (rc) {
-                       case -ETIMEDOUT:
-                               OSC_DUMP_GRANT(D_ERROR, cli,
-                                               "try to reserve %d.\n", bytes);
-                               osc_extent_tree_dump(D_ERROR, osc);
-                               rc = -EDQUOT;
-                               break;
-                       case -EINTR:
-                               /* Ensures restartability - LU-3581 */
-                               rc = -ERESTARTSYS;
-                               break;
-                       default:
-                               CDEBUG(D_CACHE, "%s: event for cache space @"
-                                      " %p never arrived due to %d\n",
-                                      cli->cl_import->imp_obd->obd_name,
-                                      &ocw, rc);
-                               break;
-                       }
+                       /* l_wait_event is interrupted by signal or timed out */
                        list_del_init(&ocw.ocw_entry);
-                       GOTO(out, rc);
+                       break;
                }
-
                LASSERT(list_empty(&ocw.ocw_entry));
                rc = ocw.ocw_rc;
 
                if (rc != -EDQUOT)
-                       GOTO(out, rc);
-               if (osc_enter_cache_try(cli, oap, bytes, 0))
-                       GOTO(out, rc = 0);
+                       break;
+               if (osc_enter_cache_try(cli, oap, bytes, 0)) {
+                       rc = 0;
+                       break;
+               }
+       }
+
+       switch (rc) {
+       case 0:
+               OSC_DUMP_GRANT(D_CACHE, cli, "finally got grant space\n");
+               break;
+       case -ETIMEDOUT:
+               OSC_DUMP_GRANT(D_CACHE, cli,
+                              "timeout, fall back to sync i/o\n");
+               osc_extent_tree_dump(D_CACHE, osc);
+               /* fall back to synchronous I/O */
+               rc = -EDQUOT;
+               break;
+       case -EINTR:
+               /* Ensures restartability - LU-3581 */
+               OSC_DUMP_GRANT(D_CACHE, cli, "interrupted\n");
+               rc = -ERESTARTSYS;
+               break;
+       case -EDQUOT:
+               OSC_DUMP_GRANT(D_CACHE, cli,
+                              "no grant space, fall back to sync i/o\n");
+               break;
+       default:
+               CDEBUG(D_CACHE, "%s: event for cache space @ %p never arrived "
+                      "due to %d, fall back to sync i/o\n",
+                      cli->cl_import->imp_obd->obd_name, &ocw, rc);
+               break;
        }
        EXIT;
 out:
        spin_unlock(&cli->cl_loi_list_lock);
-       OSC_DUMP_GRANT(D_CACHE, cli, "returned %d.\n", rc);
        RETURN(rc);
 }
 
@@ -1640,10 +1655,8 @@ void osc_wake_cache_waiters(struct client_obd *cli)
                        goto wakeup;
                }
 
-               ocw->ocw_rc = 0;
-               if (!osc_enter_cache_try(cli, ocw->ocw_oap, ocw->ocw_grant, 0))
-                       ocw->ocw_rc = -EDQUOT;
-
+               if (osc_enter_cache_try(cli, ocw->ocw_oap, ocw->ocw_grant, 0))
+                       ocw->ocw_rc = 0;
 wakeup:
                CDEBUG(D_CACHE, "wake up %p for oap %p, avail grant %ld, %d\n",
                       ocw, ocw->ocw_oap, cli->cl_avail_grant, ocw->ocw_rc);