- added more debug in osc. Currently two ideas are working for checks why osc does...

(1) cache size is choosen kind of cumbersome way and not enough, what makes writing thread waiting for cache in osc_enter_cache(). Cache size curremtly does not depend on how good wire is able to send RPCs to remote node what seems to be an artificial limit in some cases. If this is so, the fix would be to adjust cache size depending on much it is needed. This however does not explain why second subtest suffers from slowdown whereas first one does not.

(2) due to some reasons (limited cache or limited grant from OST) many async RPCs are falled back to synchronous IO what makes substential slowdown.

Added debug on real cluster should show what is going on there. Idea that issue on server has a little chance to be true, as server uses direct IO and there is almost not ability to be slowed down due to some cache issues, etc.

diff --git a/lustre/include/linux/obd.h b/lustre/include/linux/obd.h
index a6a7146..4b00043 100644 (file)
--- a/lustre/include/linux/obd.h
+++ b/lustre/include/linux/obd.h
@@ -302,7 +302,6 @@ struct mds_server_data;
 #define OSC_MAX_DIRTY_DEFAULT   (4*OSC_MAX_RIF_DEFAULT*PTLRPC_MAX_BRW_SIZE>>20)
 #define OSC_MAX_DIRTY_MB_MAX    512     /* totally arbitrary */
 
-
 struct mdc_rpc_lock;
 struct client_obd {
         struct obd_import       *cl_import;
@@ -373,6 +372,8 @@ struct client_obd {
         
         unsigned long            cl_dirty_dmax;
         unsigned long            cl_dirty_dmin;
+
+        unsigned long            cl_sync_rpcs;
 };
 
 /* Like a client, with some hangers-on.  Keep mc_client_obd first so that we

diff --git a/lustre/ldlm/ldlm_lib.c b/lustre/ldlm/ldlm_lib.c
index c4f5f60..e16c61a 100644 (file)
--- a/lustre/ldlm/ldlm_lib.c
+++ b/lustre/ldlm/ldlm_lib.c
@@ -238,6 +238,7 @@ int client_obd_setup(struct obd_device *obddev, obd_count len, void *buf)
 
         cli->cl_dirty = 0;
         cli->cl_avail_grant = 0;
+        
         /* FIXME: should limit this for the sum of all cl_dirty_max */
         cli->cl_dirty_max = OSC_MAX_DIRTY_DEFAULT * 1024 * 1024;
         if (cli->cl_dirty_max >> PAGE_SHIFT > num_physpages / 8)
@@ -269,6 +270,7 @@ int client_obd_setup(struct obd_device *obddev, obd_count len, void *buf)
         cli->cl_dirty_av = 0;
         cli->cl_dirty_dmax = 0;
         cli->cl_dirty_dmin = 0;
+        cli->cl_sync_rpcs = 0;
 
         if (num_physpages >> (20 - PAGE_SHIFT) <= 128) { /* <= 128 MB */
                 cli->cl_max_pages_per_rpc = PTLRPC_MAX_BRW_PAGES / 4;
@@ -373,6 +375,7 @@ int client_obd_cleanup(struct obd_device *obddev, int flags)
 
         if (!cli->cl_import)
                 RETURN(-EINVAL);
+
         if (cli->cl_mgmtcli_obd) {
                 mgmtcli_deregister_for_events_t dereg_f;
 
@@ -383,10 +386,10 @@ int client_obd_cleanup(struct obd_device *obddev, int flags)
 
         if (cli->cl_write_gaps) {
                 CWARN("%s: [writes num: %lu, reads num: %lu]: %lu write gaps: %lu "
-                      "av. (usec), %lu total (usec)\n", obddev->obd_name,
-                      cli->cl_write_num, cli->cl_read_num, cli->cl_write_gaps,
-                      cli->cl_write_gap_sum / cli->cl_write_gaps,
-                      cli->cl_write_gap_sum);
+                      "av. (usec), %lu total (usec), %lu rpcs falled back to sync\n",
+                      obddev->obd_name, cli->cl_write_num, cli->cl_read_num,
+                      cli->cl_write_gaps, cli->cl_write_gap_sum / cli->cl_write_gaps,
+                      cli->cl_write_gap_sum, cli->cl_sync_rpcs);
         }
         
         if (cli->cl_cache_wait_num) {
@@ -396,14 +399,13 @@ int client_obd_cleanup(struct obd_device *obddev, int flags)
         }
 
         if (cli->cl_dirty_av) {
-                CWARN("%s: pipe loading av. %lu (b), max pipe space %lu (b), pipe "
-                      "loading ratio %lu%%\n", obddev->obd_name, cli->cl_dirty_av,
-                      cli->cl_dirty_max, (cli->cl_dirty_av * 100) / cli->cl_dirty_max);
-        }
-
-        if (cli->cl_dirty_dmax) {
-                CWARN("%s: pipe dirty max %lu (b), pipe dirty min %lu (b)\n",
-                      obddev->obd_name, cli->cl_dirty_dmax, cli->cl_dirty_dmin);
+                CWARN("%s: pipe loading av. %lu (b), max pipe room %lu (b), pipe "
+                      "loading av. ratio %lu%%, pipe dirty max %lu (b), pipe dirty "
+                      "min %lu (b), pipe loading max ratio %lu%%\n", obddev->obd_name,
+                      cli->cl_dirty_av, cli->cl_dirty_max,
+                      (cli->cl_dirty_av * 100) / cli->cl_dirty_max,
+                      cli->cl_dirty_dmax, cli->cl_dirty_dmin,
+                      (cli->cl_dirty_dmax * 100) / cli->cl_dirty_max);
         }
 
         /* Here we try to drop the security structure after destroy import,

diff --git a/lustre/osc/osc_request.c b/lustre/osc/osc_request.c
index b2463ac..de0f529 100644 (file)
--- a/lustre/osc/osc_request.c
+++ b/lustre/osc/osc_request.c
@@ -1861,13 +1861,14 @@ static int osc_enter_cache(struct client_obd *cli, struct lov_oinfo *loi,
         struct osc_cache_waiter ocw;
         struct l_wait_info lwi = { 0 };
         struct timeval start, stop;
+        ENTRY;
 
         CDEBUG(D_CACHE, "dirty: %ld dirty_max: %ld dropped: %lu grant: %lu\n",
                cli->cl_dirty, cli->cl_dirty_max, cli->cl_lost_grant,
                cli->cl_avail_grant);
 
         if (cli->cl_dirty_max < PAGE_SIZE)
-                return(-EDQUOT);
+                GOTO(out, -EDQUOT);
 
         if (~0ul - cli->cl_dirty_sum <= cli->cl_dirty) {
                 cli->cl_dirty_av = (cli->cl_dirty_av +
@@ -1892,7 +1893,7 @@ static int osc_enter_cache(struct client_obd *cli, struct lov_oinfo *loi,
             cli->cl_avail_grant >= PAGE_SIZE) {
                 /* account for ourselves */
                 osc_consume_write_grant(cli, oap);
-                return(0);
+                RETURN(0);
         }
 
         /* Make sure that there are write rpcs in flight to wait for.  This
@@ -1925,7 +1926,10 @@ static int osc_enter_cache(struct client_obd *cli, struct lov_oinfo *loi,
                 RETURN(ocw.ocw_rc);
         }
 
-        RETURN(-EDQUOT);
+        EXIT;
+out:
+        cli->cl_sync_rpcs++;
+        return -EDQUOT;
 }
 
 /* the companion to enter_cache, called when an oap is no longer part of the