- small cleanups in cobd

- added a patch for debugging write gaps (av. gap time and sum gaps time)

diff --git a/lustre/cobd/cache_obd.c b/lustre/cobd/cache_obd.c
index ccc6e1d..65aad33 100644 (file)
--- a/lustre/cobd/cache_obd.c
+++ b/lustre/cobd/cache_obd.c
@@ -147,7 +147,7 @@ static int cobd_setup(struct obd_device *obd, obd_count len, void *buf)
         }
         cobd->cache_exp = class_conn2export(&conn);
         
-        /* default set cache on, but nothins is realy connected yet, will be
+        /* default set cache on, but nothing is realy connected yet, will be
          * done in cobd_connect() time. */
         cobd->cache_on = 1;
 
@@ -535,6 +535,57 @@ static int cobd_iocontrol(unsigned int cmd, struct obd_export *exp,
         RETURN(rc);
 }
 
+static int cobd_notify(struct obd_device *obd, struct obd_device *watched,
+                       int active, void *data)
+{
+        struct obd_export *cobd_exp;
+        int rc = 0;
+        ENTRY;
+
+        cobd_exp = cobd_get_exp(obd);
+        rc = obd_notify(class_exp2obd(cobd_exp), watched, active, data);
+        RETURN(rc);
+}
+
+static int cobd_pin(struct obd_export *exp, obd_id ino, __u32 gen,
+                    int type, struct obd_client_handle *handle,
+                    int flag)
+{
+        struct obd_device *obd = class_exp2obd(exp);
+        struct obd_export *cobd_exp;
+        int rc = 0;
+        ENTRY;
+
+        if (obd == NULL) {
+                CERROR("invalid client cookie "LPX64"\n", 
+                       exp->exp_handle.h_cookie);
+                RETURN(-EINVAL);
+        }
+        cobd_exp = cobd_get_exp(obd);
+        rc = obd_pin(cobd_exp, ino, gen, type, handle, flag);
+        RETURN(rc);
+}
+
+static int cobd_unpin(struct obd_export *exp,
+                      struct obd_client_handle *handle,
+                      int flag)
+{
+        struct obd_device *obd = class_exp2obd(exp);
+        struct obd_export *cobd_exp;
+        int rc = 0;
+        ENTRY;
+
+        if (obd == NULL) {
+                CERROR("invalid client cookie "LPX64"\n", 
+                       exp->exp_handle.h_cookie);
+                RETURN(-EINVAL);
+        }
+        cobd_exp = cobd_get_exp(obd);
+        rc = obd_unpin(cobd_exp, handle, flag);
+        RETURN(rc);
+}
+
+/* data related stuff */
 static int cobd_dt_packmd(struct obd_export *exp,
                           struct lov_mds_md **disk_tgt,
                           struct lov_stripe_md *mem_src)
@@ -1087,56 +1138,6 @@ static int cobd_dt_llog_finish(struct obd_device *obd,
         RETURN(rc);
 }
 
-static int cobd_dt_notify(struct obd_device *obd, struct obd_device *watched,
-                          int active, void *data)
-{
-        struct obd_export *cobd_exp;
-        int rc = 0;
-        ENTRY;
-
-        cobd_exp = cobd_get_exp(obd);
-        rc = obd_notify(class_exp2obd(cobd_exp), watched, active, data);
-        RETURN(rc);
-}
-
-static int cobd_dt_pin(struct obd_export *exp, obd_id ino, __u32 gen,
-                       int type, struct obd_client_handle *handle,
-                       int flag)
-{
-        struct obd_device *obd = class_exp2obd(exp);
-        struct obd_export *cobd_exp;
-        int rc = 0;
-        ENTRY;
-
-        if (obd == NULL) {
-                CERROR("invalid client cookie "LPX64"\n", 
-                       exp->exp_handle.h_cookie);
-                RETURN(-EINVAL);
-        }
-        cobd_exp = cobd_get_exp(obd);
-        rc = obd_pin(cobd_exp, ino, gen, type, handle, flag);
-        RETURN(rc);
-}
-
-static int cobd_dt_unpin(struct obd_export *exp,
-                         struct obd_client_handle *handle,
-                         int flag)
-{
-        struct obd_device *obd = class_exp2obd(exp);
-        struct obd_export *cobd_exp;
-        int rc = 0;
-        ENTRY;
-
-        if (obd == NULL) {
-                CERROR("invalid client cookie "LPX64"\n", 
-                       exp->exp_handle.h_cookie);
-                RETURN(-EINVAL);
-        }
-        cobd_exp = cobd_get_exp(obd);
-        rc = obd_unpin(cobd_exp, handle, flag);
-        RETURN(rc);
-}
-
 static int cobd_dt_init_ea_size(struct obd_export *exp, int easize,
                                 int cookiesize)
 {
@@ -1161,6 +1162,7 @@ static int cobd_dt_import_event(struct obd_device *obd,
         RETURN(0); 
 }
 
+/* metadata related stuff */
 static int cobd_md_getstatus(struct obd_export *exp,
                              struct lustre_id *rootid)
 {
@@ -1621,6 +1623,9 @@ static struct obd_ops cobd_obd_ops = {
         .o_get_info               = cobd_get_info,
         .o_statfs                 = cobd_statfs,
         .o_iocontrol              = cobd_iocontrol,
+        .o_notify                 = cobd_notify,
+        .o_pin                    = cobd_pin,
+        .o_unpin                  = cobd_unpin,
 
         .o_packmd                 = cobd_dt_packmd,
         .o_unpackmd               = cobd_dt_unpackmd,
@@ -1649,9 +1654,6 @@ static struct obd_ops cobd_obd_ops = {
         .o_commitrw               = cobd_dt_commitrw,
         .o_llog_init              = cobd_dt_llog_init,
         .o_llog_finish            = cobd_dt_llog_finish,
-        .o_notify                 = cobd_dt_notify,
-        .o_pin                    = cobd_dt_pin,
-        .o_unpin                  = cobd_dt_unpin,
         .o_import_event           = cobd_dt_import_event,
         .o_init_ea_size           = cobd_dt_init_ea_size,
         .o_adjust_kms             = cobd_dt_adjust_kms,

diff --git a/lustre/include/linux/obd.h b/lustre/include/linux/obd.h
index 0cf0187..af1f903 100644 (file)
--- a/lustre/include/linux/obd.h
+++ b/lustre/include/linux/obd.h
@@ -357,6 +357,11 @@ struct client_obd {
         struct mdc_rpc_lock     *cl_close_lock;
         struct osc_creator       cl_oscc;
         int                      cl_async:1;
+
+        /* debug stuff */
+        struct timeval           cl_last_write_time;
+        long                     cl_write_gap_sum;
+        long                     cl_write_gaps;
 };
 
 /* Like a client, with some hangers-on.  Keep mc_client_obd first so that we

diff --git a/lustre/ldlm/ldlm_lib.c b/lustre/ldlm/ldlm_lib.c
index 5738725..0927476 100644 (file)
--- a/lustre/ldlm/ldlm_lib.c
+++ b/lustre/ldlm/ldlm_lib.c
@@ -254,6 +254,11 @@ int client_obd_setup(struct obd_device *obddev, obd_count len, void *buf)
         spin_lock_init(&cli->cl_read_page_hist.oh_lock);
         spin_lock_init(&cli->cl_write_page_hist.oh_lock);
 
+        memset(&cli->cl_last_write_time, 0,
+               sizeof(cli->cl_last_write_time));
+        cli->cl_write_gap_sum = 0;
+        cli->cl_write_gaps = 0;
+
         if (num_physpages >> (20 - PAGE_SHIFT) <= 128) { /* <= 128 MB */
                 cli->cl_max_pages_per_rpc = PTLRPC_MAX_BRW_PAGES / 4;
                 cli->cl_max_rpcs_in_flight = OSC_MAX_RIF_DEFAULT / 4;
@@ -374,8 +379,14 @@ int client_obd_cleanup(struct obd_device *obddev, int flags)
         class_import_put(cli->cl_import);
         cli->cl_import = NULL;
 
-        ldlm_put_ref(flags & OBD_OPT_FORCE);
+        if (cli->cl_write_gap_sum) {
+                CWARN("%s: %lu write gaps: %lu av. (usec), %lu total "
+                      "(usec)\n", obddev->obd_name, cli->cl_write_gaps,
+                      cli->cl_write_gap_sum / cli->cl_write_gaps,
+                      cli->cl_write_gap_sum);
+        }
 
+        ldlm_put_ref(flags & OBD_OPT_FORCE);
         RETURN(0);
 }
 

diff --git a/lustre/ldlm/ldlm_lockd.c b/lustre/ldlm/ldlm_lockd.c
index f4fae3c..6c459fe 100644 (file)
--- a/lustre/ldlm/ldlm_lockd.c
+++ b/lustre/ldlm/ldlm_lockd.c
@@ -596,7 +596,6 @@ find_existing_lock(struct obd_export *exp, struct lustre_handle *remote_hdl)
         return NULL;
 }
 
-
 int ldlm_handle_enqueue(struct ptlrpc_request *req,
                         ldlm_completion_callback completion_callback,
                         ldlm_blocking_callback blocking_callback,
@@ -614,9 +613,9 @@ int ldlm_handle_enqueue(struct ptlrpc_request *req,
 
         LDLM_DEBUG_NOLOCK("server-side enqueue handler START");
 
-        dlm_req = lustre_swab_reqbuf (req, MDS_REQ_INTENT_LOCKREQ_OFF,
-                                      sizeof (*dlm_req),
-                                      lustre_swab_ldlm_request);
+        dlm_req = lustre_swab_reqbuf(req, MDS_REQ_INTENT_LOCKREQ_OFF,
+                                     sizeof (*dlm_req),
+                                     lustre_swab_ldlm_request);
         if (dlm_req == NULL) {
                 CERROR ("Can't unpack dlm_req\n");
                 GOTO(out, rc = -EFAULT);

diff --git a/lustre/obdfilter/filter_io.c b/lustre/obdfilter/filter_io.c
index 702c4b8..337347f 100644 (file)
--- a/lustre/obdfilter/filter_io.c
+++ b/lustre/obdfilter/filter_io.c
@@ -821,6 +821,7 @@ cleanup:
         RETURN(rc);
 
 }
+
 int filter_write_extents(struct obd_export *exp, struct obd_ioobj *obj, int nobj,
                          int niocount, struct niobuf_local *local, int rc)
 {
@@ -941,7 +942,8 @@ int filter_brw(int cmd, struct obd_export *exp, struct obdo *oa,
         obdo_to_ioobj(oa, &ioo);
         ioo.ioo_bufcnt = oa_bufs;
 
-        ret = filter_preprw(cmd, exp, oa, 1, &ioo, oa_bufs, rnb, lnb, oti,NULL);
+        ret = filter_preprw(cmd, exp, oa, 1, &ioo, oa_bufs, rnb,
+                            lnb, oti, NULL);
         if (ret != 0)
                 GOTO(out, ret);
 

diff --git a/lustre/osc/osc_request.c b/lustre/osc/osc_request.c
index 98a0804..3b4e6fe 100644 (file)
--- a/lustre/osc/osc_request.c
+++ b/lustre/osc/osc_request.c
@@ -1354,6 +1354,10 @@ static int brw_interpret_oap(struct ptlrpc_request *request,
                 osc_ap_completion(cli, aa->aa_oa, oap, 1, rc);
         }
 
+        /* no write RPCs in flight, reset the time */
+        if (request->rq_reqmsg->opc == OST_WRITE && cli->cl_w_in_flight == 0)
+                do_gettimeofday(&cli->cl_last_write_time);
+
         osc_wake_cache_waiters(cli);
         osc_check_rpcs(cli);
         spin_unlock(&cli->cl_loi_list_lock);
@@ -1438,6 +1442,9 @@ out:
         RETURN(req);
 }
 
+/* strange write gap too long (15s) */
+#define CLI_ODD_WRITE_GAP 15000000
+
 static void lop_update_pending(struct client_obd *cli,
                                struct loi_oap_pages *lop, int cmd, int delta)
 {
@@ -1448,6 +1455,12 @@ static void lop_update_pending(struct client_obd *cli,
                 cli->cl_pending_r_pages += delta;
 }
 
+static long timeval_sub(struct timeval *large, struct timeval *small)
+{
+        return (large->tv_sec - small->tv_sec) * 1000000 +
+                (large->tv_usec - small->tv_usec);
+}
+
 /* the loi lock is held across this function but it's allowed to release
  * and reacquire it during its work */
 static int osc_send_oap_rpc(struct client_obd *cli, struct lov_oinfo *loi,
@@ -1601,10 +1614,27 @@ static int osc_send_oap_rpc(struct client_obd *cli, struct lov_oinfo *loi,
 
         spin_lock(&cli->cl_loi_list_lock);
 
+        /* collect write gaps and sum of them */
+        if (cmd == OBD_BRW_WRITE && cli->cl_w_in_flight == 0) {
+                struct timeval now;
+                long diff;
+        
+                do_gettimeofday(&now);
+
+                if (cli->cl_last_write_time.tv_sec) {
+                        diff = timeval_sub(&now, &cli->cl_last_write_time);
+                        if (diff < CLI_ODD_WRITE_GAP) {
+                                cli->cl_write_gap_sum += diff;
+                                cli->cl_write_gaps++;
+                        }
+                }
+        }        
+
         if (cmd == OBD_BRW_READ)
                 cli->cl_r_in_flight++;
         else
                 cli->cl_w_in_flight++;
+        
         /* queued sync pages can be torn down while the pages
          * were between the pending list and the rpc */
         list_for_each(pos, &aa->aa_oaps) {
@@ -1618,11 +1648,11 @@ static int osc_send_oap_rpc(struct client_obd *cli, struct lov_oinfo *loi,
         }
 
         CDEBUG(D_INODE, "req %p: %d pages, aa %p.  now %dr/%dw in flight\n",
-                        request, page_count, aa, cli->cl_r_in_flight,
-                        cli->cl_w_in_flight);
+               request, page_count, aa, cli->cl_r_in_flight, cli->cl_w_in_flight);
 
         oap->oap_request = ptlrpc_request_addref(request);
         request->rq_interpret_reply = brw_interpret_oap;
+
         ptlrpcd_add_req(request);
         RETURN(1);
 }