LU-2139 ofd: Do async commit if SOFT_SYNC is seen

This change implements the server side portion of the OBD_BRW_SOFT_SYNC
agreement. When this flag is seen on an incoming BRW page, the server
will begin an asynchronous commit operation via dt_commit_async.

The intention is for this operation to commit pages belonging to a
client which has "too many" outstanding unstable pages in its cache.
Upon the commit operation completing, the last_commited value should
percolate out to the client of interest, allowing it to drop some of
its pinned unstable pages.

Unfortunately, there is currently no optimization to proactively send
the last_committed value to the client immediately after the commit
operation succeeds. It must be piggybacked on any future IO or
pings made by the client.

Added fed_soft_sync_count which is set to a limit of 16 SOFT_SYNC
RPCs, and it will be reset as soon as there is a non-SOFT_SYNC RPC.

Added ofd_soft_sync_limit as a tunable proc value for the limit of
fed_soft_sync_count.

Signed-off-by: Prakash Surya <surya1@llnl.gov>
Signed-off-by: Lai Siyao <lai.siyao@intel.com>
Change-Id: I1bac4bc517ab023a5b298f34bb695c7ba8ece896
Reviewed-on: http://review.whamcloud.com/5935
Reviewed-by: Andreas Dilger <andreas.dilger@intel.com>
Tested-by: Jenkins
Reviewed-by: Jinshan Xiong <jinshan.xiong@intel.com>
Tested-by: Maloo <hpdd-maloo@intel.com>
Reviewed-by: Oleg Drokin <oleg.drokin@intel.com>

diff --git a/lustre/include/lustre_export.h b/lustre/include/lustre_export.h
index 3759ebf..4377963 100644 (file)
--- a/lustre/include/lustre_export.h
+++ b/lustre/include/lustre_export.h
@@ -90,13 +90,16 @@ struct ec_export_data { /* echo client */
 struct filter_export_data {
        struct tg_export_data   fed_ted;
        spinlock_t              fed_lock;       /**< protects fed_mod_list */
 struct filter_export_data {
        struct tg_export_data   fed_ted;
        spinlock_t              fed_lock;       /**< protects fed_mod_list */

-        long                       fed_dirty;    /* in bytes */
-        long                       fed_grant;    /* in bytes */
-        cfs_list_t                 fed_mod_list; /* files being modified */
-        int                        fed_mod_count;/* items in fed_writing list */
-        long                       fed_pending;  /* bytes just being written */
-        __u32                      fed_group;
-       __u8                       fed_pagesize; /* log2 of client page size */
+       long                    fed_dirty;    /* in bytes */
+       long                    fed_grant;    /* in bytes */
+       cfs_list_t              fed_mod_list; /* files being modified */
+       long                    fed_pending;  /* bytes just being written */
+       /* count of SOFT_SYNC RPCs, which will be reset after
+        * ofd_soft_sync_limit number of RPCs, and trigger a sync. */
+       atomic_t                fed_soft_sync_count;
+       int                     fed_mod_count;/* items in fed_writing list */
+       __u32                   fed_group;
+       __u8                    fed_pagesize; /* log2 of client page size */

 };
 
 struct mgs_export_data {
 };
 
 struct mgs_export_data {

diff --git a/lustre/ofd/lproc_ofd.c b/lustre/ofd/lproc_ofd.c
index 88e54cb..4354497 100644 (file)
--- a/lustre/ofd/lproc_ofd.c
+++ b/lustre/ofd/lproc_ofd.c
@@ -469,6 +469,25 @@ int lprocfs_ofd_wr_grant_compat_disable(struct file *file, const char *buffer,
        return count;
 }
 
        return count;
 }
 

+int lprocfs_ofd_rd_soft_sync_limit(char *page, char **start, off_t off,
+                                  int count, int *eof, void *data)
+{
+       struct obd_device       *obd = data;
+       struct ofd_device       *ofd = ofd_dev(obd->obd_lu_dev);
+
+       return lprocfs_rd_uint(page, start, off, count, eof,
+                              &ofd->ofd_soft_sync_limit);
+}
+
+int lprocfs_ofd_wr_soft_sync_limit(struct file *file, const char *buffer,
+                                  unsigned long count, void *data)
+{
+       struct obd_device       *obd = data;
+       struct ofd_device       *ofd = ofd_dev(obd->obd_lu_dev);
+
+       return lprocfs_wr_uint(file, buffer, count, &ofd->ofd_soft_sync_limit);
+}
+

 static struct lprocfs_vars lprocfs_ofd_obd_vars[] = {
        { "uuid",                lprocfs_rd_uuid, 0, 0 },
        { "blocksize",           lprocfs_rd_blksize, 0, 0 },
 static struct lprocfs_vars lprocfs_ofd_obd_vars[] = {
        { "uuid",                lprocfs_rd_uuid, 0, 0 },
        { "blocksize",           lprocfs_rd_blksize, 0, 0 },


@@ -516,6 +535,8 @@ static struct lprocfs_vars lprocfs_ofd_obd_vars[] = {
        { "capa_count",          lprocfs_ofd_rd_capa_count, 0, 0 },
        { "job_cleanup_interval", lprocfs_rd_job_interval,
                                  lprocfs_wr_job_interval, 0},
        { "capa_count",          lprocfs_ofd_rd_capa_count, 0, 0 },
        { "job_cleanup_interval", lprocfs_rd_job_interval,
                                  lprocfs_wr_job_interval, 0},

+       { "soft_sync_limit",     lprocfs_ofd_rd_soft_sync_limit,
+                                lprocfs_ofd_wr_soft_sync_limit, 0},

        { 0 }
 };
 
        { 0 }
 };
 

diff --git a/lustre/ofd/ofd_dev.c b/lustre/ofd/ofd_dev.c
index 70c3d8c..2bdbde6 100644 (file)
--- a/lustre/ofd/ofd_dev.c
+++ b/lustre/ofd/ofd_dev.c
@@ -1638,6 +1638,7 @@ static int ofd_init0(const struct lu_env *env, struct ofd_device *m,
        m->ofd_syncjournal = 0;
        ofd_slc_set(m);
        m->ofd_grant_compat_disable = 0;
        m->ofd_syncjournal = 0;
        ofd_slc_set(m);
        m->ofd_grant_compat_disable = 0;

+       m->ofd_soft_sync_limit = OFD_SOFT_SYNC_LIMIT_DEFAULT;

 
        /* statfs data */
        spin_lock_init(&m->ofd_osfs_lock);
 
        /* statfs data */
        spin_lock_init(&m->ofd_osfs_lock);

diff --git a/lustre/ofd/ofd_internal.h b/lustre/ofd/ofd_internal.h
index d96a517..9285a1f 100644 (file)
--- a/lustre/ofd/ofd_internal.h
+++ b/lustre/ofd/ofd_internal.h
@@ -72,6 +72,8 @@ struct ofd_mod_data {
 #define OFD_FMD_MAX_NUM_DEFAULT 128
 #define OFD_FMD_MAX_AGE_DEFAULT ((obd_timeout + 10) * HZ)
 
 #define OFD_FMD_MAX_NUM_DEFAULT 128
 #define OFD_FMD_MAX_AGE_DEFAULT ((obd_timeout + 10) * HZ)
 

+#define OFD_SOFT_SYNC_LIMIT_DEFAULT 16
+

 /* request stats */
 enum {
        LPROC_OFD_STATS_READ = 0,
 /* request stats */
 enum {
        LPROC_OFD_STATS_READ = 0,


@@ -181,6 +183,8 @@ struct ofd_device {
                                  * supporting OBD_CONNECT_GRANT_PARAM? */
                                 ofd_grant_compat_disable:1;
        struct seq_server_site   ofd_seq_site;
                                  * supporting OBD_CONNECT_GRANT_PARAM? */
                                 ofd_grant_compat_disable:1;
        struct seq_server_site   ofd_seq_site;

+       /* the limit of SOFT_SYNC RPCs that will trigger a soft sync */
+       unsigned int             ofd_soft_sync_limit;

 };
 
 static inline struct ofd_device *ofd_dev(struct lu_device *d)
 };
 
 static inline struct ofd_device *ofd_dev(struct lu_device *d)

diff --git a/lustre/ofd/ofd_io.c b/lustre/ofd/ofd_io.c
index a67e6c7..e8e41b9 100644 (file)
--- a/lustre/ofd/ofd_io.c
+++ b/lustre/ofd/ofd_io.c
@@ -166,6 +166,7 @@ static int ofd_preprw_write(const struct lu_env *env, struct obd_export *exp,
                        lnb[j+k].lnb_flags = rnb[i].rnb_flags;
                        if (!(rnb[i].rnb_flags & OBD_BRW_GRANTED))
                                lnb[j+k].lnb_rc = -ENOSPC;
                        lnb[j+k].lnb_flags = rnb[i].rnb_flags;
                        if (!(rnb[i].rnb_flags & OBD_BRW_GRANTED))
                                lnb[j+k].lnb_rc = -ENOSPC;

+

                        /* remote client can't break through quota */
                        if (exp_connect_rmtclient(exp))
                                lnb[j+k].lnb_flags &= ~OBD_BRW_NOQUOTA;
                        /* remote client can't break through quota */
                        if (exp_connect_rmtclient(exp))
                                lnb[j+k].lnb_flags &= ~OBD_BRW_NOQUOTA;


@@ -395,10 +396,56 @@ out:
        return rc;
 }
 
        return rc;
 }
 

+struct ofd_soft_sync_callback {
+       struct dt_txn_commit_cb  ossc_cb;
+       struct obd_export       *ossc_exp;
+};
+
+static void ofd_cb_soft_sync(struct lu_env *env, struct thandle *th,
+                            struct dt_txn_commit_cb *cb, int err)
+{
+       struct ofd_soft_sync_callback   *ossc;
+
+       ossc = container_of(cb, struct ofd_soft_sync_callback, ossc_cb);
+
+       CDEBUG(D_INODE, "export %p soft sync count is reset\n", ossc->ossc_exp);
+       atomic_set(&ossc->ossc_exp->exp_filter_data.fed_soft_sync_count, 0);
+
+       class_export_cb_put(ossc->ossc_exp);
+       OBD_FREE_PTR(ossc);
+}
+
+static int ofd_soft_sync_cb_add(struct thandle *th, struct obd_export *exp)
+{
+       struct ofd_soft_sync_callback           *ossc;
+       struct dt_txn_commit_cb                 *dcb;
+       int                                      rc;
+
+       OBD_ALLOC_PTR(ossc);
+       if (ossc == NULL)
+               return -ENOMEM;
+
+       ossc->ossc_exp = class_export_cb_get(exp);
+
+       dcb = &ossc->ossc_cb;
+       dcb->dcb_func = ofd_cb_soft_sync;
+       CFS_INIT_LIST_HEAD(&dcb->dcb_linkage);
+       strncpy(dcb->dcb_name, "ofd_cb_soft_sync", MAX_COMMIT_CB_STR_LEN);
+       dcb->dcb_name[MAX_COMMIT_CB_STR_LEN - 1] = '\0';
+
+       rc = dt_trans_cb_add(th, dcb);
+       if (rc) {
+               class_export_cb_put(exp);
+               OBD_FREE_PTR(ossc);
+       }
+
+       return rc;
+}
+

 static int
 static int

-ofd_commitrw_write(const struct lu_env *env, struct ofd_device *ofd,
-                  struct lu_fid *fid, struct lu_attr *la,
-                  struct filter_fid *ff, int objcount,
+ofd_commitrw_write(const struct lu_env *env, struct obd_export *exp,
+                  struct ofd_device *ofd, struct lu_fid *fid,
+                  struct lu_attr *la, struct filter_fid *ff, int objcount,

                   int niocount, struct niobuf_local *lnb, int old_rc)
 {
        struct ofd_thread_info  *info = ofd_info(env);
                   int niocount, struct niobuf_local *lnb, int old_rc)
 {
        struct ofd_thread_info  *info = ofd_info(env);


@@ -408,6 +455,9 @@ ofd_commitrw_write(const struct lu_env *env, struct ofd_device *ofd,
        int                      rc = 0;
        int                      retries = 0;
        int                      i;
        int                      rc = 0;
        int                      retries = 0;
        int                      i;

+       struct filter_export_data *fed = &exp->exp_filter_data;
+       bool                     soft_sync = false;
+       bool                     cb_registered = false;

 
        ENTRY;
 
 
        ENTRY;
 


@@ -447,6 +497,8 @@ retry:
                                th->th_sync = 1;
                                break;
                        }
                                th->th_sync = 1;
                                break;
                        }

+                       if (lnb[i].lnb_flags & OBD_BRW_SOFT_SYNC)
+                               soft_sync = true;

                }
        }
 
                }
        }
 


@@ -487,6 +539,12 @@ out_stop:
        if (rc == -ENOSPC)
                th->th_sync = 1;
 
        if (rc == -ENOSPC)
                th->th_sync = 1;
 

+       /* do this before trans stop in case commit has finished */
+       if (!th->th_sync && soft_sync && !cb_registered) {
+               ofd_soft_sync_cb_add(th, exp);
+               cb_registered = true;
+       }
+

        ofd_trans_stop(env, ofd, th, rc);
        if (rc == -ENOSPC && retries++ < 3) {
                CDEBUG(D_INODE, "retry after force commit, retries:%d\n",
        ofd_trans_stop(env, ofd, th, rc);
        if (rc == -ENOSPC && retries++ < 3) {
                CDEBUG(D_INODE, "retry after force commit, retries:%d\n",


@@ -494,6 +552,13 @@ out_stop:
                goto retry;
        }
 
                goto retry;
        }
 

+       if (!soft_sync)
+               /* reset fed_soft_sync_count upon non-SOFT_SYNC RPC */
+               atomic_set(&fed->fed_soft_sync_count, 0);
+       else if (atomic_inc_return(&fed->fed_soft_sync_count) ==
+                ofd->ofd_soft_sync_limit)
+               dt_commit_async(env, ofd->ofd_osd);
+

 out:
        dt_bufs_put(env, o, lnb, niocount);
        ofd_read_unlock(env, fo);
 out:
        dt_bufs_put(env, o, lnb, niocount);
        ofd_read_unlock(env, fo);


@@ -543,7 +608,7 @@ int ofd_commitrw(const struct lu_env *env, int cmd, struct obd_export *exp,
                        ofd_prepare_fidea(ff, oa);
                }
 
                        ofd_prepare_fidea(ff, oa);
                }
 

-               rc = ofd_commitrw_write(env, ofd, &info->fti_fid,
+               rc = ofd_commitrw_write(env, exp, ofd, &info->fti_fid,

                                        &info->fti_attr, ff, objcount, npages,
                                        lnb, old_rc);
                if (rc == 0)
                                        &info->fti_attr, ff, objcount, npages,
                                        lnb, old_rc);
                if (rc == 0)

diff --git a/lustre/ofd/ofd_obd.c b/lustre/ofd/ofd_obd.c
index edb5bf2..c843e34 100644 (file)
--- a/lustre/ofd/ofd_obd.c
+++ b/lustre/ofd/ofd_obd.c
@@ -348,6 +348,7 @@ static int ofd_init_export(struct obd_export *exp)
 
        spin_lock_init(&exp->exp_filter_data.fed_lock);
        CFS_INIT_LIST_HEAD(&exp->exp_filter_data.fed_mod_list);
 
        spin_lock_init(&exp->exp_filter_data.fed_lock);
        CFS_INIT_LIST_HEAD(&exp->exp_filter_data.fed_mod_list);

+       atomic_set(&exp->exp_filter_data.fed_soft_sync_count, 0);

        spin_lock(&exp->exp_lock);
        exp->exp_connecting = 1;
        spin_unlock(&exp->exp_lock);
        spin_lock(&exp->exp_lock);
        exp->exp_connecting = 1;
        spin_unlock(&exp->exp_lock);