From: Prakash Surya Date: Mon, 22 Oct 2012 23:34:29 +0000 (-0700) Subject: LU-2139 ofd: Do async commit if SOFT_SYNC is seen X-Git-Tag: 2.5.53~55 X-Git-Url: https://git.whamcloud.com/?p=fs%2Flustre-release.git;a=commitdiff_plain;h=9761d5c52aeef31a8c3112c3fb3ec9e24b37c800;ds=sidebyside LU-2139 ofd: Do async commit if SOFT_SYNC is seen This change implements the server side portion of the OBD_BRW_SOFT_SYNC agreement. When this flag is seen on an incoming BRW page, the server will begin an asynchronous commit operation via dt_commit_async. The intention is for this operation to commit pages belonging to a client which has "too many" outstanding unstable pages in its cache. Upon the commit operation completing, the last_commited value should percolate out to the client of interest, allowing it to drop some of its pinned unstable pages. Unfortunately, there is currently no optimization to proactively send the last_committed value to the client immediately after the commit operation succeeds. It must be piggybacked on any future IO or pings made by the client. Added fed_soft_sync_count which is set to a limit of 16 SOFT_SYNC RPCs, and it will be reset as soon as there is a non-SOFT_SYNC RPC. Added ofd_soft_sync_limit as a tunable proc value for the limit of fed_soft_sync_count. Signed-off-by: Prakash Surya Signed-off-by: Lai Siyao Change-Id: I1bac4bc517ab023a5b298f34bb695c7ba8ece896 Reviewed-on: http://review.whamcloud.com/5935 Reviewed-by: Andreas Dilger Tested-by: Jenkins Reviewed-by: Jinshan Xiong Tested-by: Maloo Reviewed-by: Oleg Drokin --- diff --git a/lustre/include/lustre_export.h b/lustre/include/lustre_export.h index 3759ebf..4377963 100644 --- a/lustre/include/lustre_export.h +++ b/lustre/include/lustre_export.h @@ -90,13 +90,16 @@ struct ec_export_data { /* echo client */ struct filter_export_data { struct tg_export_data fed_ted; spinlock_t fed_lock; /**< protects fed_mod_list */ - long fed_dirty; /* in bytes */ - long fed_grant; /* in bytes */ - cfs_list_t fed_mod_list; /* files being modified */ - int fed_mod_count;/* items in fed_writing list */ - long fed_pending; /* bytes just being written */ - __u32 fed_group; - __u8 fed_pagesize; /* log2 of client page size */ + long fed_dirty; /* in bytes */ + long fed_grant; /* in bytes */ + cfs_list_t fed_mod_list; /* files being modified */ + long fed_pending; /* bytes just being written */ + /* count of SOFT_SYNC RPCs, which will be reset after + * ofd_soft_sync_limit number of RPCs, and trigger a sync. */ + atomic_t fed_soft_sync_count; + int fed_mod_count;/* items in fed_writing list */ + __u32 fed_group; + __u8 fed_pagesize; /* log2 of client page size */ }; struct mgs_export_data { diff --git a/lustre/ofd/lproc_ofd.c b/lustre/ofd/lproc_ofd.c index 88e54cb..4354497 100644 --- a/lustre/ofd/lproc_ofd.c +++ b/lustre/ofd/lproc_ofd.c @@ -469,6 +469,25 @@ int lprocfs_ofd_wr_grant_compat_disable(struct file *file, const char *buffer, return count; } +int lprocfs_ofd_rd_soft_sync_limit(char *page, char **start, off_t off, + int count, int *eof, void *data) +{ + struct obd_device *obd = data; + struct ofd_device *ofd = ofd_dev(obd->obd_lu_dev); + + return lprocfs_rd_uint(page, start, off, count, eof, + &ofd->ofd_soft_sync_limit); +} + +int lprocfs_ofd_wr_soft_sync_limit(struct file *file, const char *buffer, + unsigned long count, void *data) +{ + struct obd_device *obd = data; + struct ofd_device *ofd = ofd_dev(obd->obd_lu_dev); + + return lprocfs_wr_uint(file, buffer, count, &ofd->ofd_soft_sync_limit); +} + static struct lprocfs_vars lprocfs_ofd_obd_vars[] = { { "uuid", lprocfs_rd_uuid, 0, 0 }, { "blocksize", lprocfs_rd_blksize, 0, 0 }, @@ -516,6 +535,8 @@ static struct lprocfs_vars lprocfs_ofd_obd_vars[] = { { "capa_count", lprocfs_ofd_rd_capa_count, 0, 0 }, { "job_cleanup_interval", lprocfs_rd_job_interval, lprocfs_wr_job_interval, 0}, + { "soft_sync_limit", lprocfs_ofd_rd_soft_sync_limit, + lprocfs_ofd_wr_soft_sync_limit, 0}, { 0 } }; diff --git a/lustre/ofd/ofd_dev.c b/lustre/ofd/ofd_dev.c index 70c3d8c0..2bdbde6 100644 --- a/lustre/ofd/ofd_dev.c +++ b/lustre/ofd/ofd_dev.c @@ -1638,6 +1638,7 @@ static int ofd_init0(const struct lu_env *env, struct ofd_device *m, m->ofd_syncjournal = 0; ofd_slc_set(m); m->ofd_grant_compat_disable = 0; + m->ofd_soft_sync_limit = OFD_SOFT_SYNC_LIMIT_DEFAULT; /* statfs data */ spin_lock_init(&m->ofd_osfs_lock); diff --git a/lustre/ofd/ofd_internal.h b/lustre/ofd/ofd_internal.h index d96a517..9285a1f 100644 --- a/lustre/ofd/ofd_internal.h +++ b/lustre/ofd/ofd_internal.h @@ -72,6 +72,8 @@ struct ofd_mod_data { #define OFD_FMD_MAX_NUM_DEFAULT 128 #define OFD_FMD_MAX_AGE_DEFAULT ((obd_timeout + 10) * HZ) +#define OFD_SOFT_SYNC_LIMIT_DEFAULT 16 + /* request stats */ enum { LPROC_OFD_STATS_READ = 0, @@ -181,6 +183,8 @@ struct ofd_device { * supporting OBD_CONNECT_GRANT_PARAM? */ ofd_grant_compat_disable:1; struct seq_server_site ofd_seq_site; + /* the limit of SOFT_SYNC RPCs that will trigger a soft sync */ + unsigned int ofd_soft_sync_limit; }; static inline struct ofd_device *ofd_dev(struct lu_device *d) diff --git a/lustre/ofd/ofd_io.c b/lustre/ofd/ofd_io.c index a67e6c7..e8e41b9 100644 --- a/lustre/ofd/ofd_io.c +++ b/lustre/ofd/ofd_io.c @@ -166,6 +166,7 @@ static int ofd_preprw_write(const struct lu_env *env, struct obd_export *exp, lnb[j+k].lnb_flags = rnb[i].rnb_flags; if (!(rnb[i].rnb_flags & OBD_BRW_GRANTED)) lnb[j+k].lnb_rc = -ENOSPC; + /* remote client can't break through quota */ if (exp_connect_rmtclient(exp)) lnb[j+k].lnb_flags &= ~OBD_BRW_NOQUOTA; @@ -395,10 +396,56 @@ out: return rc; } +struct ofd_soft_sync_callback { + struct dt_txn_commit_cb ossc_cb; + struct obd_export *ossc_exp; +}; + +static void ofd_cb_soft_sync(struct lu_env *env, struct thandle *th, + struct dt_txn_commit_cb *cb, int err) +{ + struct ofd_soft_sync_callback *ossc; + + ossc = container_of(cb, struct ofd_soft_sync_callback, ossc_cb); + + CDEBUG(D_INODE, "export %p soft sync count is reset\n", ossc->ossc_exp); + atomic_set(&ossc->ossc_exp->exp_filter_data.fed_soft_sync_count, 0); + + class_export_cb_put(ossc->ossc_exp); + OBD_FREE_PTR(ossc); +} + +static int ofd_soft_sync_cb_add(struct thandle *th, struct obd_export *exp) +{ + struct ofd_soft_sync_callback *ossc; + struct dt_txn_commit_cb *dcb; + int rc; + + OBD_ALLOC_PTR(ossc); + if (ossc == NULL) + return -ENOMEM; + + ossc->ossc_exp = class_export_cb_get(exp); + + dcb = &ossc->ossc_cb; + dcb->dcb_func = ofd_cb_soft_sync; + CFS_INIT_LIST_HEAD(&dcb->dcb_linkage); + strncpy(dcb->dcb_name, "ofd_cb_soft_sync", MAX_COMMIT_CB_STR_LEN); + dcb->dcb_name[MAX_COMMIT_CB_STR_LEN - 1] = '\0'; + + rc = dt_trans_cb_add(th, dcb); + if (rc) { + class_export_cb_put(exp); + OBD_FREE_PTR(ossc); + } + + return rc; +} + static int -ofd_commitrw_write(const struct lu_env *env, struct ofd_device *ofd, - struct lu_fid *fid, struct lu_attr *la, - struct filter_fid *ff, int objcount, +ofd_commitrw_write(const struct lu_env *env, struct obd_export *exp, + struct ofd_device *ofd, struct lu_fid *fid, + struct lu_attr *la, struct filter_fid *ff, int objcount, int niocount, struct niobuf_local *lnb, int old_rc) { struct ofd_thread_info *info = ofd_info(env); @@ -408,6 +455,9 @@ ofd_commitrw_write(const struct lu_env *env, struct ofd_device *ofd, int rc = 0; int retries = 0; int i; + struct filter_export_data *fed = &exp->exp_filter_data; + bool soft_sync = false; + bool cb_registered = false; ENTRY; @@ -447,6 +497,8 @@ retry: th->th_sync = 1; break; } + if (lnb[i].lnb_flags & OBD_BRW_SOFT_SYNC) + soft_sync = true; } } @@ -487,6 +539,12 @@ out_stop: if (rc == -ENOSPC) th->th_sync = 1; + /* do this before trans stop in case commit has finished */ + if (!th->th_sync && soft_sync && !cb_registered) { + ofd_soft_sync_cb_add(th, exp); + cb_registered = true; + } + ofd_trans_stop(env, ofd, th, rc); if (rc == -ENOSPC && retries++ < 3) { CDEBUG(D_INODE, "retry after force commit, retries:%d\n", @@ -494,6 +552,13 @@ out_stop: goto retry; } + if (!soft_sync) + /* reset fed_soft_sync_count upon non-SOFT_SYNC RPC */ + atomic_set(&fed->fed_soft_sync_count, 0); + else if (atomic_inc_return(&fed->fed_soft_sync_count) == + ofd->ofd_soft_sync_limit) + dt_commit_async(env, ofd->ofd_osd); + out: dt_bufs_put(env, o, lnb, niocount); ofd_read_unlock(env, fo); @@ -543,7 +608,7 @@ int ofd_commitrw(const struct lu_env *env, int cmd, struct obd_export *exp, ofd_prepare_fidea(ff, oa); } - rc = ofd_commitrw_write(env, ofd, &info->fti_fid, + rc = ofd_commitrw_write(env, exp, ofd, &info->fti_fid, &info->fti_attr, ff, objcount, npages, lnb, old_rc); if (rc == 0) diff --git a/lustre/ofd/ofd_obd.c b/lustre/ofd/ofd_obd.c index edb5bf2..c843e34 100644 --- a/lustre/ofd/ofd_obd.c +++ b/lustre/ofd/ofd_obd.c @@ -348,6 +348,7 @@ static int ofd_init_export(struct obd_export *exp) spin_lock_init(&exp->exp_filter_data.fed_lock); CFS_INIT_LIST_HEAD(&exp->exp_filter_data.fed_mod_list); + atomic_set(&exp->exp_filter_data.fed_soft_sync_count, 0); spin_lock(&exp->exp_lock); exp->exp_connecting = 1; spin_unlock(&exp->exp_lock);