LU-169 clio: restart clio operations if layout changes

For glimpse and setattr operations, the layout may change as they may
not open file while running, so we need to restart these operations
in case layout is changed during the IO time.

Signed-off-by: Jinshan Xiong <jinshan.xiong@whamcloud.com>
Change-Id: Ia7ba1829164259e06af342be3f94e654ed0f89b8
Reviewed-on: http://review.whamcloud.com/2026
Tested-by: Hudson
Tested-by: Maloo <whamcloud.maloo@gmail.com>
Reviewed-by: Fan Yong <yong.fan@whamcloud.com>
Reviewed-by: Johann Lombardi <johann@whamcloud.com>
Reviewed-by: Oleg Drokin <green@whamcloud.com>

diff --git a/lustre/include/cl_object.h b/lustre/include/cl_object.h
index 8f27bd6..d8f6f17 100644 (file)
--- a/lustre/include/cl_object.h
+++ b/lustre/include/cl_object.h
@@ -2304,11 +2304,6 @@ struct cl_io {
         struct cl_lockset              ci_lockset;
         /** lock requirements, this is just a help info for sublayers. */
         enum cl_io_lock_dmd            ci_lockreq;
-        /**
-         * This io has held grouplock, to inform sublayers that
-         * don't do lockless i/o.
-         */
-        int                            ci_no_srvlock;
         union {
                 struct cl_rd_io {
                         struct cl_io_rw_common rd;
@@ -2352,7 +2347,29 @@ struct cl_io {
         struct cl_2queue     ci_queue;
         size_t               ci_nob;
         int                  ci_result;
-        int                  ci_continue;
+       unsigned int         ci_continue:1,
+       /**
+        * This io has held grouplock, to inform sublayers that
+        * don't do lockless i/o.
+        */
+                            ci_no_srvlock:1,
+       /**
+        * The whole IO need to be restarted because layout has been changed
+        */
+                            ci_need_restart:1,
+       /**
+        * Ignore layout change.
+        * Most of the CIT_MISC operations can ignore layout change, because
+        * the purpose to create this kind of cl_io is to give an environment
+        * to run clio methods, for example:
+        *   1. request group lock;
+        *   2. flush caching pages by osc;
+        *   3. writepage
+        *   4. echo client
+        * So far, only direct IO and glimpse clio need restart if layout
+        * change during IO time.
+        */
+                            ci_ignore_layout:1;
         /**
          * Number of pages owned by this IO. For invariant checking.
          */

diff --git a/lustre/include/lclient.h b/lustre/include/lclient.h
index c79ddfc..8be6c5b 100644 (file)
--- a/lustre/include/lclient.h
+++ b/lustre/include/lclient.h
@@ -107,6 +107,10 @@ struct ccc_io {
          * True iff io is processing glimpse right now.
          */
         int                  cui_glimpse;
+       /**
+        * Layout version when this IO is initialized
+        */
+       __u32                cui_layout_gen;
         /**
          * File descriptor against which IO is done.
          */

diff --git a/lustre/lclient/glimpse.c b/lustre/lclient/glimpse.c
index 31acda0..d03b855 100644 (file)
--- a/lustre/lclient/glimpse.c
+++ b/lustre/lclient/glimpse.c
@@ -224,6 +224,7 @@ int cl_glimpse_size0(struct inode *inode, int agl)
 
         result = cl_io_get(inode, &env, &io, &refcheck);
         if (result > 0) {
+       again:
                 result = cl_io_init(env, io, CIT_MISC, io->ci_obj);
                 if (result > 0)
                         /*
@@ -235,9 +236,11 @@ int cl_glimpse_size0(struct inode *inode, int agl)
                         result = cl_glimpse_lock(env, io, inode, io->ci_obj,
                                                  agl);
                 cl_io_fini(env, io);
-                cl_env_put(env, &refcheck);
-        }
-        RETURN(result);
+               if (unlikely(io->ci_need_restart))
+                       goto again;
+               cl_env_put(env, &refcheck);
+       }
+       RETURN(result);
 }
 
 int cl_local_size(struct inode *inode)

diff --git a/lustre/lclient/lcommon_cl.c b/lustre/lclient/lcommon_cl.c
index 19cb4aa..98e67b1 100644 (file)
--- a/lustre/lclient/lcommon_cl.c
+++ b/lustre/lclient/lcommon_cl.c
@@ -1031,6 +1031,7 @@ int cl_setattr_ost(struct inode *inode, const struct iattr *attr,
         io->u.ci_setattr.sa_valid = attr->ia_valid;
         io->u.ci_setattr.sa_capa = capa;
 
+again:
         if (cl_io_init(env, io, CIT_SETATTR, io->ci_obj) == 0) {
                 struct ccc_io *cio = ccc_env_io(env);
 
@@ -1044,8 +1045,10 @@ int cl_setattr_ost(struct inode *inode, const struct iattr *attr,
                 result = io->ci_result;
         }
         cl_io_fini(env, io);
-        cl_env_put(env, &refcheck);
-        RETURN(result);
+       if (unlikely(io->ci_need_restart))
+               goto again;
+       cl_env_put(env, &refcheck);
+       RETURN(result);
 }
 
 /*****************************************************************************

diff --git a/lustre/lclient/lcommon_misc.c b/lustre/lclient/lcommon_misc.c
index b5f9d7e..8f7ffc9 100644 (file)
--- a/lustre/lclient/lcommon_misc.c
+++ b/lustre/lclient/lcommon_misc.c
@@ -138,6 +138,7 @@ int cl_get_grouplock(struct cl_object *obj, unsigned long gid, int nonblock,
 
         io = ccc_env_thread_io(env);
         io->ci_obj = obj;
+       io->ci_ignore_layout = 1;
 
         rc = cl_io_init(env, io, CIT_MISC, io->ci_obj);
         if (rc) {

diff --git a/lustre/llite/file.c b/lustre/llite/file.c
index 8ac9cdf..06e61d8 100644 (file)
--- a/lustre/llite/file.c
+++ b/lustre/llite/file.c
@@ -2028,6 +2028,7 @@ int cl_sync_file_range(struct inode *inode, loff_t start, loff_t end,
 
        io = ccc_env_thread_io(env);
        io->ci_obj = cl_i2info(inode)->lli_clob;
+       io->ci_ignore_layout = 1;
 
        /* initialize parameters for sync */
        fio = &io->u.ci_fsync;

diff --git a/lustre/llite/rw.c b/lustre/llite/rw.c
index 01c995e..5040d62 100644 (file)
--- a/lustre/llite/rw.c
+++ b/lustre/llite/rw.c
@@ -1186,6 +1186,7 @@ int ll_writepage(struct page *vmpage, struct writeback_control *wbc)
 
         io = ccc_env_thread_io(env);
         io->ci_obj = clob;
+       io->ci_ignore_layout = 1;
         result = cl_io_init(env, io, CIT_MISC, clob);
         if (result == 0) {
                 page = cl_page_find(env, clob, vmpage->index,

diff --git a/lustre/llite/vvp_io.c b/lustre/llite/vvp_io.c
index b5f6995..befe985 100644 (file)
--- a/lustre/llite/vvp_io.c
+++ b/lustre/llite/vvp_io.c
@@ -85,18 +85,18 @@ static int vvp_io_fault_iter_init(const struct lu_env *env,
 
 static void vvp_io_fini(const struct lu_env *env, const struct cl_io_slice *ios)
 {
-        struct cl_io     *io  = ios->cis_io;
-        struct cl_object *obj = io->ci_obj;
+       struct cl_io     *io  = ios->cis_io;
+       struct cl_object *obj = io->ci_obj;
+       struct ccc_io    *cio = cl2ccc_io(env, ios);
+       __u32 gen;
+       int   result;
 
         CLOBINVRNT(env, obj, ccc_object_invariant(obj));
-        if (io->ci_type == CIT_READ) {
-                struct vvp_io     *vio  = cl2vvp_io(env, ios);
-                struct ccc_io     *cio  = cl2ccc_io(env, ios);
-
-                if (vio->cui_ra_window_set)
-                        ll_ra_read_ex(cio->cui_fd->fd_file, &vio->cui_bead);
-        }
 
+       /* check layout version */
+       result = ll_layout_refresh(ccc_object_inode(obj), &gen);
+       if (cio->cui_layout_gen > 0)
+               io->ci_need_restart = cio->cui_layout_gen == gen;
 }
 
 static void vvp_io_fault_fini(const struct lu_env *env,
@@ -540,6 +540,17 @@ out:
         return result;
 }
 
+static void vvp_io_read_fini(const struct lu_env *env, const struct cl_io_slice *ios)
+{
+       struct vvp_io *vio = cl2vvp_io(env, ios);
+       struct ccc_io *cio = cl2ccc_io(env, ios);
+
+       if (vio->cui_ra_window_set)
+               ll_ra_read_ex(cio->cui_fd->fd_file, &vio->cui_bead);
+
+       vvp_io_fini(env, ios);
+}
+
 static int vvp_io_write_start(const struct lu_env *env,
                               const struct cl_io_slice *ios)
 {
@@ -1057,7 +1068,7 @@ static int vvp_io_commit_write(const struct lu_env *env,
 static const struct cl_io_operations vvp_io_ops = {
         .op = {
                 [CIT_READ] = {
-                        .cio_fini      = vvp_io_fini,
+                        .cio_fini      = vvp_io_read_fini,
                         .cio_lock      = vvp_io_read_lock,
                         .cio_start     = vvp_io_read_start,
                         .cio_advance   = ccc_io_advance
@@ -1098,8 +1109,9 @@ static const struct cl_io_operations vvp_io_ops = {
 int vvp_io_init(const struct lu_env *env, struct cl_object *obj,
                 struct cl_io *io)
 {
-        struct vvp_io      *vio   = vvp_env_io(env);
-        struct ccc_io      *cio   = ccc_env_io(env);
+       struct vvp_io      *vio   = vvp_env_io(env);
+       struct ccc_io      *cio   = ccc_env_io(env);
+       struct inode       *inode = ccc_object_inode(obj);
         int                 result;
 
         CLOBINVRNT(env, obj, ccc_object_invariant(obj));
@@ -1108,10 +1120,10 @@ int vvp_io_init(const struct lu_env *env, struct cl_object *obj,
         CL_IO_SLICE_CLEAN(cio, cui_cl);
         cl_io_slice_add(io, &cio->cui_cl, obj, &vvp_io_ops);
         vio->cui_ra_window_set = 0;
-        result = 0;
-        if (io->ci_type == CIT_READ || io->ci_type == CIT_WRITE) {
-                size_t count;
-               struct ll_inode_info *lli = ll_i2info(ccc_object_inode(obj));
+       result = 0;
+       if (io->ci_type == CIT_READ || io->ci_type == CIT_WRITE) {
+               size_t count;
+               struct ll_inode_info *lli = ll_i2info(inode);
 
                 count = io->u.ci_rw.crw_count;
                 /* "If nbyte is 0, read() will return 0 and have no other
@@ -1129,11 +1141,18 @@ int vvp_io_init(const struct lu_env *env, struct cl_object *obj,
                 * jobs.
                 */
                lustre_get_jobid(lli->lli_jobid);
-        } else if (io->ci_type == CIT_SETATTR) {
-                if (!cl_io_is_trunc(io))
-                        io->ci_lockreq = CILR_MANDATORY;
-        }
-        RETURN(result);
+       } else if (io->ci_type == CIT_SETATTR) {
+               if (!cl_io_is_trunc(io))
+                       io->ci_lockreq = CILR_MANDATORY;
+       }
+
+       /* Enqueue layout lock and get layout version. We need to do this
+        * even for operations requiring to open file, such as read and write,
+        * because it might not grant layout lock in IT_OPEN. */
+       if (result == 0 && !io->ci_ignore_layout)
+               result = ll_layout_refresh(inode, &cio->cui_layout_gen);
+
+       RETURN(result);
 }
 
 static struct vvp_io *cl2vvp_io(const struct lu_env *env,

diff --git a/lustre/obdclass/cl_io.c b/lustre/obdclass/cl_io.c
index 2e8619a..6619073 100644 (file)
--- a/lustre/obdclass/cl_io.c
+++ b/lustre/obdclass/cl_io.c
@@ -104,8 +104,8 @@ static int cl_io_invariant(const struct cl_io *io)
  */
 void cl_io_fini(const struct lu_env *env, struct cl_io *io)
 {
-        struct cl_io_slice    *slice;
-        struct cl_thread_info *info;
+       struct cl_io_slice    *slice;
+       struct cl_thread_info *info;
 
         LINVRNT(cl_io_type_is_valid(io->ci_type));
         LINVRNT(cl_io_invariant(io));
@@ -128,7 +128,24 @@ void cl_io_fini(const struct lu_env *env, struct cl_io *io)
         info = cl_env_info(env);
         if (info->clt_current_io == io)
                 info->clt_current_io = NULL;
-        EXIT;
+
+       /* sanity check for layout change */
+       switch(io->ci_type) {
+       case CIT_READ:
+       case CIT_WRITE:
+       case CIT_FAULT:
+       case CIT_FSYNC:
+               LASSERT(!io->ci_need_restart);
+               break;
+       case CIT_MISC:
+               /* Check ignore layout change conf */
+               LASSERT(ergo(io->ci_ignore_layout, !io->ci_need_restart));
+       case CIT_SETATTR:
+               break;
+       default:
+               LBUG();
+       }
+       EXIT;
 }
 EXPORT_SYMBOL(cl_io_fini);
 

diff --git a/lustre/obdclass/cl_lock.c b/lustre/obdclass/cl_lock.c
index fdfeed8..d5efa9b 100644 (file)
--- a/lustre/obdclass/cl_lock.c
+++ b/lustre/obdclass/cl_lock.c
@@ -1982,6 +1982,7 @@ int cl_lock_discard_pages(const struct lu_env *env, struct cl_lock *lock)
         ENTRY;
 
         io->ci_obj = cl_object_top(descr->cld_obj);
+       io->ci_ignore_layout = 1;
         result = cl_io_init(env, io, CIT_MISC, io->ci_obj);
         if (result != 0)
                 GOTO(out, result);

diff --git a/lustre/obdclass/cl_page.c b/lustre/obdclass/cl_page.c
index e2f6f93..1266c79 100644 (file)
--- a/lustre/obdclass/cl_page.c
+++ b/lustre/obdclass/cl_page.c
@@ -1513,6 +1513,7 @@ int cl_pages_prune(const struct lu_env *env, struct cl_object *clobj)
          * function, we just make cl_page_list functions happy. -jay
          */
         io->ci_obj = obj;
+       io->ci_ignore_layout = 1;
         result = cl_io_init(env, io, CIT_MISC, obj);
         if (result != 0) {
                 cl_io_fini(env, io);

diff --git a/lustre/obdecho/echo_client.c b/lustre/obdecho/echo_client.c
index 0145f3f..d52f9ea 100644 (file)
--- a/lustre/obdecho/echo_client.c
+++ b/lustre/obdecho/echo_client.c
@@ -1180,6 +1180,7 @@ static int cl_echo_enqueue(struct echo_object *eco, obd_off start, obd_off end,
         info = echo_env_info(env);
         io = &info->eti_io;
 
+       io->ci_ignore_layout = 1;
         result = cl_io_init(env, io, CIT_MISC, echo_obj2cl(eco));
         if (result < 0)
                 GOTO(out, result);
@@ -1289,6 +1290,8 @@ static int cl_echo_object_brw(struct echo_object *eco, int rw, obd_off offset,
         queue   = &info->eti_queue;
 
         cl_2queue_init(queue);
+
+       io->ci_ignore_layout = 1;
         rc = cl_io_init(env, io, CIT_MISC, obj);
         if (rc < 0)
                 GOTO(out, rc);

diff --git a/lustre/osc/osc_lock.c b/lustre/osc/osc_lock.c
index c673e9f..d5555d1 100644 (file)
--- a/lustre/osc/osc_lock.c
+++ b/lustre/osc/osc_lock.c
@@ -1469,6 +1469,7 @@ static int osc_lock_has_pages(struct osc_lock *olck)
         cfs_mutex_lock(&oob->oo_debug_mutex);
 
         io->ci_obj = cl_object_top(obj);
+       io->ci_ignore_layout = 1;
         cl_io_init(env, io, CIT_MISC, io->ci_obj);
         do {
                 result = cl_page_gang_lookup(env, obj, io,