LU-17048 mdd: protect layout change in MDD layer

We need to detect changes to the LOD layout in between transaction
declaration and when the objects are locked during transaction
execution. Otherwise, if another thread has modified the layout
of an object used by the transaction then the declaration may
be incorrect.

This patch save objects' layout generation in transaction delaration
phase, and check whether they have been changed by others in the
transaction execution phase, if that's the case, the transaction will
be retried for several times.

Fixes: b7bd4e3422 ("LU-14621 mdd: fix lock-tx order in mdd_xattr_merge()")
Signed-off-by: Bobi Jam <bobijam@whamcloud.com>
Change-Id: I25fe03c6e8fc4eebccc039e62dfc88db1179cb26
Reviewed-on: https://review.whamcloud.com/c/fs/lustre-release/+/52146
Tested-by: jenkins <devops@whamcloud.com>
Tested-by: Maloo <maloo@whamcloud.com>
Reviewed-by: Alex Zhuravlev <bzzz@whamcloud.com>
Reviewed-by: Andreas Dilger <adilger@whamcloud.com>
Reviewed-by: Oleg Drokin <green@whamcloud.com>

diff --git a/lustre/lod/lod_dev.c b/lustre/lod/lod_dev.c
index b70245f..d5fdefc 100644 (file)
--- a/lustre/lod/lod_dev.c
+++ b/lustre/lod/lod_dev.c
@@ -1682,6 +1682,7 @@ out:
 static struct thandle *lod_trans_create(const struct lu_env *env,
                                        struct dt_device *dt)
 {
+       struct lod_thread_info *info = lod_env_info(env);
        struct thandle *th;
 
        th = top_trans_create(env, dt2lod_dev(dt)->lod_child);
@@ -1690,6 +1691,9 @@ static struct thandle *lod_trans_create(const struct lu_env *env,
 
        th->th_dev = dt;
 
+       /* initialize some lod_thread_info members */
+       info->lti_obj_count = 0;
+
        return th;
 }
 

diff --git a/lustre/lod/lod_internal.h b/lustre/lod/lod_internal.h
index bd14482..486169f 100644 (file)
--- a/lustre/lod/lod_internal.h
+++ b/lustre/lod/lod_internal.h
@@ -407,6 +407,7 @@ struct lod_it {
        struct dt_it            *lit_it;  /* iterator from the layer below */
 };
 
+#define LOD_OBJS_INTRANS 4
 struct lod_thread_info {
        /* per-thread buffer for LOV EA, may be vmalloc'd */
        void                           *lti_ea_store;
@@ -440,8 +441,70 @@ struct lod_thread_info {
        struct lod_avoid_guide          lti_avoid;
        union lmv_mds_md                lti_lmv;
        struct dt_allocation_hint       lti_ah;
+       int lti_obj_count;
+       struct lod_object *lti_obj[LOD_OBJS_INTRANS];
+       __u32 lti_gen[LOD_OBJS_INTRANS];
 };
 
+/**
+ * \retval     0 object's layout hasn't changed in the transaction
+ * \retval     > 0 object's layout has changed
+ * \retval     -ENOENT object's layout gen hasn't saved in transatoin
+ *             declaration
+ */
+static inline int
+lod_check_layout_gen_intrans(struct lod_thread_info *info,
+                            struct lod_object *lo)
+{
+       int i;
+       int rc = -ENOENT;
+
+       for (i = 0; i < info->lti_obj_count; i++) {
+               if (info->lti_obj[i] != lo)
+                       continue;
+
+               if (info->lti_gen[i] == lo->ldo_layout_gen)
+                       rc = 0;
+               else
+                       rc = i + 1;
+               break;
+       }
+
+       return rc;
+}
+
+static inline int
+lod_save_layout_gen_intrans(struct lod_thread_info *info, struct lod_object *lo)
+{
+       struct lu_object *luo = &lo->ldo_obj.do_lu;
+       int rc;
+
+       rc = lod_check_layout_gen_intrans(info, lo);
+       if (rc == 0)
+               return 0;
+       if (rc > 0) {
+               CDEBUG(D_LAYOUT,
+                      "%s: obj %p gen changed from %d to %d in trans declaration\n",
+                      luo->lo_dev->ld_obd->obd_name, lo, info->lti_gen[rc - 1],
+                      lo->ldo_layout_gen);
+               return -EINVAL;
+       }
+
+       if (unlikely(info->lti_obj_count == LOD_OBJS_INTRANS)) {
+               CERROR("%s: "DFID
+                      " save too many lod_object (%d) in one transaction, use bigger LOD_OBJS_INTRANS: rc = %d\n",
+                      luo->lo_dev->ld_obd->obd_name,
+                      PFID(lu_object_fid(luo)), LOD_OBJS_INTRANS, -E2BIG);
+               return -E2BIG;
+       }
+
+       info->lti_obj[info->lti_obj_count] = lo;
+       info->lti_gen[info->lti_obj_count] = lo->ldo_layout_gen;
+       info->lti_obj_count++;
+
+       return 0;
+}
+
 extern const struct lu_device_operations lod_lu_ops;
 
 static inline int lu_device_is_lod(struct lu_device *d)

diff --git a/lustre/lod/lod_object.c b/lustre/lod/lod_object.c
index 3ce2118..6dca2cb 100644 (file)
--- a/lustre/lod/lod_object.c
+++ b/lustre/lod/lod_object.c
@@ -3763,8 +3763,10 @@ static int lod_declare_xattr_set(const struct lu_env *env,
                                 const char *name, int fl,
                                 struct thandle *th)
 {
+       struct lod_thread_info *info = lod_env_info(env);
        struct dt_object *next = dt_object_child(dt);
-       struct lu_attr   *attr = &lod_env_info(env)->lti_attr;
+       struct lu_attr   *attr = &info->lti_attr;
+       struct lod_object *lo = lod_dt_obj(dt);
        __u32             mode;
        int               rc;
        ENTRY;
@@ -3825,6 +3827,11 @@ static int lod_declare_xattr_set(const struct lu_env *env,
                rc = lod_sub_declare_xattr_set(env, next, buf, name, fl, th);
        }
 
+       if (rc == 0 &&
+           (strcmp(name, XATTR_NAME_LOV) == 0 ||
+            strcmp(name, XATTR_LUSTRE_LOV) == 0 || allowed_lustre_lov(name)))
+               rc = lod_save_layout_gen_intrans(info, lo);
+
        RETURN(rc);
 }
 
@@ -5054,8 +5061,9 @@ static int lod_xattr_set(const struct lu_env *env,
                         struct dt_object *dt, const struct lu_buf *buf,
                         const char *name, int fl, struct thandle *th)
 {
+       struct lod_thread_info *info = lod_env_info(env);
        struct dt_object *next = dt_object_child(dt);
-       struct lu_attr *layout_attr = &lod_env_info(env)->lti_layout_attr;
+       struct lu_attr *layout_attr = &info->lti_layout_attr;
        struct lod_object *lo = lod_dt_obj(dt);
        struct lod_obj_stripe_cb_data data = { {0} };
        int rc = 0;
@@ -5093,6 +5101,17 @@ static int lod_xattr_set(const struct lu_env *env,
                   (strcmp(name, XATTR_NAME_LOV) == 0 ||
                    strcmp(name, XATTR_LUSTRE_LOV) == 0 ||
                    allowed_lustre_lov(name))) {
+               /* layout has been changed by others in the transaction */
+               rc = lod_check_layout_gen_intrans(info, lo);
+               if (rc > 0) {
+                       CDEBUG(D_LAYOUT,
+                              "%s: obj "DFID" gen changed from %d to %d in transaction, retry the transaction\n",
+                              dt->do_lu.lo_dev->ld_obd->obd_name,
+                              PFID(lu_object_fid(&dt->do_lu)),
+                              info->lti_gen[rc - 1], lo->ldo_layout_gen);
+                       RETURN(-EAGAIN);
+               }
+
                /* in case of lov EA swap, just set it
                 * if not, it is a replay so check striping match what we
                 * already have during req replay, declare_xattr_set()
@@ -5142,7 +5161,7 @@ static int lod_xattr_set(const struct lu_env *env,
                } else {
                        /*
                         * When 'name' is XATTR_LUSTRE_LOV or XATTR_NAME_LOV,
-                        * it's going to create create file with specified
+                        * it's going to create file with specified
                         * component(s), the striping must have not being
                         * cached in this case;
                         *
@@ -5150,9 +5169,10 @@ static int lod_xattr_set(const struct lu_env *env,
                         * an existing file, the striping must have been cached
                         * in this case.
                         */
-                       LASSERT(equi(!strcmp(name, XATTR_LUSTRE_LOV) ||
-                                    !strcmp(name, XATTR_NAME_LOV),
-                               !lod_dt_obj(dt)->ldo_comp_cached));
+                       if (!(fl & LU_XATTR_MERGE))
+                               LASSERT(equi(!strcmp(name, XATTR_LUSTRE_LOV) ||
+                                            !strcmp(name, XATTR_NAME_LOV),
+                                       !lod_dt_obj(dt)->ldo_comp_cached));
 
                        rc = lod_striped_create(env, dt, NULL, NULL, th);
                        if (rc)
@@ -7455,8 +7475,13 @@ static int lod_declare_update_extents(const struct lu_env *env,
        ENTRY;
 
        /* This makes us work on the components of the chosen mirror */
-       start_index = lo->ldo_mirrors[pick].lme_start;
-       max_comp = lo->ldo_mirrors[pick].lme_end + 1;
+       if (lo->ldo_mirrors) {
+               start_index = lo->ldo_mirrors[pick].lme_start;
+               max_comp = lo->ldo_mirrors[pick].lme_end + 1;
+       } else {
+               start_index = 0;
+               max_comp = lo->ldo_comp_cnt;
+       }
        if (lo->ldo_flr_state == LCM_FL_NONE)
                LASSERT(start_index == 0 && max_comp == lo->ldo_comp_cnt);
 
@@ -7485,12 +7510,14 @@ static int lod_declare_update_extents(const struct lu_env *env,
        /* We may have added or removed components.  If so, we must update the
         * start & ends of all the mirrors after the current one, and the end
         * of the current mirror. */
-       change = max_comp - 1 - lo->ldo_mirrors[pick].lme_end;
-       if (change) {
-               lo->ldo_mirrors[pick].lme_end += change;
-               for (i = pick + 1; i < lo->ldo_mirror_count; i++) {
-                       lo->ldo_mirrors[i].lme_start += change;
-                       lo->ldo_mirrors[i].lme_end += change;
+       if (lo->ldo_mirrors) {
+               change = max_comp - 1 - lo->ldo_mirrors[pick].lme_end;
+               if (change) {
+                       lo->ldo_mirrors[pick].lme_end += change;
+                       for (i = pick + 1; i < lo->ldo_mirror_count; i++) {
+                               lo->ldo_mirrors[i].lme_start += change;
+                               lo->ldo_mirrors[i].lme_end += change;
+                       }
                }
        }
 
@@ -9252,6 +9279,9 @@ static int lod_declare_layout_change(const struct lu_env *env,
                rc = -ENOTSUPP;
                break;
        }
+       if (rc == 0)
+               rc = lod_save_layout_gen_intrans(info, lo);
+
 out:
        RETURN(rc);
 }
@@ -9262,8 +9292,9 @@ out:
 static int lod_layout_change(const struct lu_env *env, struct dt_object *dt,
                             struct md_layout_change *mlc, struct thandle *th)
 {
+       struct lod_thread_info *info = lod_env_info(env);
        struct lu_attr *attr = &lod_env_info(env)->lti_attr;
-       struct lu_attr *layout_attr = &lod_env_info(env)->lti_layout_attr;
+       struct lu_attr *layout_attr = &info->lti_layout_attr;
        struct lod_object *lo = lod_dt_obj(dt);
        int rc;
 
@@ -9275,6 +9306,16 @@ static int lod_layout_change(const struct lu_env *env, struct dt_object *dt,
                RETURN(rc);
        }
 
+       rc = lod_check_layout_gen_intrans(info, lo);
+       if (rc > 0) {
+               CDEBUG(D_LAYOUT,
+                      "%s: obj "DFID" gen changed from %d to %d in transaction, retry the transaction \n",
+                      dt->do_lu.lo_dev->ld_obd->obd_name,
+                      PFID(lu_object_fid(&dt->do_lu)),
+                      info->lti_gen[rc - 1], lo->ldo_layout_gen);
+               RETURN(-EAGAIN);
+       }
+
        rc = lod_striped_create(env, dt, attr, NULL, th);
        if (!rc && layout_attr->la_valid & LA_LAYOUT_VERSION) {
                layout_attr->la_layout_version |= lo->ldo_layout_gen;

diff --git a/lustre/mdd/mdd_dir.c b/lustre/mdd/mdd_dir.c
index f893aeb..8d90676 100644 (file)
--- a/lustre/mdd/mdd_dir.c
+++ b/lustre/mdd/mdd_dir.c
@@ -4499,6 +4499,7 @@ static int mdd_migrate_object(const struct lu_env *env,
        struct mdd_xattrs xattrs;
        struct lmv_mds_md_v1 *lmv;
        struct thandle *handle;
+       int retried = 0;
        int rc;
 
        ENTRY;
@@ -4508,6 +4509,7 @@ static int mdd_migrate_object(const struct lu_env *env,
               PFID(mdd_object_fid(sobj)), PFID(mdd_object_fid(tpobj)),
               PFID(mdd_object_fid(tobj)));
 
+retry:
        rc = mdd_la_get(env, sobj, attr);
        if (rc)
                RETURN(rc);
@@ -4635,9 +4637,6 @@ static int mdd_migrate_object(const struct lu_env *env,
                                    spobj, spattr, mdd_object_fid(sobj),
                                    tpobj, tpattr, tname, sname,
                                    handle);
-       if (rc)
-               GOTO(stop, rc);
-       EXIT;
 
 stop:
        rc = mdd_trans_stop(env, mdd, rc, handle);
@@ -4645,7 +4644,14 @@ out:
        mdd_xattrs_fini(&xattrs);
        lu_buf_free(&sbuf);
 
-       return rc;
+       /**
+        * -EAGAIN means transaction execution phase detect the layout
+        * has been changed by others.
+        */
+       if (rc == -EAGAIN && retried++ < MAX_TRANS_RETRIED)
+               GOTO(retry, retried);
+
+       RETURN(rc);
 }
 
 /**

diff --git a/lustre/mdd/mdd_internal.h b/lustre/mdd/mdd_internal.h
index 7ba9f35..e8e00f3 100644 (file)
--- a/lustre/mdd/mdd_internal.h
+++ b/lustre/mdd/mdd_internal.h
@@ -894,4 +894,45 @@ bool mdd_changelog_is_space_safe(const struct lu_env *env,
                                 struct llog_handle *lgh,
                                 bool estimate);
 
+#define MAX_TRANS_RETRIED 20
+
+static inline int
+mdd_write_lock_two_objects(const struct lu_env *env,
+                          struct mdd_object *obj1,
+                          struct mdd_object *obj2)
+{
+       int order;
+
+       /* there shouldn't be callers with obj1 == NULL */
+       LASSERT(obj1 != NULL);
+       if (!obj2)
+               goto out_lock1;
+
+       order = lu_fid_cmp(mdd_object_fid(obj1), mdd_object_fid(obj2));
+       if (unlikely(order == 0))
+               return -EPERM;
+
+       if (order > 0) {
+               mdd_write_lock(env, obj1, DT_TGT_CHILD);
+               mdd_write_lock(env, obj2, DT_TGT_CHILD);
+       } else {
+               mdd_write_lock(env, obj2, DT_TGT_CHILD);
+out_lock1:
+               mdd_write_lock(env, obj1, DT_TGT_CHILD);
+       }
+
+       return 0;
+}
+
+static inline void
+mdd_write_unlock_two_objects(const struct lu_env *env,
+                            struct mdd_object *obj1,
+                            struct mdd_object *obj2)
+{
+       mdd_write_unlock(env, obj1);
+       if (obj2)
+               mdd_write_unlock(env, obj2);
+}
+
+
 #endif

diff --git a/lustre/mdd/mdd_object.c b/lustre/mdd/mdd_object.c
index 9fe0dcd..07db69a 100644 (file)
--- a/lustre/mdd/mdd_object.c
+++ b/lustre/mdd/mdd_object.c
@@ -1614,14 +1614,16 @@ static int mdd_xattr_merge(const struct lu_env *env, struct md_object *md_obj,
        struct mdd_device *mdd = mdo2mdd(md_obj);
        struct mdd_object *obj = md2mdd_obj(md_obj);
        struct mdd_object *vic = md2mdd_obj(md_vic);
-       struct lu_buf *buf = &mdd_env_info(env)->mdi_buf[0];
-       struct lu_buf *buf_vic = &mdd_env_info(env)->mdi_buf[1];
+       struct lu_buf *buf;
+       struct lu_buf *buf_vic;
        struct lov_mds_md *lmm;
        struct thandle *handle;
        struct lu_attr *cattr = MDD_ENV_VAR(env, cattr);
        struct lu_attr *tattr = MDD_ENV_VAR(env, tattr);
        bool is_same_projid;
-       int rc, lock_order;
+       int rc;
+       int retried = 0;
+
        ENTRY;
 
        rc = mdd_la_get(env, obj, cattr);
@@ -1634,14 +1636,16 @@ static int mdd_xattr_merge(const struct lu_env *env, struct md_object *md_obj,
 
        is_same_projid = cattr->la_projid == tattr->la_projid;
 
-       lock_order = lu_fid_cmp(mdd_object_fid(obj), mdd_object_fid(vic));
-       if (lock_order == 0) /* same fid */
+       if (lu_fid_cmp(mdd_object_fid(obj), mdd_object_fid(vic)) == 0)
                RETURN(-EPERM);
-
+retry:
        handle = mdd_trans_create(env, mdd);
        if (IS_ERR(handle))
                RETURN(PTR_ERR(handle));
 
+       buf = &mdd_env_info(env)->mdi_buf[0];
+       buf_vic = &mdd_env_info(env)->mdi_buf[1];
+
        if (!is_same_projid) {
                rc = mdd_declare_attr_set(env, mdd, vic, cattr, handle);
                if (rc)
@@ -1682,13 +1686,9 @@ static int mdd_xattr_merge(const struct lu_env *env, struct md_object *md_obj,
        if (rc != 0)
                GOTO(stop, rc);
 
-       if (lock_order > 0) {
-               mdd_write_lock(env, obj, DT_TGT_CHILD);
-               mdd_write_lock(env, vic, DT_TGT_CHILD);
-       } else {
-               mdd_write_lock(env, vic, DT_TGT_CHILD);
-               mdd_write_lock(env, obj, DT_TGT_CHILD);
-       }
+       rc = mdd_write_lock_two_objects(env, obj, vic);
+       if (rc)
+               GOTO(stop, rc);
 
        if (!is_same_projid) {
                cattr->la_valid = LA_PROJID;
@@ -1710,7 +1710,6 @@ static int mdd_xattr_merge(const struct lu_env *env, struct md_object *md_obj,
                                       NULL);
        (void)mdd_changelog_data_store(env, mdd, CL_LAYOUT, 0, vic, handle,
                                       NULL);
-       EXIT;
 
 out_restore:
        if (rc) {
@@ -1723,17 +1722,22 @@ out_restore:
        }
 
 out:
-       mdd_write_unlock(env, obj);
-       mdd_write_unlock(env, vic);
+       mdd_write_unlock_two_objects(env, obj, vic);
 stop:
        mdd_trans_stop(env, mdd, rc, handle);
        lu_buf_free(buf);
        lu_buf_free(buf_vic);
 
+       /**
+        * -EAGAIN means transaction execution phase detect the layout
+        * has been changed by others.
+        */
        if (!rc)
                (void) mdd_object_pfid_replace(env, obj);
+       else if (rc == -EAGAIN && retried++ < MAX_TRANS_RETRIED)
+               GOTO(retry, retried);
 
-       return rc;
+       RETURN(rc);
 }
 
 /**
@@ -1868,13 +1872,14 @@ static int mdd_xattr_split(const struct lu_env *env, struct md_object *md_obj,
        struct mdd_device *mdd = mdo2mdd(md_obj);
        struct mdd_object *obj = md2mdd_obj(md_obj);
        struct mdd_object *vic = NULL;
-       struct lu_buf *buf = &mdd_env_info(env)->mdi_buf[0];
-       struct lu_buf *buf_save = &mdd_env_info(env)->mdi_buf[1];
-       struct lu_buf *buf_vic = &mdd_env_info(env)->mdi_buf[2];
+       struct lu_buf *buf;
+       struct lu_buf *buf_save;
+       struct lu_buf *buf_vic;
        struct lov_comp_md_v1 *lcm;
        struct thandle *handle;
        int rc;
        bool dom_stripe = false;
+       int retried = 0;
 
        ENTRY;
 
@@ -1885,6 +1890,11 @@ static int mdd_xattr_split(const struct lu_env *env, struct md_object *md_obj,
        if (mrd->mrd_obj)
                vic = md2mdd_obj(mrd->mrd_obj);
 
+retry:
+       buf = &mdd_env_info(env)->mdi_buf[0];
+       buf_save = &mdd_env_info(env)->mdi_buf[1];
+       buf_vic = &mdd_env_info(env)->mdi_buf[2];
+
        handle = mdd_trans_create(env, mdd);
        if (IS_ERR(handle))
                RETURN(PTR_ERR(handle));
@@ -1956,22 +1966,9 @@ static int mdd_xattr_split(const struct lu_env *env, struct md_object *md_obj,
        if (rc)
                GOTO(stop, rc);
 
-       if (vic) {
-               /* don't use the same file to save the splitted mirror */
-               rc = lu_fid_cmp(mdd_object_fid(obj), mdd_object_fid(vic));
-               if (rc == 0)
-                       GOTO(stop, rc = -EPERM);
-
-               if (rc > 0) {
-                       mdd_write_lock(env, obj, DT_TGT_CHILD);
-                       mdd_write_lock(env, vic, DT_TGT_CHILD);
-               } else {
-                       mdd_write_lock(env, vic, DT_TGT_CHILD);
-                       mdd_write_lock(env, obj, DT_TGT_CHILD);
-               }
-       } else {
-               mdd_write_lock(env, obj, DT_TGT_CHILD);
-       }
+       rc = mdd_write_lock_two_objects(env, obj, vic);
+       if (rc)
+               GOTO(stop, rc);
 
        /* set obj's layout in @buf */
        rc = mdo_xattr_set(env, obj, buf, XATTR_NAME_LOV, LU_XATTR_SPLIT,
@@ -2018,9 +2015,7 @@ out_restore:
        }
 
 unlock:
-       mdd_write_unlock(env, obj);
-       if (vic)
-               mdd_write_unlock(env, vic);
+       mdd_write_unlock_two_objects(env, obj, vic);
 stop:
        rc = mdd_trans_stop(env, mdd, rc, handle);
 
@@ -2032,10 +2027,16 @@ stop:
        lu_buf_free(buf);
        lu_buf_free(buf_vic);
 
+       /**
+        * -EAGAIN means transaction execution phase detect the layout
+        * has been changed by others.
+        */
        if (!rc)
                (void) mdd_object_pfid_replace(env, obj);
+       else if (rc == -EAGAIN && retried++ < MAX_TRANS_RETRIED)
+               GOTO(retry, retried);
 
-       return rc;
+       RETURN(rc);
 }
 
 static int mdd_layout_merge_allowed(const struct lu_env *env,
@@ -2068,9 +2069,12 @@ static int mdd_xattr_set(const struct lu_env *env, struct md_object *obj,
        struct thandle *handle;
        enum changelog_rec_type  cl_type;
        enum changelog_rec_flags clf_flags = 0;
+       int retried = 0;
        int rc;
+
        ENTRY;
 
+retry:
        rc = mdd_la_get(env, mdd_obj, attr);
        if (rc)
                RETURN(rc);
@@ -2138,27 +2142,33 @@ static int mdd_xattr_set(const struct lu_env *env, struct md_object *obj,
 
        if (strcmp(XATTR_NAME_HSM, name) == 0) {
                rc = mdd_hsm_update_locked(env, obj, buf, handle, &clf_flags);
-               if (rc) {
-                       mdd_write_unlock(env, mdd_obj);
-                       GOTO(stop, rc);
-               }
+               if (rc)
+                       GOTO(out_unlock, rc);
        }
 
        rc = mdo_xattr_set(env, mdd_obj, buf, name, fl, handle);
+out_unlock:
        mdd_write_unlock(env, mdd_obj);
        if (rc)
                GOTO(stop, rc);
 
        cl_type = mdd_xattr_changelog_type(env, mdd, name);
-       if (cl_type < 0)
-               GOTO(stop, rc = 0);
-
-       rc = mdd_changelog_data_store_xattr(env, mdd, cl_type, clf_flags,
-                                           mdd_obj, name, handle);
+       if (cl_type != CL_NONE)
+               rc = mdd_changelog_data_store_xattr(env, mdd, cl_type,
+                                                   clf_flags, mdd_obj, name,
+                                                   handle);
 
-       EXIT;
 stop:
-       return mdd_trans_stop(env, mdd, rc, handle);
+       rc = mdd_trans_stop(env, mdd, rc, handle);
+
+       /**
+        * -EAGAIN means transaction execution phase detect the layout
+        * has been changed by others.
+        */
+       if (rc == -EAGAIN && retried++ < MAX_TRANS_RETRIED)
+               GOTO(retry, retried);
+
+       RETURN(rc);
 }
 
 static int mdd_declare_xattr_del(const struct lu_env *env,
@@ -2530,10 +2540,10 @@ static int mdd_swap_layouts(const struct lu_env *env, struct md_object *obj1,
        struct lu_attr *snd_la = MDD_ENV_VAR(env, tattr);
        struct mdd_device *mdd = mdo2mdd(obj1);
        struct lov_mds_md *fst_lmm, *snd_lmm;
-       struct lu_buf *fst_buf = &info->mdi_buf[0];
-       struct lu_buf *snd_buf = &info->mdi_buf[1];
-       struct lu_buf *fst_hsm_buf = &info->mdi_buf[2];
-       struct lu_buf *snd_hsm_buf = &info->mdi_buf[3];
+       struct lu_buf *fst_buf;
+       struct lu_buf *snd_buf;
+       struct lu_buf *fst_hsm_buf;
+       struct lu_buf *snd_hsm_buf;
        struct ost_id *saved_oi = NULL;
        struct thandle *handle;
        struct mdd_object *dom_o = NULL, *vlt_o = NULL;
@@ -2541,10 +2551,17 @@ static int mdd_swap_layouts(const struct lu_env *env, struct md_object *obj1,
        __u32 fst_gen, snd_gen, saved_gen;
        int fst_fl;
        int rc, rc2;
+       int retried = 0;
 
        ENTRY;
 
        BUILD_BUG_ON(ARRAY_SIZE(info->mdi_buf) < 4);
+retry:
+       fst_buf = &info->mdi_buf[0];
+       snd_buf = &info->mdi_buf[1];
+       fst_hsm_buf = &info->mdi_buf[2];
+       snd_hsm_buf = &info->mdi_buf[3];
+
        memset(info->mdi_buf, 0, sizeof(info->mdi_buf));
 
        /* we have to sort the 2 obj, so locking will always
@@ -2744,9 +2761,9 @@ static int mdd_swap_layouts(const struct lu_env *env, struct md_object *obj1,
        if (rc != 0)
                GOTO(stop, rc);
 
-       /* objects are already sorted */
-       mdd_write_lock(env, fst_o, DT_TGT_CHILD);
-       mdd_write_lock(env, snd_o, DT_TGT_CHILD);
+       rc = mdd_write_lock_two_objects(env, fst_o, snd_o);
+       if (rc < 0)
+               GOTO(stop, rc);
 
        if (!mdd_object_exists(fst_o))
                GOTO(unlock, rc = -ENOENT);
@@ -2844,8 +2861,7 @@ out_restore:
        }
 
 unlock:
-       mdd_write_unlock(env, snd_o);
-       mdd_write_unlock(env, fst_o);
+       mdd_write_unlock_two_objects(env, fst_o, snd_o);
 
 stop:
        rc = mdd_trans_stop(env, mdd, rc, handle);
@@ -2865,8 +2881,15 @@ stop:
        if (!rc) {
                (void) mdd_object_pfid_replace(env, fst_o);
                (void) mdd_object_pfid_replace(env, snd_o);
+       } else if (rc == -EAGAIN && retried++ < MAX_TRANS_RETRIED) {
+               /**
+                * -EAGAIN means transaction execution phase detect the layout
+                * has been changed by others.
+                */
+               GOTO(retry, retried);
        }
-       return rc;
+
+       RETURN(rc);
 }
 
 static int mdd_declare_layout_change(const struct lu_env *env,
@@ -3009,13 +3032,9 @@ mdd_layout_update_rdonly(const struct lu_env *env, struct mdd_object *obj,
                GOTO(out, rc);
 
        rc = mdd_changelog_data_store(env, mdd, CL_FLRW, 0, obj, handle, NULL);
-       if (rc)
-               GOTO(out, rc);
-
-       EXIT;
 
 out:
-       return rc;
+       RETURN(rc);
 }
 
 /**
@@ -3098,13 +3117,9 @@ mdd_layout_update_write_pending(const struct lu_env *env,
                                   fl, handle);
        }
        mdd_write_unlock(env, obj);
-       if (rc)
-               GOTO(out, rc);
-
-       EXIT;
 
 out:
-       return rc;
+       RETURN(rc);
 }
 
 /**
@@ -3180,7 +3195,9 @@ mdd_object_update_sync_pending(const struct lu_env *env, struct mdd_object *obj,
        /* it needs a sync tx to make FLR to work properly */
        handle->th_sync = 1;
 
+       mdd_write_lock(env, obj, DT_TGT_CHILD);
        rc = mdo_layout_change(env, obj, mlc, handle);
+       mdd_write_unlock(env, obj);
        if (rc)
                GOTO(out, rc);
 
@@ -3193,11 +3210,8 @@ mdd_object_update_sync_pending(const struct lu_env *env, struct mdd_object *obj,
 
        rc = mdd_changelog_data_store(env, mdd, CL_RESYNC, 0, obj, handle,
                                      NULL);
-       if (rc)
-               GOTO(out, rc);
-       EXIT;
 out:
-       return rc;
+       RETURN(rc);
 }
 
 /**
@@ -3261,6 +3275,7 @@ mdd_layout_change(const struct lu_env *env, struct md_object *o,
        struct lov_comp_md_v1   *lcm;
        struct thandle          *handle;
        int flr_state;
+       int retried = 0;
        int rc;
 
        ENTRY;
@@ -3296,6 +3311,7 @@ mdd_layout_change(const struct lu_env *env, struct md_object *o,
                RETURN(-ENOTSUPP);
        }
 
+retry:
        handle = mdd_trans_create(env, mdd);
        if (IS_ERR(handle))
                RETURN(PTR_ERR(handle));
@@ -3337,6 +3353,14 @@ mdd_layout_change(const struct lu_env *env, struct md_object *o,
 out:
        mdd_trans_stop(env, mdd, rc, handle);
        lu_buf_free(buf);
+
+       /**
+        * -EAGAIN means transaction execution phase detect the layout
+        * has been changed by others.
+        */
+       if (rc == -EAGAIN && retried++ < MAX_TRANS_RETRIED)
+               GOTO(retry, retried);
+
        return rc;
 }