Whamcloud - gitweb
LU-8592 mdt: hold mdt_device::mdt_md_root until service stop
[fs/lustre-release.git] / lustre / mdt / mdt_open.c
index fc67526..9bd929f 100644 (file)
  *
  * You should have received a copy of the GNU General Public License
  * version 2 along with this program; If not, see
- * http://www.sun.com/software/products/lustre/docs/GPLv2.pdf
- *
- * Please contact Sun Microsystems, Inc., 4150 Network Circle, Santa Clara,
- * CA 95054 USA or visit www.sun.com if you need additional information or
- * have any questions.
+ * http://www.gnu.org/licenses/gpl-2.0.html
  *
  * GPL HEADER END
  */
@@ -27,7 +23,7 @@
  * Copyright (c) 2007, 2010, Oracle and/or its affiliates. All rights reserved.
  * Use is subject to license terms.
  *
- * Copyright (c) 2011, 2013, Intel Corporation.
+ * Copyright (c) 2011, 2015, Intel Corporation.
  */
 /*
  * This file is part of Lustre, http://www.lustre.org/
@@ -45,6 +41,7 @@
 #include <lustre_acl.h>
 #include <lustre_mds.h>
 #include "mdt_internal.h"
+#include <lustre_nodemap.h>
 
 /* we do nothing because we do not have refcount now */
 static void mdt_mfd_get(void *mfdp)
@@ -65,9 +62,9 @@ struct mdt_file_data *mdt_mfd_new(const struct mdt_export_data *med)
 
        OBD_ALLOC_PTR(mfd);
        if (mfd != NULL) {
-               CFS_INIT_LIST_HEAD(&mfd->mfd_handle.h_link);
+               INIT_LIST_HEAD(&mfd->mfd_handle.h_link);
                mfd->mfd_handle.h_owner = med;
-               CFS_INIT_LIST_HEAD(&mfd->mfd_list);
+               INIT_LIST_HEAD(&mfd->mfd_list);
                class_handle_hash(&mfd->mfd_handle, &mfd_handle_ops);
        }
 
@@ -91,7 +88,7 @@ struct mdt_file_data *mdt_handle2mfd(struct mdt_export_data *med,
        mfd = class_handle2object(handle->cookie, med);
        /* during dw/setattr replay the mfd can be found by old handle */
        if (mfd == NULL && is_replay_or_resent) {
-               cfs_list_for_each_entry(mfd, &med->med_open_head, mfd_list) {
+               list_for_each_entry(mfd, &med->med_open_head, mfd_list) {
                        if (mfd->mfd_old_handle.cookie == handle->cookie)
                                RETURN(mfd);
                }
@@ -104,8 +101,8 @@ struct mdt_file_data *mdt_handle2mfd(struct mdt_export_data *med,
 /* free mfd */
 void mdt_mfd_free(struct mdt_file_data *mfd)
 {
-        LASSERT(cfs_list_empty(&mfd->mfd_list));
-        OBD_FREE_RCU(mfd, sizeof *mfd, &mfd->mfd_handle);
+       LASSERT(list_empty(&mfd->mfd_list));
+       OBD_FREE_RCU(mfd, sizeof *mfd, &mfd->mfd_handle);
 }
 
 static int mdt_create_data(struct mdt_thread_info *info,
@@ -122,10 +119,7 @@ static int mdt_create_data(struct mdt_thread_info *info,
        ma->ma_need = MA_INODE | MA_LOV;
        ma->ma_valid = 0;
        mutex_lock(&o->mot_lov_mutex);
-       if (!(o->mot_flags & MOF_LOV_CREATED)) {
-               if (p != NULL && !fid_is_md_operative(mdt_object_fid(p)))
-                       GOTO(unlock, rc = -EPERM);
-
+       if (!o->mot_lov_created) {
                rc = mdo_create_data(info->mti_env,
                                     p ? mdt_object_child(p) : NULL,
                                     mdt_object_child(o), spec, ma);
@@ -133,437 +127,66 @@ static int mdt_create_data(struct mdt_thread_info *info,
                        rc = mdt_attr_get_complex(info, o, ma);
 
                if (rc == 0 && ma->ma_valid & MA_LOV)
-                       o->mot_flags |= MOF_LOV_CREATED;
+                       o->mot_lov_created = 1;
        }
-unlock:
+
        mutex_unlock(&o->mot_lov_mutex);
        RETURN(rc);
 }
 
-static int mdt_ioepoch_opened(struct mdt_object *mo)
-{
-        return mo->mot_ioepoch_count;
-}
-
-int mdt_object_is_som_enabled(struct mdt_object *mo)
-{
-        return !mo->mot_ioepoch;
-}
-
-/**
- * Re-enable Size-on-MDS.
- * Call under ->mot_ioepoch_mutex.
- */
-static void mdt_object_som_enable(struct mdt_object *mo, __u64 ioepoch)
-{
-        if (ioepoch == mo->mot_ioepoch) {
-                LASSERT(!mdt_ioepoch_opened(mo));
-                mo->mot_ioepoch = 0;
-                mo->mot_flags = 0;
-        }
-}
-
-/**
- * Open the IOEpoch. It is allowed if @writecount is not negative.
- * The epoch and writecount handling is performed under the mot_ioepoch_mutex.
- */
-int mdt_ioepoch_open(struct mdt_thread_info *info, struct mdt_object *o,
-                     int created)
-{
-        struct mdt_device *mdt = info->mti_mdt;
-        int cancel = 0;
-        int rc = 0;
-        ENTRY;
-
-       if (!(mdt_conn_flags(info) & OBD_CONNECT_SOM) ||
-           !S_ISREG(lu_object_attr(&o->mot_obj)))
-               RETURN(0);
-
-       mutex_lock(&o->mot_ioepoch_mutex);
-       if (mdt_ioepoch_opened(o)) {
-               /* Epoch continues even if there is no writers yet. */
-               CDEBUG(D_INODE, "continue epoch "LPU64" for "DFID"\n",
-                      o->mot_ioepoch, PFID(mdt_object_fid(o)));
-       } else {
-               /* XXX: ->mdt_ioepoch is not initialized at the mount */
-               spin_lock(&mdt->mdt_ioepoch_lock);
-                if (mdt->mdt_ioepoch < info->mti_replayepoch)
-                        mdt->mdt_ioepoch = info->mti_replayepoch;
-
-                if (info->mti_replayepoch)
-                        o->mot_ioepoch = info->mti_replayepoch;
-                else if (++mdt->mdt_ioepoch == IOEPOCH_INVAL)
-                        o->mot_ioepoch = ++mdt->mdt_ioepoch;
-                else
-                        o->mot_ioepoch = mdt->mdt_ioepoch;
-
-               spin_unlock(&mdt->mdt_ioepoch_lock);
-
-               CDEBUG(D_INODE, "starting epoch "LPU64" for "DFID"\n",
-                      o->mot_ioepoch, PFID(mdt_object_fid(o)));
-               if (created)
-                       o->mot_flags |= MOF_SOM_CREATED;
-               cancel = 1;
-       }
-       o->mot_ioepoch_count++;
-       mutex_unlock(&o->mot_ioepoch_mutex);
-
-        /* Cancel Size-on-MDS attributes cached on clients for the open case.
-         * In the truncate case, see mdt_reint_setattr(). */
-        if (cancel && (info->mti_rr.rr_fid1 != NULL)) {
-                struct mdt_lock_handle  *lh = &info->mti_lh[MDT_LH_CHILD];
-                mdt_lock_reg_init(lh, LCK_EX);
-                rc = mdt_object_lock(info, o, lh, MDS_INODELOCK_UPDATE,
-                                     MDT_LOCAL_LOCK);
-                if (rc == 0)
-                        mdt_object_unlock(info, o, lh, 1);
-        }
-        RETURN(rc);
-}
-
-/**
- * Update SOM on-disk attributes.
- * If enabling, write update inodes and lustre-ea with the proper IOEpoch,
- * mountid and attributes. If disabling, clean SOM xattr.
- * Call under ->mot_ioepoch_mutex.
- */
-static int mdt_som_attr_set(struct mdt_thread_info *info,
-                           struct mdt_object *obj, __u64 ioepoch, bool enable)
-{
-       struct md_object        *next = mdt_object_child(obj);
-       int                      rc;
-        ENTRY;
-
-        CDEBUG(D_INODE, "Size-on-MDS attribute %s for epoch "LPU64
-               " on "DFID".\n", enable ? "update" : "disabling",
-               ioepoch, PFID(mdt_object_fid(obj)));
-
-       if (enable) {
-               struct lu_buf           *buf = &info->mti_buf;
-               struct som_attrs        *attrs;
-               struct md_attr          *ma = &info->mti_attr;
-               struct lu_attr          *la = &ma->ma_attr;
-               struct obd_device       *obd = info->mti_mdt->mdt_lut.lut_obd;
-
-               attrs = (struct som_attrs *)info->mti_xattr_buf;
-               CLASSERT(sizeof(info->mti_xattr_buf) >= sizeof(*attrs));
-
-               /* pack SOM attributes */
-               memset(attrs, 0, sizeof(*attrs));
-               attrs->som_ioepoch = ioepoch;
-               attrs->som_mountid = obd->u.obt.obt_mount_count;
-               if ((la->la_valid & LA_SIZE) != 0)
-                       attrs->som_size = la->la_size;
-               if ((la->la_valid & LA_BLOCKS) != 0)
-                       attrs->som_blocks = la->la_blocks;
-               lustre_som_swab(attrs);
-
-               /* update SOM attributes */
-               buf->lb_buf = attrs;
-               buf->lb_len = sizeof(*attrs);
-               rc = mo_xattr_set(info->mti_env, next, buf, XATTR_NAME_SOM, 0);
-       } else {
-               /* delete SOM attributes */
-               rc = mo_xattr_del(info->mti_env, next, XATTR_NAME_SOM);
-       }
-
-        RETURN(rc);
-}
-
-/** Perform the eviction specific actions on ioepoch close. */
-static inline int mdt_ioepoch_close_on_eviction(struct mdt_thread_info *info,
-                                                struct mdt_object *o)
-{
-        int rc = 0;
-
-       mutex_lock(&o->mot_ioepoch_mutex);
-        CDEBUG(D_INODE, "Eviction. Closing IOepoch "LPU64" on "DFID". "
-               "Count %d\n", o->mot_ioepoch, PFID(mdt_object_fid(o)),
-               o->mot_ioepoch_count);
-        o->mot_ioepoch_count--;
-
-        /* If eviction occured set MOF_SOM_RECOV,
-         * if no other epoch holders, disable SOM on disk. */
-        o->mot_flags |= MOF_SOM_CHANGE | MOF_SOM_RECOV;
-        if (!mdt_ioepoch_opened(o)) {
-                rc = mdt_som_attr_set(info, o, o->mot_ioepoch, MDT_SOM_DISABLE);
-                mdt_object_som_enable(o, o->mot_ioepoch);
-        }
-       mutex_unlock(&o->mot_ioepoch_mutex);
-        RETURN(rc);
-}
-
-/**
- * Perform the replay specific actions on ioepoch close.
- * Skip SOM attribute update if obtained and just forget about the inode state
- * for the last ioepoch holder. The SOM cache is invalidated on MDS failure.
- */
-static inline int mdt_ioepoch_close_on_replay(struct mdt_thread_info *info,
-                                              struct mdt_object *o)
-{
-        int rc = MDT_IOEPOCH_CLOSED;
-        ENTRY;
-
-       mutex_lock(&o->mot_ioepoch_mutex);
-        CDEBUG(D_INODE, "Replay. Closing epoch "LPU64" on "DFID". Count %d\n",
-               o->mot_ioepoch, PFID(mdt_object_fid(o)), o->mot_ioepoch_count);
-        o->mot_ioepoch_count--;
-
-        /* Get an info from the replayed request if client is supposed
-         * to send an Attibute Update, reconstruct @rc if so */
-        if (info->mti_ioepoch->flags & MF_SOM_AU)
-                rc = MDT_IOEPOCH_GETATTR;
-
-        if (!mdt_ioepoch_opened(o))
-                mdt_object_som_enable(o, info->mti_ioepoch->ioepoch);
-       mutex_unlock(&o->mot_ioepoch_mutex);
-
-        RETURN(rc);
-}
-
-/**
- * Regular file IOepoch close.
- * Closes the ioepoch, checks the object state, apply obtained attributes and
- * re-enable SOM on the object, if possible. Also checks if the recovery is
- * needed and packs OBD_MD_FLGETATTRLOCK flag into the reply to force the client
- * to obtain SOM attributes under the server-side OST locks.
- *
- * Return value:
- * MDT_IOEPOCH_CLOSED if ioepoch is closed.
- * MDT_IOEPOCH_GETATTR if ioepoch is closed but another SOM update is needed.
- */
-static inline int mdt_ioepoch_close_reg(struct mdt_thread_info *info,
-                                        struct mdt_object *o)
-{
-        struct md_attr *tmp_ma;
-        struct lu_attr *la;
-        int achange, opened;
-        int recovery = 0;
-        int rc = 0, ret = MDT_IOEPOCH_CLOSED;
-        ENTRY;
-
-        la = &info->mti_attr.ma_attr;
-        achange = (info->mti_ioepoch->flags & MF_SOM_CHANGE);
-
-       mutex_lock(&o->mot_ioepoch_mutex);
-        o->mot_ioepoch_count--;
-
-        tmp_ma = &info->mti_u.som.attr;
-        tmp_ma->ma_lmm = info->mti_attr.ma_lmm;
-        tmp_ma->ma_lmm_size = info->mti_attr.ma_lmm_size;
-        tmp_ma->ma_som = &info->mti_u.som.data;
-        tmp_ma->ma_need = MA_INODE | MA_LOV | MA_SOM;
-        tmp_ma->ma_valid = 0;
-       rc = mdt_attr_get_complex(info, o, tmp_ma);
-        if (rc)
-                GOTO(error_up, rc);
-
-        /* Check the on-disk SOM state. */
-        if (o->mot_flags & MOF_SOM_RECOV)
-                recovery = 1;
-        else if (!(o->mot_flags & MOF_SOM_CREATED) &&
-                 !(tmp_ma->ma_valid & MA_SOM))
-                recovery = 1;
-
-        CDEBUG(D_INODE, "Closing epoch "LPU64" on "DFID". Count %d\n",
-               o->mot_ioepoch, PFID(mdt_object_fid(o)), o->mot_ioepoch_count);
-
-        opened = mdt_ioepoch_opened(o);
-        /**
-         * If IOEpoch is not opened, check if a Size-on-MDS update is needed.
-         * Skip the check for file with no LOV  or for unlink files.
-         */
-        if (!opened && tmp_ma->ma_valid & MA_LOV &&
-            !(tmp_ma->ma_valid & MA_INODE && tmp_ma->ma_attr.la_nlink == 0)) {
-                if (recovery)
-                        /* If some previous writer was evicted, re-ask the
-                         * client for attributes. Even if attributes are
-                         * provided, we cannot believe in them.
-                         * Another use case is that there is no SOM cache on
-                         * disk -- first access with SOM or there was an MDS
-                         * failure. */
-                        ret = MDT_IOEPOCH_GETATTR;
-                else if (o->mot_flags & MOF_SOM_CHANGE)
-                        /* Some previous writer changed the attribute.
-                         * Do not believe to the current Size-on-MDS
-                         * update, re-ask client. */
-                        ret = MDT_IOEPOCH_GETATTR;
-                else if (!(la->la_valid & LA_SIZE) && achange)
-                        /* Attributes were changed by the last writer
-                         * only but no Size-on-MDS update is received.*/
-                        ret = MDT_IOEPOCH_GETATTR;
-        }
-
-        if (achange || ret == MDT_IOEPOCH_GETATTR)
-                o->mot_flags |= MOF_SOM_CHANGE;
-
-        /* If epoch ends and relable SOM attributes are obtained, update them.
-         * Create SOM ea for new files even if there is no attributes obtained
-         * (0-length file). */
-        if (ret == MDT_IOEPOCH_CLOSED && !opened) {
-                if (achange || o->mot_flags & MOF_SOM_CREATED) {
-                        LASSERT(achange || !(la->la_valid & LA_SIZE));
-                        rc = mdt_som_attr_set(info, o, o->mot_ioepoch,
-                                              MDT_SOM_ENABLE);
-                        /* Avoid the following setattrs of these attributes,
-                         * e.g. for atime update. */
-                        info->mti_attr.ma_valid = 0;
-                }
-                mdt_object_som_enable(o, o->mot_ioepoch);
-        }
-
-       mutex_unlock(&o->mot_ioepoch_mutex);
-        /* If recovery is needed, tell the client to perform GETATTR under
-         * the lock. */
-        if (ret == MDT_IOEPOCH_GETATTR && recovery) {
-                struct mdt_body *rep;
-                rep = req_capsule_server_get(info->mti_pill, &RMF_MDT_BODY);
-                rep->valid |= OBD_MD_FLGETATTRLOCK;
-        }
-
-        RETURN(rc ? : ret);
-
-error_up:
-       mutex_unlock(&o->mot_ioepoch_mutex);
-        return rc;
-}
-
-/**
- * Close IOEpoch (opened file or MDS_FMODE_EPOCH state). It happens if:
- * - a client closes the IOEpoch;
- * - a client eviction occured.
- * Return values:
- * MDT_IOEPOCH_OPENED if the client does not close IOEpoch.
- * MDT_IOEPOCH_CLOSED if the client closes IOEpoch.
- * MDT_IOEPOCH_GETATTR if the client closes IOEpoch but another SOM attribute
- * update is needed.
- */
-static int mdt_ioepoch_close(struct mdt_thread_info *info, struct mdt_object *o)
+int mdt_write_read(struct mdt_object *o)
 {
-       struct ptlrpc_request *req = mdt_info_req(info);
+       int rc = 0;
        ENTRY;
-
-       if (!(mdt_conn_flags(info) & OBD_CONNECT_SOM) ||
-           !S_ISREG(lu_object_attr(&o->mot_obj)))
-               RETURN(0);
-
-        LASSERT(o->mot_ioepoch_count);
-        LASSERT(info->mti_ioepoch == NULL ||
-                info->mti_ioepoch->ioepoch == o->mot_ioepoch);
-
-        /* IOEpoch is closed only if client tells about it or eviction occures.
-         * In the replay case, always close the epoch. */
-        if (req == NULL)
-                RETURN(mdt_ioepoch_close_on_eviction(info, o));
-        if (lustre_msg_get_flags(req->rq_reqmsg) & MSG_REPLAY)
-                RETURN(mdt_ioepoch_close_on_replay(info, o));
-       if (info->mti_ioepoch && (info->mti_ioepoch->flags & MF_EPOCH_CLOSE))
-                RETURN(mdt_ioepoch_close_reg(info, o));
-        /* IO epoch is not closed. */
-        RETURN(MDT_IOEPOCH_OPENED);
+       spin_lock(&o->mot_write_lock);
+       rc = o->mot_write_count;
+       spin_unlock(&o->mot_write_lock);
+       RETURN(rc);
 }
 
-/**
- * Close MDS_FMODE_SOM state, when IOEpoch is already closed and we are waiting
- * for attribute update. It happens if:
- * - SOM Attribute Update is obtained;
- * - the client failed to obtain it and informs MDS about it;
- * - a client eviction occured.
- * Apply obtained attributes for the 1st case, wipe out the on-disk SOM
- * cache otherwise.
- */
-int mdt_som_au_close(struct mdt_thread_info *info, struct mdt_object *o)
+int mdt_write_get(struct mdt_object *o)
 {
-       struct ptlrpc_request   *req = mdt_info_req(info);
-       __u64                    ioepoch = 0;
-       int                      act = MDT_SOM_ENABLE;
-       int                      rc = 0;
+       int rc = 0;
        ENTRY;
+       spin_lock(&o->mot_write_lock);
+       if (o->mot_write_count < 0)
+               rc = -ETXTBSY;
+       else
+               o->mot_write_count++;
+       spin_unlock(&o->mot_write_lock);
 
-       LASSERT(!req || info->mti_ioepoch);
-       if (!(mdt_conn_flags(info) & OBD_CONNECT_SOM) ||
-           !S_ISREG(lu_object_attr(&o->mot_obj)))
-               RETURN(0);
-
-        /* No size whereas MF_SOM_CHANGE is set means client failed to
-         * obtain ost attributes, drop the SOM cache on disk if so. */
-        if (!req ||
-            (info->mti_ioepoch &&
-             info->mti_ioepoch->flags & MF_SOM_CHANGE &&
-             !(info->mti_attr.ma_attr.la_valid & LA_SIZE)))
-                act = MDT_SOM_DISABLE;
-
-       mutex_lock(&o->mot_ioepoch_mutex);
-        /* Mark the object it is the recovery state if we failed to obtain
-         * SOM attributes. */
-        if (act == MDT_SOM_DISABLE)
-                o->mot_flags |= MOF_SOM_RECOV;
-
-        if (!mdt_ioepoch_opened(o)) {
-                ioepoch =  info->mti_ioepoch ?
-                        info->mti_ioepoch->ioepoch : o->mot_ioepoch;
-
-               if (req != NULL
-                   && !(lustre_msg_get_flags(req->rq_reqmsg) & MSG_REPLAY))
-                        rc = mdt_som_attr_set(info, o, ioepoch, act);
-                mdt_object_som_enable(o, ioepoch);
-        }
-       mutex_unlock(&o->mot_ioepoch_mutex);
-        RETURN(rc);
-}
-
-int mdt_write_read(struct mdt_object *o)
-{
-        int rc = 0;
-        ENTRY;
-       mutex_lock(&o->mot_ioepoch_mutex);
-        rc = o->mot_writecount;
-       mutex_unlock(&o->mot_ioepoch_mutex);
-        RETURN(rc);
-}
-
-int mdt_write_get(struct mdt_object *o)
-{
-        int rc = 0;
-        ENTRY;
-       mutex_lock(&o->mot_ioepoch_mutex);
-        if (o->mot_writecount < 0)
-                rc = -ETXTBSY;
-        else
-                o->mot_writecount++;
-       mutex_unlock(&o->mot_ioepoch_mutex);
-        RETURN(rc);
+       RETURN(rc);
 }
 
 void mdt_write_put(struct mdt_object *o)
 {
-        ENTRY;
-       mutex_lock(&o->mot_ioepoch_mutex);
-        o->mot_writecount--;
-       mutex_unlock(&o->mot_ioepoch_mutex);
-        EXIT;
+       ENTRY;
+       spin_lock(&o->mot_write_lock);
+       o->mot_write_count--;
+       spin_unlock(&o->mot_write_lock);
+       EXIT;
 }
 
 static int mdt_write_deny(struct mdt_object *o)
 {
-        int rc = 0;
-        ENTRY;
-       mutex_lock(&o->mot_ioepoch_mutex);
-        if (o->mot_writecount > 0)
-                rc = -ETXTBSY;
-        else
-                o->mot_writecount--;
-       mutex_unlock(&o->mot_ioepoch_mutex);
-        RETURN(rc);
+       int rc = 0;
+       ENTRY;
+       spin_lock(&o->mot_write_lock);
+       if (o->mot_write_count > 0)
+               rc = -ETXTBSY;
+       else
+               o->mot_write_count--;
+       spin_unlock(&o->mot_write_lock);
+       RETURN(rc);
 }
 
 static void mdt_write_allow(struct mdt_object *o)
 {
-        ENTRY;
-       mutex_lock(&o->mot_ioepoch_mutex);
-        o->mot_writecount++;
-       mutex_unlock(&o->mot_ioepoch_mutex);
-        EXIT;
+       ENTRY;
+       spin_lock(&o->mot_write_lock);
+       o->mot_write_count++;
+       spin_unlock(&o->mot_write_lock);
+       EXIT;
 }
 
 /* there can be no real transaction so prepare the fake one */
@@ -579,12 +202,25 @@ static void mdt_empty_transno(struct mdt_thread_info *info, int rc)
         if (lustre_msg_get_transno(req->rq_repmsg) != 0)
                 RETURN_EXIT;
 
+       if (tgt_is_multimodrpcs_client(req->rq_export)) {
+               struct thandle         *th;
+
+               /* generate an empty transaction to get a transno
+                * and reply data */
+               th = dt_trans_create(info->mti_env, mdt->mdt_bottom);
+               if (!IS_ERR(th)) {
+                       rc = dt_trans_start(info->mti_env, mdt->mdt_bottom, th);
+                       dt_trans_stop(info->mti_env, mdt->mdt_bottom, th);
+               }
+               RETURN_EXIT;
+       }
+
        spin_lock(&mdt->mdt_lut.lut_translock);
        if (rc != 0) {
                if (info->mti_transno != 0) {
                        struct obd_export *exp = req->rq_export;
 
-                       CERROR("%s: replay trans "LPU64" NID %s: rc = %d\n",
+                       CERROR("%s: replay trans %llu NID %s: rc = %d\n",
                               mdt_obd_name(mdt), info->mti_transno,
                               libcfs_nid2str(exp->exp_connection->c_peer.nid),
                               rc);
@@ -600,7 +236,7 @@ static void mdt_empty_transno(struct mdt_thread_info *info, int rc)
        }
        spin_unlock(&mdt->mdt_lut.lut_translock);
 
-       CDEBUG(D_INODE, "transno = "LPU64", last_committed = "LPU64"\n",
+       CDEBUG(D_INODE, "transno = %llu, last_committed = %llu\n",
               info->mti_transno,
               req->rq_export->exp_obd->obd_last_committed);
 
@@ -619,14 +255,13 @@ static void mdt_empty_transno(struct mdt_thread_info *info, int rc)
                 * otherwise the following resend(after replay) can not
                 * be checked correctly by xid */
                mutex_unlock(&ted->ted_lcd_lock);
-               CDEBUG(D_HA, "%s: transno = "LPU64" < last_transno = "LPU64"\n",
+               CDEBUG(D_HA, "%s: transno = %llu < last_transno = %llu\n",
                       mdt_obd_name(mdt), info->mti_transno,
                       lcd->lcd_last_transno);
                RETURN_EXIT;
        }
 
-        if (lustre_msg_get_opc(req->rq_reqmsg) == MDS_CLOSE ||
-            lustre_msg_get_opc(req->rq_reqmsg) == MDS_DONE_WRITING) {
+       if (lustre_msg_get_opc(req->rq_reqmsg) == MDS_CLOSE) {
                if (info->mti_transno != 0)
                        lcd->lcd_last_close_transno = info->mti_transno;
                 lcd->lcd_last_close_xid = req->rq_xid;
@@ -656,7 +291,7 @@ void mdt_mfd_set_mode(struct mdt_file_data *mfd, __u64 mode)
 {
        LASSERT(mfd != NULL);
 
-       CDEBUG(D_HA, DFID " Change mfd mode "LPO64" -> "LPO64".\n",
+       CDEBUG(D_HA, DFID " Change mfd mode %#llo -> %#llo.\n",
               PFID(mdt_object_fid(mfd->mfd_object)), mfd->mfd_mode, mode);
 
        mfd->mfd_mode = mode;
@@ -665,9 +300,9 @@ void mdt_mfd_set_mode(struct mdt_file_data *mfd, __u64 mode)
 /**
  * prep ma_lmm/ma_lmv for md_attr from reply
  */
-void mdt_prep_ma_buf_from_rep(struct mdt_thread_info *info,
-                             struct mdt_object *obj,
-                             struct md_attr *ma)
+static void mdt_prep_ma_buf_from_rep(struct mdt_thread_info *info,
+                                    struct mdt_object *obj,
+                                    struct md_attr *ma)
 {
        LASSERT(ma->ma_lmv == NULL && ma->ma_lmm == NULL);
        if (S_ISDIR(obj->mot_header.loh_attr)) {
@@ -724,41 +359,47 @@ static int mdt_mfd_open(struct mdt_thread_info *info, struct mdt_object *p,
                        mdt_set_disposition(info, rep, DISP_OPEN_STRIPE);
         }
 
-        CDEBUG(D_INODE, "after open, ma_valid bit = "LPX64" lmm_size = %d\n",
-               ma->ma_valid, ma->ma_lmm_size);
+       CDEBUG(D_INODE, "after open, ma_valid bit = %#llx lmm_size = %d\n",
+              ma->ma_valid, ma->ma_lmm_size);
 
         if (ma->ma_valid & MA_LOV) {
                 LASSERT(ma->ma_lmm_size != 0);
-                repbody->eadatasize = ma->ma_lmm_size;
-                if (isdir)
-                        repbody->valid |= OBD_MD_FLDIREA;
-                else
-                        repbody->valid |= OBD_MD_FLEASIZE;
-        }
+               repbody->mbo_eadatasize = ma->ma_lmm_size;
+               if (isdir)
+                       repbody->mbo_valid |= OBD_MD_FLDIREA;
+               else
+                       repbody->mbo_valid |= OBD_MD_FLEASIZE;
+       }
 
        if (ma->ma_valid & MA_LMV) {
                LASSERT(ma->ma_lmv_size != 0);
-               repbody->eadatasize = ma->ma_lmv_size;
+               repbody->mbo_eadatasize = ma->ma_lmv_size;
                LASSERT(isdir);
-               repbody->valid |= OBD_MD_FLDIREA | OBD_MD_MEA;
+               repbody->mbo_valid |= OBD_MD_FLDIREA | OBD_MD_MEA;
        }
 
-        if (flags & FMODE_WRITE) {
-                rc = mdt_write_get(o);
-                if (rc == 0) {
-                        mdt_ioepoch_open(info, o, created);
-                        repbody->ioepoch = o->mot_ioepoch;
-                }
-        } else if (flags & MDS_FMODE_EXEC) {
+       if (flags & FMODE_WRITE)
+               rc = mdt_write_get(o);
+       else if (flags & MDS_FMODE_EXEC)
                rc = mdt_write_deny(o);
-        }
+
         if (rc)
                 RETURN(rc);
 
         rc = mo_open(info->mti_env, mdt_object_child(o),
                      created ? flags | MDS_OPEN_CREATED : flags);
-        if (rc)
-                GOTO(err_out, rc);
+       if (rc != 0) {
+               /* If we allow the client to chgrp (CFS_SETGRP_PERM), but the
+                * client does not know which suppgid should be sent to the MDS,
+                * or some other(s) changed the target file's GID after this RPC
+                * sent to the MDS with the suppgid as the original GID, then we
+                * give the client another chance to send the right suppgid. */
+               if (rc == -EACCES &&
+                   allow_client_chgrp(info, lu_ucred(info->mti_env)))
+                       mdt_set_disposition(info, rep, DISP_OPEN_DENY);
+
+               GOTO(err_out, rc);
+       }
 
        mfd = mdt_mfd_new(med);
        if (mfd == NULL)
@@ -798,11 +439,11 @@ static int mdt_mfd_open(struct mdt_thread_info *info, struct mdt_object *p,
                old_mfd = mdt_handle2mfd(med, info->mti_rr.rr_handle, true);
                if (old_mfd != NULL) {
                        CDEBUG(D_HA, "delete orphan mfd = %p, fid = "DFID", "
-                              "cookie = "LPX64"\n", mfd,
+                              "cookie = %#llx\n", mfd,
                               PFID(mdt_object_fid(mfd->mfd_object)),
                               info->mti_rr.rr_handle->cookie);
                        class_handle_unhash(&old_mfd->mfd_handle);
-                       cfs_list_del_init(&old_mfd->mfd_list);
+                       list_del_init(&old_mfd->mfd_list);
                        spin_unlock(&med->med_open_lock);
                        /* no attr update for that close */
                        la->la_valid = 0;
@@ -814,28 +455,28 @@ static int mdt_mfd_open(struct mdt_thread_info *info, struct mdt_object *p,
                } else {
                        spin_unlock(&med->med_open_lock);
                        CDEBUG(D_HA, "orphan mfd not found, fid = "DFID", "
-                              "cookie = "LPX64"\n",
+                              "cookie = %#llx\n",
                               PFID(mdt_object_fid(mfd->mfd_object)),
                               info->mti_rr.rr_handle->cookie);
                }
 
-               CDEBUG(D_HA, "Store old cookie "LPX64" in new mfd\n",
+               CDEBUG(D_HA, "Store old cookie %#llx in new mfd\n",
                       info->mti_rr.rr_handle->cookie);
 
                mfd->mfd_old_handle.cookie = info->mti_rr.rr_handle->cookie;
        }
 
-       repbody->handle.cookie = mfd->mfd_handle.h_cookie;
+       repbody->mbo_handle.cookie = mfd->mfd_handle.h_cookie;
 
        if (req->rq_export->exp_disconnected) {
                spin_lock(&med->med_open_lock);
                class_handle_unhash(&mfd->mfd_handle);
-               cfs_list_del_init(&mfd->mfd_list);
+               list_del_init(&mfd->mfd_list);
                spin_unlock(&med->med_open_lock);
                mdt_mfd_close(info, mfd);
        } else {
                spin_lock(&med->med_open_lock);
-               cfs_list_add(&mfd->mfd_list, &med->med_open_head);
+               list_add(&mfd->mfd_list, &med->med_open_head);
                spin_unlock(&med->med_open_lock);
        }
 
@@ -845,8 +486,6 @@ static int mdt_mfd_open(struct mdt_thread_info *info, struct mdt_object *p,
 
 err_out:
        if (flags & FMODE_WRITE)
-               /* XXX We also need to close io epoch here.
-                * See LU-1220 - green */
                mdt_write_put(o);
        else if (flags & MDS_FMODE_EXEC)
                mdt_write_allow(o);
@@ -854,21 +493,21 @@ err_out:
        return rc;
 }
 
-int mdt_finish_open(struct mdt_thread_info *info,
-                    struct mdt_object *p, struct mdt_object *o,
-                    __u64 flags, int created, struct ldlm_reply *rep)
+static int mdt_finish_open(struct mdt_thread_info *info,
+                          struct mdt_object *p, struct mdt_object *o,
+                          __u64 flags, int created, struct ldlm_reply *rep)
 {
-        struct ptlrpc_request   *req = mdt_info_req(info);
-        struct obd_export       *exp = req->rq_export;
-        struct mdt_export_data  *med = &req->rq_export->exp_mdt_data;
-        struct md_attr          *ma  = &info->mti_attr;
-        struct lu_attr          *la  = &ma->ma_attr;
-        struct mdt_file_data    *mfd;
-        struct mdt_body         *repbody;
-        int                      rc = 0;
-        int                      isreg, isdir, islnk;
-        cfs_list_t              *t;
-        ENTRY;
+       struct ptlrpc_request   *req = mdt_info_req(info);
+       struct obd_export       *exp = req->rq_export;
+       struct mdt_export_data  *med = &req->rq_export->exp_mdt_data;
+       struct md_attr          *ma  = &info->mti_attr;
+       struct lu_attr          *la  = &ma->ma_attr;
+       struct mdt_file_data    *mfd;
+       struct mdt_body         *repbody;
+       int                      rc = 0;
+       int                      isreg, isdir, islnk;
+       struct list_head        *t;
+       ENTRY;
 
         LASSERT(ma->ma_valid & MA_INODE);
 
@@ -881,8 +520,8 @@ int mdt_finish_open(struct mdt_thread_info *info,
 
        /* LU-2275, simulate broken behaviour (esp. prevalent in
         * pre-2.4 servers where a very strange reply is sent on error
-        * that looks like it was actually almost succesful and a failure at the
-        * same time */
+        * that looks like it was actually almost successful and a
+        * failure at the same time.) */
        if (OBD_FAIL_CHECK(OBD_FAIL_MDS_NEGATIVE_POSITIVE)) {
                mdt_set_disposition(info, rep, DISP_OPEN_OPEN |
                                               DISP_LOOKUP_NEG |
@@ -894,75 +533,19 @@ int mdt_finish_open(struct mdt_thread_info *info,
                RETURN(-ENOENT);
        }
 
-        if (exp_connect_rmtclient(exp)) {
-                void *buf = req_capsule_server_get(info->mti_pill, &RMF_ACL);
-
-                rc = mdt_pack_remote_perm(info, o, buf);
-                if (rc) {
-                        repbody->valid &= ~OBD_MD_FLRMTPERM;
-                        repbody->aclsize = 0;
-                } else {
-                        repbody->valid |= OBD_MD_FLRMTPERM;
-                        repbody->aclsize = sizeof(struct mdt_remote_perm);
-                }
-        }
 #ifdef CONFIG_FS_POSIX_ACL
-       else if (exp_connect_flags(exp) & OBD_CONNECT_ACL) {
-                const struct lu_env *env = info->mti_env;
-                struct md_object *next = mdt_object_child(o);
-                struct lu_buf *buf = &info->mti_buf;
-
-                buf->lb_buf = req_capsule_server_get(info->mti_pill, &RMF_ACL);
-                buf->lb_len = req_capsule_get_size(info->mti_pill, &RMF_ACL,
-                                                   RCL_SERVER);
-                if (buf->lb_len > 0) {
-                        rc = mo_xattr_get(env, next, buf,
-                                          XATTR_NAME_ACL_ACCESS);
-                        if (rc < 0) {
-                                if (rc == -ENODATA) {
-                                        repbody->aclsize = 0;
-                                        repbody->valid |= OBD_MD_FLACL;
-                                        rc = 0;
-                                } else if (rc == -EOPNOTSUPP) {
-                                        rc = 0;
-                                } else {
-                                        CERROR("got acl size: %d\n", rc);
-                                }
-                        } else {
-                                repbody->aclsize = rc;
-                                repbody->valid |= OBD_MD_FLACL;
-                                rc = 0;
-                        }
-                }
-        }
+       if (exp_connect_flags(exp) & OBD_CONNECT_ACL) {
+               struct lu_nodemap *nodemap = nodemap_get_from_exp(exp);
+               if (IS_ERR(nodemap))
+                       RETURN(PTR_ERR(nodemap));
+
+               rc = mdt_pack_acl2body(info, repbody, o, nodemap);
+               nodemap_putref(nodemap);
+               if (rc)
+                       RETURN(rc);
+       }
 #endif
 
-       if (info->mti_mdt->mdt_lut.lut_mds_capa &&
-           exp_connect_flags(exp) & OBD_CONNECT_MDS_CAPA) {
-                struct lustre_capa *capa;
-
-                capa = req_capsule_server_get(info->mti_pill, &RMF_CAPA1);
-                LASSERT(capa);
-                capa->lc_opc = CAPA_OPC_MDS_DEFAULT;
-                rc = mo_capa_get(info->mti_env, mdt_object_child(o), capa, 0);
-                if (rc)
-                        RETURN(rc);
-                repbody->valid |= OBD_MD_FLMDSCAPA;
-        }
-       if (info->mti_mdt->mdt_lut.lut_oss_capa &&
-           exp_connect_flags(exp) & OBD_CONNECT_OSS_CAPA &&
-           S_ISREG(lu_object_attr(&o->mot_obj))) {
-                struct lustre_capa *capa;
-
-                capa = req_capsule_server_get(info->mti_pill, &RMF_CAPA2);
-                LASSERT(capa);
-                capa->lc_opc = CAPA_OPC_OSS_DEFAULT | capa_open_opc(flags);
-                rc = mo_capa_get(info->mti_env, mdt_object_child(o), capa, 0);
-                if (rc)
-                        RETURN(rc);
-                repbody->valid |= OBD_MD_FLOSSCAPA;
-        }
-
         /*
          * If we are following a symlink, don't open; and do not return open
          * handle for special nodes as client required.
@@ -994,11 +577,11 @@ int mdt_finish_open(struct mdt_thread_info *info,
                 RETURN(-EAGAIN);
         }
 
-        mfd = NULL;
-        if (lustre_msg_get_flags(req->rq_reqmsg) & MSG_RESENT) {
+       mfd = NULL;
+       if (lustre_msg_get_flags(req->rq_reqmsg) & MSG_RESENT) {
                spin_lock(&med->med_open_lock);
-               cfs_list_for_each(t, &med->med_open_head) {
-                       mfd = cfs_list_entry(t, struct mdt_file_data, mfd_list);
+               list_for_each(t, &med->med_open_head) {
+                       mfd = list_entry(t, struct mdt_file_data, mfd_list);
                        if (mfd->mfd_xid == req->rq_xid)
                                break;
                        mfd = NULL;
@@ -1006,15 +589,15 @@ int mdt_finish_open(struct mdt_thread_info *info,
                spin_unlock(&med->med_open_lock);
 
                 if (mfd != NULL) {
-                        repbody->handle.cookie = mfd->mfd_handle.h_cookie;
-                        /*set repbody->ea_size for resent case*/
-                        if (ma->ma_valid & MA_LOV) {
-                                LASSERT(ma->ma_lmm_size != 0);
-                                repbody->eadatasize = ma->ma_lmm_size;
-                                if (isdir)
-                                        repbody->valid |= OBD_MD_FLDIREA;
-                                else
-                                        repbody->valid |= OBD_MD_FLEASIZE;
+                       repbody->mbo_handle.cookie = mfd->mfd_handle.h_cookie;
+                       /* set repbody->ea_size for resent case */
+                       if (ma->ma_valid & MA_LOV) {
+                               LASSERT(ma->ma_lmm_size != 0);
+                               repbody->mbo_eadatasize = ma->ma_lmm_size;
+                               if (isdir)
+                                       repbody->mbo_valid |= OBD_MD_FLDIREA;
+                               else
+                                       repbody->mbo_valid |= OBD_MD_FLEASIZE;
                         }
                        mdt_set_disposition(info, rep, DISP_OPEN_OPEN);
                        RETURN(0);
@@ -1028,9 +611,6 @@ int mdt_finish_open(struct mdt_thread_info *info,
        RETURN(rc);
 }
 
-extern void mdt_req_from_lcd(struct ptlrpc_request *req,
-                             struct lsd_client_data *lcd);
-
 void mdt_reconstruct_open(struct mdt_thread_info *info,
                           struct mdt_lock_handle *lhc)
 {
@@ -1038,8 +618,6 @@ void mdt_reconstruct_open(struct mdt_thread_info *info,
         struct mdt_device       *mdt  = info->mti_mdt;
         struct req_capsule      *pill = info->mti_pill;
         struct ptlrpc_request   *req  = mdt_info_req(info);
-        struct tg_export_data   *ted  = &req->rq_export->exp_target_data;
-        struct lsd_client_data  *lcd  = ted->ted_lcd;
         struct md_attr          *ma   = &info->mti_attr;
         struct mdt_reint_record *rr   = &info->mti_rr;
        __u64                   flags = info->mti_spec.sp_cr_flags;
@@ -1048,19 +626,20 @@ void mdt_reconstruct_open(struct mdt_thread_info *info,
         struct mdt_object       *child;
         struct mdt_body         *repbody;
         int                      rc;
-        ENTRY;
+       __u64                    opdata;
+       ENTRY;
 
         LASSERT(pill->rc_fmt == &RQF_LDLM_INTENT_OPEN);
         ldlm_rep = req_capsule_server_get(pill, &RMF_DLM_REP);
         repbody = req_capsule_server_get(pill, &RMF_MDT_BODY);
 
        ma->ma_need = MA_INODE | MA_HSM;
-        ma->ma_valid = 0;
+       ma->ma_valid = 0;
 
-        mdt_req_from_lcd(req, lcd);
-        mdt_set_disposition(info, ldlm_rep, lcd->lcd_last_data);
+       opdata = mdt_req_from_lrd(req, info->mti_reply_data);
+       mdt_set_disposition(info, ldlm_rep, opdata);
 
-        CDEBUG(D_INODE, "This is reconstruct open: disp="LPX64", result=%d\n",
+       CDEBUG(D_INODE, "This is reconstruct open: disp=%#llx, result=%d\n",
                ldlm_rep->lock_policy_res1, req->rq_status);
 
         if (mdt_get_disposition(ldlm_rep, DISP_OPEN_CREATE) &&
@@ -1107,13 +686,11 @@ void mdt_reconstruct_open(struct mdt_thread_info *info,
                                mdt_object_put(env, child);
                                GOTO(out, rc = -EIO);
                        }
-                       repbody->fid1 = *rr->rr_fid2;
-                       repbody->valid |= (OBD_MD_FLID | OBD_MD_MDS);
+                       repbody->mbo_fid1 = *rr->rr_fid2;
+                       repbody->mbo_valid |= (OBD_MD_FLID | OBD_MD_MDS);
                        rc = 0;
                } else {
                        if (mdt_object_exists(child)) {
-                               mdt_set_capainfo(info, 1, rr->rr_fid2,
-                                                BYPASS_CAPA);
                                mdt_prep_ma_buf_from_rep(info, child, ma);
                                rc = mdt_attr_get_complex(info, child, ma);
                                if (rc == 0)
@@ -1144,7 +721,7 @@ out:
         LASSERT(ergo(rc < 0, lustre_msg_get_transno(req->rq_repmsg) == 0));
 }
 
-int mdt_open_by_fid(struct mdt_thread_info *info, struct ldlm_reply *rep)
+static int mdt_open_by_fid(struct mdt_thread_info *info, struct ldlm_reply *rep)
 {
        __u64                    flags = info->mti_spec.sp_cr_flags;
         struct mdt_reint_record *rr = &info->mti_rr;
@@ -1165,8 +742,8 @@ int mdt_open_by_fid(struct mdt_thread_info *info, struct ldlm_reply *rep)
                                                DISP_LOOKUP_EXECD |
                                                DISP_LOOKUP_POS));
                 repbody = req_capsule_server_get(info->mti_pill, &RMF_MDT_BODY);
-                repbody->fid1 = *rr->rr_fid2;
-                repbody->valid |= (OBD_MD_FLID | OBD_MD_MDS);
+               repbody->mbo_fid1 = *rr->rr_fid2;
+               repbody->mbo_valid |= (OBD_MD_FLID | OBD_MD_MDS);
                 rc = 0;
        } else {
                if (mdt_object_exists(o)) {
@@ -1193,13 +770,13 @@ static int mdt_object_open_lock(struct mdt_thread_info *info,
                                struct mdt_lock_handle *lhc,
                                __u64 *ibits)
 {
-       struct md_attr  *ma = &info->mti_attr;
-       __u64            open_flags = info->mti_spec.sp_cr_flags;
-       ldlm_mode_t      lm = LCK_CR;
-       bool             acq_lease = !!(open_flags & MDS_OPEN_LEASE);
-       bool             try_layout = false;
-       bool             create_layout = false;
-       int              rc = 0;
+       struct md_attr *ma = &info->mti_attr;
+       __u64 open_flags = info->mti_spec.sp_cr_flags;
+       enum ldlm_mode lm = LCK_CR;
+       bool acq_lease = !!(open_flags & MDS_OPEN_LEASE);
+       bool try_layout = false;
+       bool create_layout = false;
+       int rc = 0;
        ENTRY;
 
        *ibits = 0;
@@ -1230,9 +807,10 @@ static int mdt_object_open_lock(struct mdt_thread_info *info,
 
                /* Lease must be with open lock */
                if (!(open_flags & MDS_OPEN_LOCK)) {
-                       CERROR("Request lease for file:"DFID ", but open lock "
-                               "is missed, open_flags = "LPO64".\n",
-                               PFID(mdt_object_fid(obj)), open_flags);
+                       CERROR("%s: Request lease for file:"DFID ", but open lock "
+                              "is missed, open_flags = %#llo : rc = %d\n",
+                              mdt_obd_name(info->mti_mdt),
+                              PFID(mdt_object_fid(obj)), open_flags, -EPROTO);
                        GOTO(out, rc = -EPROTO);
                }
 
@@ -1246,29 +824,30 @@ static int mdt_object_open_lock(struct mdt_thread_info *info,
                /* normal open holds read mode of open sem */
                down_read(&obj->mot_open_sem);
 
-               if (open_flags & FMODE_WRITE)
-                       lm = LCK_CW;
-               else if (open_flags & MDS_FMODE_EXEC)
-                       lm = LCK_PR;
-               else
-                       lm = LCK_CR;
-
                if (open_flags & MDS_OPEN_LOCK) {
+                       if (open_flags & FMODE_WRITE)
+                               lm = LCK_CW;
+                       else if (open_flags & MDS_FMODE_EXEC)
+                               lm = LCK_PR;
+                       else
+                               lm = LCK_CR;
+
                        *ibits = MDS_INODELOCK_LOOKUP | MDS_INODELOCK_OPEN;
-               } else if (open_flags & (FMODE_WRITE | MDS_FMODE_EXEC) ||
-                          atomic_read(&obj->mot_lease_count) > 0) {
-                       /* We need to flush conflicting locks or revoke a lease.
-                        * In either case there is no need to acquire a layout
-                        * lock since it won't be returned to the client. */
-                       try_layout = false;
+               } else if (atomic_read(&obj->mot_lease_count) > 0) {
+                       if (open_flags & FMODE_WRITE)
+                               lm = LCK_CW;
+                       else
+                               lm = LCK_CR;
+
+                       /* revoke lease */
                        *ibits = MDS_INODELOCK_OPEN;
+                       try_layout = false;
+
                        lhc = &info->mti_lh[MDT_LH_LOCAL];
                }
-
-               CDEBUG(D_INODE, "normal open FID = "DFID", open_count = %d, "
-                      "lm = %d\n",
-                      PFID(mdt_object_fid(obj)),
-                      atomic_read(&obj->mot_open_count), lm);
+               CDEBUG(D_INODE, "normal open:"DFID" lease count: %d, lm: %d\n",
+                       PFID(mdt_object_fid(obj)),
+                       atomic_read(&obj->mot_open_count), lm);
        }
 
        mdt_lock_reg_init(lhc, lm);
@@ -1282,27 +861,26 @@ static int mdt_object_open_lock(struct mdt_thread_info *info,
                 * However this is a double-edged sword because changing
                 * permission will revoke huge # of LOOKUP locks. */
                *ibits |= MDS_INODELOCK_LAYOUT | MDS_INODELOCK_LOOKUP;
-               if (!mdt_object_lock_try(info, obj, lhc, *ibits,
-                                        MDT_CROSS_LOCK)) {
+               if (!mdt_object_lock_try(info, obj, lhc, *ibits)) {
                        *ibits &= ~(MDS_INODELOCK_LAYOUT|MDS_INODELOCK_LOOKUP);
                        if (*ibits != 0)
-                               rc = mdt_object_lock(info, obj, lhc, *ibits,
-                                               MDT_CROSS_LOCK);
+                               rc = mdt_object_lock(info, obj, lhc, *ibits);
                }
        } else if (*ibits != 0) {
-               rc = mdt_object_lock(info, obj, lhc, *ibits, MDT_CROSS_LOCK);
+               rc = mdt_object_lock(info, obj, lhc, *ibits);
        }
 
-       CDEBUG(D_INODE, "Requested bits lock:"DFID ", ibits = "LPX64
-               ", open_flags = "LPO64", try_layout = %d, rc = %d\n",
-               PFID(mdt_object_fid(obj)), *ibits, open_flags, try_layout, rc);
+       CDEBUG(D_INODE, "%s: Requested bits lock:"DFID ", ibits = %#llx"
+              ", open_flags = %#llo, try_layout = %d : rc = %d\n",
+              mdt_obd_name(info->mti_mdt), PFID(mdt_object_fid(obj)),
+              *ibits, open_flags, try_layout, rc);
 
        /* will change layout, revoke layout locks by enqueuing EX lock. */
        if (rc == 0 && create_layout) {
                struct mdt_lock_handle *ll = &info->mti_lh[MDT_LH_LAYOUT];
 
                CDEBUG(D_INODE, "Will create layout, get EX layout lock:"DFID
-                       ", open_flags = "LPO64"\n",
+                       ", open_flags = %#llo\n",
                        PFID(mdt_object_fid(obj)), open_flags);
 
                /* We cannot enqueue another lock for the same resource we
@@ -1317,8 +895,7 @@ static int mdt_object_open_lock(struct mdt_thread_info *info,
                LASSERT(!try_layout);
                mdt_lock_handle_init(ll);
                mdt_lock_reg_init(ll, LCK_EX);
-               rc = mdt_object_lock(info, obj, ll, MDS_INODELOCK_LAYOUT,
-                                       MDT_LOCAL_LOCK);
+               rc = mdt_object_lock(info, obj, ll, MDS_INODELOCK_LAYOUT);
 
                OBD_FAIL_TIMEOUT(OBD_FAIL_MDS_LL_BLOCK, 2);
        }
@@ -1394,7 +971,7 @@ static void mdt_object_open_unlock(struct mdt_thread_info *info,
                up_read(&obj->mot_open_sem);
 
        /* Cross-ref case, the lock should be returned to the client */
-       if (ibits == 0 || rc == -EREMOTE)
+       if (ibits == 0 || rc == -MDT_EREMOTE_OPEN)
                RETURN_EXIT;
 
        if (!(open_flags & MDS_OPEN_LOCK) && !(ibits & MDS_INODELOCK_LAYOUT)) {
@@ -1431,8 +1008,9 @@ static bool mdt_hsm_release_allow(const struct md_attr *ma)
        return true;
 }
 
-int mdt_open_by_fid_lock(struct mdt_thread_info *info, struct ldlm_reply *rep,
-                        struct mdt_lock_handle *lhc)
+static int mdt_open_by_fid_lock(struct mdt_thread_info *info,
+                               struct ldlm_reply *rep,
+                               struct mdt_lock_handle *lhc)
 {
         const struct lu_env     *env   = info->mti_env;
         struct mdt_device       *mdt   = info->mti_mdt;
@@ -1442,6 +1020,7 @@ int mdt_open_by_fid_lock(struct mdt_thread_info *info, struct ldlm_reply *rep,
         struct mdt_object       *parent= NULL;
         struct mdt_object       *o;
         int                      rc;
+       bool                     object_locked = false;
        __u64                    ibits = 0;
         ENTRY;
 
@@ -1460,9 +1039,9 @@ int mdt_open_by_fid_lock(struct mdt_thread_info *info, struct ldlm_reply *rep,
                         ma->ma_need |= MA_PFID;
         }
 
-        o = mdt_object_find(env, mdt, rr->rr_fid2);
-        if (IS_ERR(o))
-                RETURN(rc = PTR_ERR(o));
+       o = mdt_object_find(env, mdt, rr->rr_fid2);
+       if (IS_ERR(o))
+               GOTO(out_parent_put, rc = PTR_ERR(o));
 
        if (mdt_object_remote(o)) {
                CDEBUG(D_INFO, "%s: "DFID" is on remote MDT.\n",
@@ -1486,14 +1065,25 @@ int mdt_open_by_fid_lock(struct mdt_thread_info *info, struct ldlm_reply *rep,
        if (rc)
                GOTO(out, rc);
 
+       /* We should not change file's existing LOV EA */
+       if (S_ISREG(lu_object_attr(&o->mot_obj)) &&
+           flags & MDS_OPEN_HAS_EA && ma->ma_valid & MA_LOV)
+               GOTO(out, rc = -EEXIST);
+
        /* If a release request, check file flags are fine and ask for an
         * exclusive open access. */
        if (flags & MDS_OPEN_RELEASE && !mdt_hsm_release_allow(ma))
                GOTO(out, rc = -EPERM);
 
-       rc = mdt_object_open_lock(info, o, lhc, &ibits);
-        if (rc)
-                GOTO(out_unlock, rc);
+       rc = mdt_check_resent_lock(info, o, lhc);
+       if (rc < 0) {
+               GOTO(out, rc);
+       } else if (rc > 0) {
+               rc = mdt_object_open_lock(info, o, lhc, &ibits);
+               object_locked = true;
+               if (rc)
+                       GOTO(out_unlock, rc);
+       }
 
         if (ma->ma_valid & MA_PFID) {
                 parent = mdt_object_find(env, mdt, &ma->ma_pfid);
@@ -1517,9 +1107,11 @@ int mdt_open_by_fid_lock(struct mdt_thread_info *info, struct ldlm_reply *rep,
        GOTO(out_unlock, rc);
 
 out_unlock:
-       mdt_object_open_unlock(info, o, lhc, ibits, rc);
+       if (object_locked)
+               mdt_object_open_unlock(info, o, lhc, ibits, rc);
 out:
        mdt_object_put(env, o);
+out_parent_put:
        if (parent != NULL)
                mdt_object_put(env, parent);
        return rc;
@@ -1546,7 +1138,7 @@ static int mdt_cross_open(struct mdt_thread_info *info,
                       mdt_obd_name(info->mti_mdt), PFID(fid), -EFAULT);
                LU_OBJECT_DEBUG(D_WARNING, info->mti_env,
                                &o->mot_obj,
-                               "Object isn't on this server! FLD error?\n");
+                               "Object isn't on this server! FLD error?");
                 rc = -EFAULT;
        } else {
                if (mdt_object_exists(o)) {
@@ -1558,18 +1150,10 @@ static int mdt_cross_open(struct mdt_thread_info *info,
                                goto out;
 
                        mdt_prep_ma_buf_from_rep(info, o, ma);
-                       mdt_set_capainfo(info, 0, fid, BYPASS_CAPA);
                        rc = mdt_attr_get_complex(info, o, ma);
                        if (rc != 0)
                                GOTO(out, rc);
 
-                       /* Do not create lov object if the fid is opened
-                        * under OBF */
-                       if (S_ISREG(ma->ma_attr.la_mode) &&
-                           !(ma->ma_valid & MA_LOV) && (flags & FMODE_WRITE) &&
-                           fid_is_obf(parent_fid))
-                               GOTO(out, rc = -EPERM);
-
                        rc = mdt_finish_open(info, NULL, o, flags, 0, rep);
                } else {
                        /*
@@ -1586,6 +1170,60 @@ out:
         RETURN(rc);
 }
 
+/*
+ * find root object and take its xattr lock if it's on remote MDT, later create
+ * may use fs default striping (which is stored in root xattr).
+ */
+static int mdt_lock_root_xattr(struct mdt_thread_info *info,
+                              struct mdt_device *mdt)
+{
+       struct mdt_object *md_root = mdt->mdt_md_root;
+       struct lustre_handle lhroot;
+       int rc;
+
+       if (md_root == NULL) {
+               lu_root_fid(&info->mti_tmp_fid1);
+               md_root = mdt_object_find(info->mti_env, mdt,
+                                         &info->mti_tmp_fid1);
+               if (IS_ERR(md_root))
+                       return PTR_ERR(md_root);
+
+               spin_lock(&mdt->mdt_lock);
+               if (mdt->mdt_md_root != NULL) {
+                       spin_unlock(&mdt->mdt_lock);
+
+                       LASSERTF(mdt->mdt_md_root == md_root,
+                                "Different root object ("
+                                DFID") instances, %p, %p\n",
+                                PFID(&info->mti_tmp_fid1),
+                                mdt->mdt_md_root, md_root);
+                       LASSERT(atomic_read(
+                               &md_root->mot_obj.lo_header->loh_ref) > 1);
+
+                       mdt_object_put(info->mti_env, md_root);
+               } else {
+                       mdt->mdt_md_root = md_root;
+                       spin_unlock(&mdt->mdt_lock);
+               }
+       }
+
+       if (md_root->mot_cache_attr || !mdt_object_remote(md_root))
+               return 0;
+
+       rc = mdt_remote_object_lock(info, md_root, mdt_object_fid(md_root),
+                                   &lhroot, LCK_PR, MDS_INODELOCK_XATTR,
+                                   false, true);
+       if (rc < 0)
+               return rc;
+
+       md_root->mot_cache_attr = 1;
+
+       /* don't cancel this lock, so that we know the cached xattr is valid. */
+       ldlm_lock_decref(&lhroot, LCK_PR);
+
+       return 0;
+}
+
 int mdt_reint_open(struct mdt_thread_info *info, struct mdt_lock_handle *lhc)
 {
         struct mdt_device       *mdt = info->mti_mdt;
@@ -1602,6 +1240,7 @@ int mdt_reint_open(struct mdt_thread_info *info, struct mdt_lock_handle *lhc)
         struct mdt_reint_record *rr = &info->mti_rr;
         int                      result, rc;
         int                      created = 0;
+       int                      object_locked = 0;
         __u32                    msg_flags;
         ENTRY;
 
@@ -1627,8 +1266,12 @@ int mdt_reint_open(struct mdt_thread_info *info, struct mdt_lock_handle *lhc)
             info->mti_spec.u.sp_ea.eadata == NULL)
                 GOTO(out, result = err_serious(-EINVAL));
 
+       if (create_flags & FMODE_WRITE &&
+           exp_connect_flags(req->rq_export) & OBD_CONNECT_RDONLY)
+               GOTO(out, result = -EROFS);
+
        CDEBUG(D_INODE, "I am going to open "DFID"/("DNAME"->"DFID") "
-              "cr_flag="LPO64" mode=0%06o msg_flag=0x%x\n",
+              "cr_flag=%#llo mode=0%06o msg_flag=0x%x\n",
               PFID(rr->rr_fid1), PNAME(&rr->rr_name),
               PFID(rr->rr_fid2), create_flags,
               ma->ma_attr.la_mode, msg_flags);
@@ -1660,43 +1303,46 @@ int mdt_reint_open(struct mdt_thread_info *info, struct mdt_lock_handle *lhc)
                        GOTO(out, result = -EFAULT);
                }
                CDEBUG(D_INFO, "No object(1), continue as regular open.\n");
-       } else if ((!lu_name_is_valid(&rr->rr_name) &&
-                   (create_flags & MDS_OPEN_LOCK)) ||
-                  (create_flags & MDS_OPEN_BY_FID)) {
+       } else if (create_flags & (MDS_OPEN_BY_FID | MDS_OPEN_LOCK)) {
+               /*
+                * MDS_OPEN_LOCK is checked for backward compatibility with 2.1
+                * client.
+                */
                result = mdt_open_by_fid_lock(info, ldlm_rep, lhc);
-               /* If result is 0 then open by FID has found the file
-                * and there is nothing left for us to do here.  More
-                * generally if it is anything other than -ENOENT or
-                * -EREMOTE then we return that now.  If -ENOENT and
-                * MDS_OPEN_CREAT is set then we must create the file
-                * below.  If -EREMOTE then we need to return a LOOKUP
-                * lock to the client, which we do below.  Hence this
-                * odd looking condition.  See LU-2523. */
-               if (!(result == -ENOENT && (create_flags & MDS_OPEN_CREAT)) &&
-                   result != -EREMOTE)
-                       GOTO(out, result);
-
-               CDEBUG(D_INFO, "No object(2), continue as regular open.\n");
+               if (result < 0)
+                       CDEBUG(D_INFO, "no object for "DFID": %d\n",
+                              PFID(rr->rr_fid2), result);
+               GOTO(out, result);
        }
 
-        if (OBD_FAIL_CHECK(OBD_FAIL_MDS_OPEN_PACK))
-                GOTO(out, result = err_serious(-ENOMEM));
+       if (OBD_FAIL_CHECK(OBD_FAIL_MDS_OPEN_PACK))
+               GOTO(out, result = err_serious(-ENOMEM));
 
-        mdt_set_disposition(info, ldlm_rep,
-                            (DISP_IT_EXECD | DISP_LOOKUP_EXECD));
+       mdt_set_disposition(info, ldlm_rep,
+                           (DISP_IT_EXECD | DISP_LOOKUP_EXECD));
 
        if (!lu_name_is_valid(&rr->rr_name))
                GOTO(out, result = -EPROTO);
 
-        lh = &info->mti_lh[MDT_LH_PARENT];
+       result = mdt_lock_root_xattr(info, mdt);
+       if (result < 0)
+               GOTO(out, result);
+
+again:
+       lh = &info->mti_lh[MDT_LH_PARENT];
        mdt_lock_pdo_init(lh,
                          (create_flags & MDS_OPEN_CREAT) ? LCK_PW : LCK_PR,
                          &rr->rr_name);
 
-        parent = mdt_object_find_lock(info, rr->rr_fid1, lh,
-                                      MDS_INODELOCK_UPDATE);
-        if (IS_ERR(parent))
-                GOTO(out, result = PTR_ERR(parent));
+       parent = mdt_object_find(info->mti_env, mdt, rr->rr_fid1);
+       if (IS_ERR(parent))
+               GOTO(out, result = PTR_ERR(parent));
+
+       result = mdt_object_lock(info, parent, lh, MDS_INODELOCK_UPDATE);
+       if (result != 0) {
+               mdt_object_put(info->mti_env, parent);
+               GOTO(out, result);
+       }
 
         /* get and check version of parent */
         result = mdt_version_get_check(info, parent, 0);
@@ -1716,16 +1362,45 @@ int mdt_reint_open(struct mdt_thread_info *info, struct mdt_lock_handle *lhc)
         if (result != 0 && result != -ENOENT && result != -ESTALE)
                 GOTO(out_parent, result);
 
-        if (result == -ENOENT || result == -ESTALE) {
-                mdt_set_disposition(info, ldlm_rep, DISP_LOOKUP_NEG);
-                if (result == -ESTALE) {
-                        /*
-                         * -ESTALE means the parent is a dead(unlinked) dir, so
-                         * it should return -ENOENT to in accordance with the
-                         * original mds implementaion.
-                         */
-                        GOTO(out_parent, result = -ENOENT);
-                }
+       if (result == -ENOENT || result == -ESTALE) {
+               /* If the object is dead, let's check if the object
+                * is being migrated to a new object */
+               if (result == -ESTALE) {
+                       struct lu_buf lmv_buf;
+
+                       lmv_buf.lb_buf = info->mti_xattr_buf;
+                       lmv_buf.lb_len = sizeof(info->mti_xattr_buf);
+                       rc = mo_xattr_get(info->mti_env,
+                                         mdt_object_child(parent),
+                                         &lmv_buf, XATTR_NAME_LMV);
+                       if (rc > 0) {
+                               struct lmv_mds_md_v1 *lmv;
+
+                               lmv = lmv_buf.lb_buf;
+                               if (le32_to_cpu(lmv->lmv_hash_type) &
+                                               LMV_HASH_FLAG_MIGRATION) {
+                                       /* Get the new parent FID and retry */
+                                       mdt_object_unlock_put(info, parent,
+                                                             lh, 1);
+                                       mdt_lock_handle_init(lh);
+                                       fid_le_to_cpu(
+                                               (struct lu_fid *)rr->rr_fid1,
+                                               &lmv->lmv_stripe_fids[1]);
+                                       goto again;
+                               }
+                       }
+               }
+
+               mdt_set_disposition(info, ldlm_rep, DISP_LOOKUP_NEG);
+               if (result == -ESTALE) {
+                       /*
+                        * -ESTALE means the parent is a dead(unlinked) dir, so
+                        * it should return -ENOENT to in accordance with the
+                        * original mds implementaion.
+                        */
+                       GOTO(out_parent, result = -ENOENT);
+               }
+
                 if (!(create_flags & MDS_OPEN_CREAT))
                         GOTO(out_parent, result);
                if (exp_connect_flags(req->rq_export) & OBD_CONNECT_RDONLY)
@@ -1753,7 +1428,6 @@ int mdt_reint_open(struct mdt_thread_info *info, struct mdt_lock_handle *lhc)
         if (rc)
                 GOTO(out_child, result = rc);
 
-        mdt_set_capainfo(info, 1, child_fid, BYPASS_CAPA);
         if (result == -ENOENT) {
                /* Create under OBF and .lustre is not permitted */
                if (!fid_is_md_operative(rr->rr_fid1))
@@ -1798,7 +1472,8 @@ int mdt_reint_open(struct mdt_thread_info *info, struct mdt_lock_handle *lhc)
                         if (result != 0)
                                 GOTO(out_child, result);
                 }
-                created = 1;
+               created = 1;
+               mdt_counter_incr(req, LPROC_MDT_MKNOD);
         } else {
                 /*
                  * The object is on remote node, return its FID for remote open.
@@ -1811,35 +1486,22 @@ int mdt_reint_open(struct mdt_thread_info *info, struct mdt_lock_handle *lhc)
                          */
                         LASSERT(lhc != NULL);
 
-                        if (lustre_handle_is_used(&lhc->mlh_reg_lh)) {
-                                struct ldlm_lock *lock;
-
-                                LASSERT(msg_flags & MSG_RESENT);
-
-                                lock = ldlm_handle2lock(&lhc->mlh_reg_lh);
-                                if (!lock) {
-                                        CERROR("Invalid lock handle "LPX64"\n",
-                                               lhc->mlh_reg_lh.cookie);
-                                        LBUG();
-                                }
-                                LASSERT(fid_res_name_eq(mdt_object_fid(child),
-                                                        &lock->l_resource->lr_name));
-                                LDLM_LOCK_PUT(lock);
-                                rc = 0;
-                        } else {
-                                mdt_lock_handle_init(lhc);
-                                mdt_lock_reg_init(lhc, LCK_PR);
-
-                                rc = mdt_object_lock(info, child, lhc,
-                                                     MDS_INODELOCK_LOOKUP,
-                                                     MDT_CROSS_LOCK);
-                        }
-                        repbody->fid1 = *mdt_object_fid(child);
-                        repbody->valid |= (OBD_MD_FLID | OBD_MD_MDS);
+                       rc = mdt_check_resent_lock(info, child, lhc);
+                       if (rc < 0) {
+                               GOTO(out_child, result = rc);
+                       } else if (rc > 0) {
+                               mdt_lock_handle_init(lhc);
+                               mdt_lock_reg_init(lhc, LCK_PR);
+
+                               rc = mdt_object_lock(info, child, lhc,
+                                                    MDS_INODELOCK_LOOKUP);
+                       }
+                       repbody->mbo_fid1 = *mdt_object_fid(child);
+                       repbody->mbo_valid |= (OBD_MD_FLID | OBD_MD_MDS);
                         if (rc != 0)
                                 result = rc;
                        else
-                               result = -EREMOTE;
+                               result = -MDT_EREMOTE_OPEN;
                         GOTO(out_child, result);
                } else if (mdt_object_exists(child)) {
                        /* We have to get attr & LOV EA & HSM for this
@@ -1847,6 +1509,8 @@ int mdt_reint_open(struct mdt_thread_info *info, struct mdt_lock_handle *lhc)
                        mdt_prep_ma_buf_from_rep(info, child, ma);
                        ma->ma_need |= MA_HSM;
                        result = mdt_attr_get_complex(info, child, ma);
+                       if (result != 0)
+                               GOTO(out_child, result);
                } else {
                        /* Object does not exist. Likely FS corruption. */
                        CERROR("%s: name '"DNAME"' present, but FID "
@@ -1856,9 +1520,12 @@ int mdt_reint_open(struct mdt_thread_info *info, struct mdt_lock_handle *lhc)
                }
         }
 
-       if (lustre_handle_is_used(&lhc->mlh_reg_lh)) {
+       rc = mdt_check_resent_lock(info, child, lhc);
+       if (rc < 0) {
+               GOTO(out_child, result = rc);
+       } else if (rc == 0) {
                /* the open lock might already be gotten in
-                * mdt_intent_fixup_resent */
+                * ldlm_handle_enqueue() */
                LASSERT(lustre_msg_get_flags(req->rq_reqmsg) & MSG_RESENT);
                if (create_flags & MDS_OPEN_LOCK)
                        mdt_set_disposition(info, ldlm_rep, DISP_OPEN_LOCK);
@@ -1866,6 +1533,7 @@ int mdt_reint_open(struct mdt_thread_info *info, struct mdt_lock_handle *lhc)
                /* get openlock if this isn't replay and client requested it */
                if (!req_is_replay(req)) {
                        rc = mdt_object_open_lock(info, child, lhc, &ibits);
+                       object_locked = 1;
                        if (rc != 0)
                                GOTO(out_child_unlock, result = rc);
                        else if (create_flags & MDS_OPEN_LOCK)
@@ -1892,7 +1560,6 @@ int mdt_reint_open(struct mdt_thread_info *info, struct mdt_lock_handle *lhc)
                if (created) {
                        ma->ma_need = 0;
                        ma->ma_valid = 0;
-                       ma->ma_cookie_size = 0;
                        rc = mdo_unlink(info->mti_env,
                                        mdt_object_child(parent),
                                        mdt_object_child(child),
@@ -1907,7 +1574,8 @@ int mdt_reint_open(struct mdt_thread_info *info, struct mdt_lock_handle *lhc)
        }
        EXIT;
 out_child_unlock:
-       mdt_object_open_unlock(info, child, lhc, ibits, result);
+       if (object_locked)
+               mdt_object_open_unlock(info, child, lhc, ibits, result);
 out_child:
        mdt_object_put(info->mti_env, child);
 out_parent:
@@ -2030,7 +1698,7 @@ static int mdt_hsm_release(struct mdt_thread_info *info, struct mdt_object *o,
        lease_broken = ldlm_is_cancel(lease);
        unlock_res_and_lock(lease);
 
-       LDLM_DEBUG(lease, DFID " lease broken? %d\n",
+       LDLM_DEBUG(lease, DFID " lease broken? %d",
                   PFID(mdt_object_fid(o)), lease_broken);
 
        /* Cancel server side lease. Client side counterpart should
@@ -2060,8 +1728,8 @@ static int mdt_hsm_release(struct mdt_thread_info *info, struct mdt_object *o,
 
        /* Compare on-disk and packed data_version */
        if (data->cd_data_version != ma->ma_hsm.mh_arch_ver) {
-               CDEBUG(D_HSM, DFID" data_version mismatches: packed="LPU64
-                      " and on-disk="LPU64"\n", PFID(mdt_object_fid(o)),
+               CDEBUG(D_HSM, DFID" data_version mismatches: packed=%llu"
+                      " and on-disk=%llu\n", PFID(mdt_object_fid(o)),
                       data->cd_data_version, ma->ma_hsm.mh_arch_ver);
                GOTO(out_unlock, rc = -EPERM);
        }
@@ -2099,8 +1767,7 @@ static int mdt_hsm_release(struct mdt_thread_info *info, struct mdt_object *o,
        /* Set file as released */
        ma->ma_lmm->lmm_pattern |= cpu_to_le32(LOV_PATTERN_F_RELEASED);
 
-       /* Hopefully it's not used in this call path */
-       orp_ma = &info->mti_u.som.attr;
+       orp_ma = &info->mti_u.hsm.attr;
        orp_ma->ma_attr.la_mode = S_IFREG | S_IWUSR;
        orp_ma->ma_attr.la_uid = ma->ma_attr.la_uid;
        orp_ma->ma_attr.la_gid = ma->ma_attr.la_gid;
@@ -2128,7 +1795,7 @@ static int mdt_hsm_release(struct mdt_thread_info *info, struct mdt_object *o,
 
        mdt_lock_reg_init(lh, LCK_EX);
        rc = mdt_object_lock(info, o, lh, MDS_INODELOCK_LAYOUT |
-                            MDS_INODELOCK_XATTR, MDT_LOCAL_LOCK);
+                            MDS_INODELOCK_XATTR);
        if (rc != 0)
                GOTO(out_close, rc);
 
@@ -2154,17 +1821,19 @@ out_close:
                CERROR("%s: error closing volatile file "DFID": rc = %d\n",
                       mdt_obd_name(info->mti_mdt), PFID(&data->cd_fid), rc2);
        LU_OBJECT_DEBUG(D_HSM, info->mti_env, &orphan->mot_obj,
-                       "object closed\n");
+                       "object closed");
        mdt_object_put(info->mti_env, orphan);
 
 out_unlock:
        up_write(&o->mot_open_sem);
 
-       if (rc == 0) { /* already released */
+       /* already released */
+       if (rc == 0) {
                struct mdt_body *repbody;
+
                repbody = req_capsule_server_get(info->mti_pill, &RMF_MDT_BODY);
                LASSERT(repbody != NULL);
-               repbody->valid |= OBD_MD_FLRELEASED;
+               repbody->mbo_valid |= OBD_MD_CLOSE_INTENT_EXECED;
        }
 
 out_reprocess:
@@ -2177,9 +1846,143 @@ out_reprocess:
        return rc;
 }
 
-#define MFD_CLOSED(mode) (((mode) & ~(MDS_FMODE_EPOCH | MDS_FMODE_SOM | \
-                                      MDS_FMODE_TRUNC)) == MDS_FMODE_CLOSED)
+int mdt_close_swap_layouts(struct mdt_thread_info *info,
+                          struct mdt_object *o, struct md_attr *ma)
+{
+       struct mdt_lock_handle  *lh1 = &info->mti_lh[MDT_LH_NEW];
+       struct mdt_lock_handle  *lh2 = &info->mti_lh[MDT_LH_OLD];
+       struct close_data       *data;
+       struct ldlm_lock        *lease;
+       struct mdt_object       *o1 = o, *o2;
+       bool                     lease_broken;
+       bool                     swap_objects;
+       int                      rc;
+       ENTRY;
+
+       if (exp_connect_flags(info->mti_exp) & OBD_CONNECT_RDONLY)
+               RETURN(-EROFS);
+
+       if (!S_ISREG(lu_object_attr(&o1->mot_obj)))
+               RETURN(-EINVAL);
+
+       data = req_capsule_client_get(info->mti_pill, &RMF_CLOSE_DATA);
+       if (data == NULL)
+               RETURN(-EPROTO);
+
+       if (fid_is_zero(&data->cd_fid) || !fid_is_sane(&data->cd_fid))
+               RETURN(-EINVAL);
+
+       rc = lu_fid_cmp(&data->cd_fid, mdt_object_fid(o));
+       if (unlikely(rc == 0))
+               RETURN(-EINVAL);
+
+       /* Exchange o1 and o2, to enforce locking order */
+       swap_objects = (rc < 0);
+
+       lease = ldlm_handle2lock(&data->cd_handle);
+       if (lease == NULL)
+               RETURN(-ESTALE);
+
+       o2 = mdt_object_find(info->mti_env, info->mti_mdt, &data->cd_fid);
+       if (IS_ERR(o2))
+               GOTO(out_lease, rc = PTR_ERR(o2));
+
+       if (!S_ISREG(lu_object_attr(&o2->mot_obj))) {
+               swap_objects = false; /* not swapped yet */
+               GOTO(out_obj, rc = -EINVAL);
+       }
+
+       if (swap_objects)
+               swap(o1, o2);
+
+       rc = mo_permission(info->mti_env, NULL, mdt_object_child(o1), NULL,
+                          MAY_WRITE);
+       if (rc < 0)
+               GOTO(out_obj, rc);
+
+       rc = mo_permission(info->mti_env, NULL, mdt_object_child(o2), NULL,
+                          MAY_WRITE);
+       if (rc < 0)
+               GOTO(out_obj, rc);
+
+       /* try to hold open_sem so that nobody else can open the file */
+       if (!down_write_trylock(&o->mot_open_sem)) {
+               ldlm_lock_cancel(lease);
+               GOTO(out_obj, rc = -EBUSY);
+       }
+
+       /* Check if the lease open lease has already canceled */
+       lock_res_and_lock(lease);
+       lease_broken = ldlm_is_cancel(lease);
+       unlock_res_and_lock(lease);
+
+       LDLM_DEBUG(lease, DFID " lease broken? %d",
+                  PFID(mdt_object_fid(o)), lease_broken);
+
+       /* Cancel server side lease. Client side counterpart should
+        * have been cancelled. It's okay to cancel it now as we've
+        * held mot_open_sem. */
+       ldlm_lock_cancel(lease);
+
+       if (lease_broken)
+               GOTO(out_unlock_sem, rc = -ESTALE);
+
+       mdt_lock_reg_init(lh1, LCK_EX);
+       rc = mdt_object_lock(info, o1, lh1, MDS_INODELOCK_LAYOUT |
+                            MDS_INODELOCK_XATTR);
+       if (rc < 0)
+               GOTO(out_unlock_sem, rc);
+
+       mdt_lock_reg_init(lh2, LCK_EX);
+       rc = mdt_object_lock(info, o2, lh2, MDS_INODELOCK_LAYOUT |
+                            MDS_INODELOCK_XATTR);
+       if (rc < 0)
+               GOTO(out_unlock1, rc);
+
+       /* Swap layout with orphan object */
+       rc = mo_swap_layouts(info->mti_env, mdt_object_child(o1),
+                            mdt_object_child(o2), 0);
+       if (rc < 0)
+               GOTO(out_unlock2, rc);
+
+       EXIT;
+
+out_unlock2:
+       /* Release exclusive LL */
+       mdt_object_unlock(info, o2, lh2, 1);
+
+out_unlock1:
+       mdt_object_unlock(info, o1, lh1, 1);
+
+out_unlock_sem:
+       up_write(&o->mot_open_sem);
+
+       /* already swapped */
+       if (rc == 0) {
+               struct mdt_body *repbody;
+
+               repbody = req_capsule_server_get(info->mti_pill, &RMF_MDT_BODY);
+               LASSERT(repbody != NULL);
+               repbody->mbo_valid |= OBD_MD_CLOSE_INTENT_EXECED;
+       }
+
+out_obj:
+       mdt_object_put(info->mti_env, swap_objects ? o1 : o2);
 
+       ldlm_reprocess_all(lease->l_resource);
+
+out_lease:
+       LDLM_LOCK_PUT(lease);
+
+       if (ma != NULL) {
+               ma->ma_valid = 0;
+               ma->ma_need = 0;
+       }
+
+       return rc;
+}
+
+#define MFD_CLOSED(mode) ((mode) == MDS_FMODE_CLOSED)
 static int mdt_mfd_closed(struct mdt_file_data *mfd)
 {
         return ((mfd == NULL) || MFD_CLOSED(mfd->mfd_mode));
@@ -2190,7 +1993,6 @@ int mdt_mfd_close(struct mdt_thread_info *info, struct mdt_file_data *mfd)
         struct mdt_object *o = mfd->mfd_object;
         struct md_object *next = mdt_object_child(o);
         struct md_attr *ma = &info->mti_attr;
-        int ret = MDT_IOEPOCH_CLOSED;
         int rc = 0;
        __u64 mode;
         ENTRY;
@@ -2201,22 +2003,27 @@ int mdt_mfd_close(struct mdt_thread_info *info, struct mdt_file_data *mfd)
                rc = mdt_hsm_release(info, o, ma);
                if (rc < 0) {
                        CDEBUG(D_HSM, "%s: File " DFID " release failed: %d\n",
-                               mdt_obd_name(info->mti_mdt),
-                               PFID(mdt_object_fid(o)), rc);
+                              mdt_obd_name(info->mti_mdt),
+                              PFID(mdt_object_fid(o)), rc);
                        /* continue to close even error occurred. */
                }
        }
 
-        if ((mode & FMODE_WRITE) || (mode & MDS_FMODE_TRUNC)) {
-                mdt_write_put(o);
-                ret = mdt_ioepoch_close(info, o);
-        } else if (mode & MDS_FMODE_EXEC) {
-                mdt_write_allow(o);
-        } else if (mode & MDS_FMODE_EPOCH) {
-                ret = mdt_ioepoch_close(info, o);
-        } else if (mode & MDS_FMODE_SOM) {
-                ret = mdt_som_au_close(info, o);
-        }
+       if (ma->ma_attr_flags & MDS_CLOSE_LAYOUT_SWAP) {
+               rc = mdt_close_swap_layouts(info, o, ma);
+               if (rc < 0) {
+                       CDEBUG(D_INODE,
+                              "%s: cannot swap layout of "DFID": rc=%d\n",
+                              mdt_obd_name(info->mti_mdt),
+                              PFID(mdt_object_fid(o)), rc);
+                       /* continue to close even if error occurred. */
+               }
+       }
+
+       if (mode & FMODE_WRITE)
+               mdt_write_put(o);
+       else if (mode & MDS_FMODE_EXEC)
+               mdt_write_allow(o);
 
         /* Update atime on close only. */
         if ((mode & MDS_FMODE_EXEC || mode & FMODE_READ || mode & FMODE_WRITE)
@@ -2237,39 +2044,52 @@ int mdt_mfd_close(struct mdt_thread_info *info, struct mdt_file_data *mfd)
         if (!MFD_CLOSED(mode))
                 rc = mo_close(info->mti_env, next, ma, mode);
 
-        if (ret == MDT_IOEPOCH_GETATTR || ret == MDT_IOEPOCH_OPENED) {
-                struct mdt_export_data *med;
+       /* adjust open and lease count */
+       if (mode & MDS_OPEN_LEASE) {
+               LASSERT(atomic_read(&o->mot_lease_count) > 0);
+               atomic_dec(&o->mot_lease_count);
+       }
 
-                /* The IOepoch is still opened or SOM update is needed.
-                 * Put mfd back into the list. */
-                LASSERT(mdt_conn_flags(info) & OBD_CONNECT_SOM);
-                mdt_mfd_set_mode(mfd, ret == MDT_IOEPOCH_OPENED ?
-                                      MDS_FMODE_EPOCH : MDS_FMODE_SOM);
+       LASSERT(atomic_read(&o->mot_open_count) > 0);
+       atomic_dec(&o->mot_open_count);
+       mdt_mfd_free(mfd);
+       mdt_object_put(info->mti_env, o);
 
-                LASSERT(mdt_info_req(info));
-                med = &mdt_info_req(info)->rq_export->exp_mdt_data;
-               spin_lock(&med->med_open_lock);
-               cfs_list_add(&mfd->mfd_list, &med->med_open_head);
-               class_handle_hash_back(&mfd->mfd_handle);
-               spin_unlock(&med->med_open_lock);
+       RETURN(rc);
+}
 
-                if (ret == MDT_IOEPOCH_OPENED) {
-                        ret = 0;
-                } else {
-                        ret = -EAGAIN;
-                        CDEBUG(D_INODE, "Size-on-MDS attribute update is "
-                               "needed on "DFID"\n", PFID(mdt_object_fid(o)));
-                }
-        } else {
-               /* adjust open and lease count */
-               if (mode & MDS_OPEN_LEASE) {
-                       LASSERT(atomic_read(&o->mot_lease_count) > 0);
-                       atomic_dec(&o->mot_lease_count);
-               }
-               LASSERT(atomic_read(&o->mot_open_count) > 0);
-               atomic_dec(&o->mot_open_count);
+int mdt_close_internal(struct mdt_thread_info *info, struct ptlrpc_request *req,
+                      struct mdt_body *repbody)
+{
+       struct mdt_export_data *med;
+       struct mdt_file_data   *mfd;
+       struct mdt_object      *o;
+       struct md_attr         *ma = &info->mti_attr;
+       int                     ret = 0;
+       int                     rc = 0;
+       ENTRY;
+
+       med = &req->rq_export->exp_mdt_data;
+       spin_lock(&med->med_open_lock);
+       mfd = mdt_handle2mfd(med, &info->mti_close_handle, req_is_replay(req));
+       if (mdt_mfd_closed(mfd)) {
+               spin_unlock(&med->med_open_lock);
+               CDEBUG(D_INODE, "no handle for file close: fid = "DFID
+                      ": cookie = %#llx\n", PFID(info->mti_rr.rr_fid1),
+                      info->mti_close_handle.cookie);
+               /** not serious error since bug 3633 */
+               rc = -ESTALE;
+       } else {
+               class_handle_unhash(&mfd->mfd_handle);
+               list_del_init(&mfd->mfd_list);
+               spin_unlock(&med->med_open_lock);
 
-               mdt_mfd_free(mfd);
+               /* Do not lose object before last unlink. */
+               o = mfd->mfd_object;
+               mdt_object_get(info->mti_env, o);
+               ret = mdt_mfd_close(info, mfd);
+               if (repbody != NULL)
+                       rc = mdt_handle_last_unlink(info, o, ma);
                mdt_object_put(info->mti_env, o);
        }
 
@@ -2280,9 +2100,6 @@ int mdt_close(struct tgt_session_info *tsi)
 {
        struct mdt_thread_info  *info = tsi2mdt_info(tsi);
        struct ptlrpc_request   *req = tgt_ses_req(tsi);
-        struct mdt_export_data *med;
-        struct mdt_file_data   *mfd;
-        struct mdt_object      *o;
         struct md_attr         *ma = &info->mti_attr;
         struct mdt_body        *repbody = NULL;
         int rc, ret = 0;
@@ -2294,12 +2111,12 @@ int mdt_close(struct tgt_session_info *tsi)
        if (rc)
                GOTO(out, rc = err_serious(rc));
 
-        LASSERT(info->mti_ioepoch);
-
+       /* These fields are no longer used and are left for compatibility.
+        * size is always zero */
         req_capsule_set_size(info->mti_pill, &RMF_MDT_MD, RCL_SERVER,
-                             info->mti_mdt->mdt_max_mdsize);
+                            0);
         req_capsule_set_size(info->mti_pill, &RMF_LOGCOOKIES, RCL_SERVER,
-                             info->mti_mdt->mdt_max_cookiesize);
+                            0);
         rc = req_capsule_server_pack(info->mti_pill);
         if (mdt_check_resent(info, mdt_reconstruct_generic, NULL)) {
                 mdt_client_compatibility(info);
@@ -2318,43 +2135,17 @@ int mdt_close(struct tgt_session_info *tsi)
                 ma->ma_lmm_size = req_capsule_get_size(info->mti_pill,
                                                        &RMF_MDT_MD,
                                                        RCL_SERVER);
-                ma->ma_cookie = req_capsule_server_get(info->mti_pill,
-                                                       &RMF_LOGCOOKIES);
-                ma->ma_cookie_size = req_capsule_get_size(info->mti_pill,
-                                                          &RMF_LOGCOOKIES,
-                                                          RCL_SERVER);
-                ma->ma_need = MA_INODE | MA_LOV | MA_COOKIE;
-                repbody->eadatasize = 0;
-                repbody->aclsize = 0;
+               ma->ma_need = MA_INODE | MA_LOV;
+               repbody->mbo_eadatasize = 0;
+               repbody->mbo_aclsize = 0;
         } else {
                 rc = err_serious(rc);
         }
 
-        med = &req->rq_export->exp_mdt_data;
-       spin_lock(&med->med_open_lock);
-       mfd = mdt_handle2mfd(med, &info->mti_ioepoch->handle,
-                            req_is_replay(req));
-       if (mdt_mfd_closed(mfd)) {
-               spin_unlock(&med->med_open_lock);
-               CDEBUG(D_INODE, "no handle for file close: fid = "DFID
-                      ": cookie = "LPX64"\n", PFID(info->mti_rr.rr_fid1),
-                      info->mti_ioepoch->handle.cookie);
-               /** not serious error since bug 3633 */
-               rc = -ESTALE;
-       } else {
-               class_handle_unhash(&mfd->mfd_handle);
-               cfs_list_del_init(&mfd->mfd_list);
-               spin_unlock(&med->med_open_lock);
+       rc = mdt_close_internal(info, req, repbody);
+       if (rc != -ESTALE)
+               mdt_empty_transno(info, rc);
 
-                /* Do not lose object before last unlink. */
-                o = mfd->mfd_object;
-                mdt_object_get(info->mti_env, o);
-                ret = mdt_mfd_close(info, mfd);
-                if (repbody != NULL)
-                        rc = mdt_handle_last_unlink(info, o, ma);
-                mdt_empty_transno(info, rc);
-                mdt_object_put(info->mti_env, o);
-        }
         if (repbody != NULL) {
                 mdt_client_compatibility(info);
                 rc = mdt_fix_reply(info);
@@ -2371,86 +2162,3 @@ out:
        mdt_thread_info_fini(info);
        RETURN(rc ? rc : ret);
 }
-
-/**
- * DONE_WRITING rpc handler.
- *
- * As mfd is not kept after replayed CLOSE (see mdt_ioepoch_close_on_replay()),
- * only those DONE_WRITING rpc will be replayed which really wrote smth on disk,
- * and got a trasid. Waiting for such DONE_WRITING is not reliable, so just
- * skip attributes and reconstruct the reply here.
- */
-int mdt_done_writing(struct tgt_session_info *tsi)
-{
-       struct ptlrpc_request   *req = tgt_ses_req(tsi);
-       struct mdt_thread_info  *info = tsi2mdt_info(tsi);
-        struct mdt_body         *repbody = NULL;
-        struct mdt_export_data  *med;
-        struct mdt_file_data    *mfd;
-        int rc;
-        ENTRY;
-
-       rc = req_capsule_server_pack(tsi->tsi_pill);
-       if (rc)
-               GOTO(out, rc = err_serious(rc));
-
-       repbody = req_capsule_server_get(tsi->tsi_pill, &RMF_MDT_BODY);
-       repbody->eadatasize = 0;
-       repbody->aclsize = 0;
-
-       /* Done Writing may come with the Size-on-MDS update. Unpack it. */
-       rc = mdt_close_unpack(info);
-       if (rc)
-               GOTO(out, rc = err_serious(rc));
-
-       if (mdt_check_resent(info, mdt_reconstruct_generic, NULL)) {
-               mdt_exit_ucred(info);
-               GOTO(out, rc = lustre_msg_get_status(req->rq_repmsg));
-       }
-
-       med = &info->mti_exp->exp_mdt_data;
-       spin_lock(&med->med_open_lock);
-       mfd = mdt_handle2mfd(med, &info->mti_ioepoch->handle,
-                            req_is_replay(req));
-       if (mfd == NULL) {
-               spin_unlock(&med->med_open_lock);
-                CDEBUG(D_INODE, "no handle for done write: fid = "DFID
-                       ": cookie = "LPX64" ioepoch = "LPU64"\n",
-                       PFID(info->mti_rr.rr_fid1),
-                       info->mti_ioepoch->handle.cookie,
-                       info->mti_ioepoch->ioepoch);
-                /* If this is a replay, reconstruct the transno. */
-                if (lustre_msg_get_flags(req->rq_reqmsg) & MSG_REPLAY) {
-                        rc = info->mti_ioepoch->flags & MF_SOM_AU ?
-                             -EAGAIN : 0;
-                        mdt_empty_transno(info, rc);
-               } else
-                       rc = -ESTALE;
-               GOTO(error_ucred, rc);
-        }
-
-        LASSERT(mfd->mfd_mode == MDS_FMODE_EPOCH ||
-                mfd->mfd_mode == MDS_FMODE_TRUNC);
-        class_handle_unhash(&mfd->mfd_handle);
-        cfs_list_del_init(&mfd->mfd_list);
-       spin_unlock(&med->med_open_lock);
-
-        /* Set EPOCH CLOSE flag if not set by client. */
-        info->mti_ioepoch->flags |= MF_EPOCH_CLOSE;
-        info->mti_attr.ma_valid = 0;
-
-        info->mti_attr.ma_lmm_size = info->mti_mdt->mdt_max_mdsize;
-        OBD_ALLOC_LARGE(info->mti_attr.ma_lmm, info->mti_mdt->mdt_max_mdsize);
-       if (info->mti_attr.ma_lmm == NULL)
-               GOTO(error_ucred, rc = -ENOMEM);
-
-        rc = mdt_mfd_close(info, mfd);
-
-        OBD_FREE_LARGE(info->mti_attr.ma_lmm, info->mti_mdt->mdt_max_mdsize);
-        mdt_empty_transno(info, rc);
-error_ucred:
-       mdt_exit_ucred(info);
-out:
-       mdt_thread_info_fini(info);
-       RETURN(rc);
-}