X-Git-Url: https://git.whamcloud.com/?p=fs%2Flustre-release.git;a=blobdiff_plain;f=lustre%2Fmdt%2Fmdt_reint.c;h=62308a8beaab9a5f08eef97a80dcbc7ada3f4e58;hp=8cb8cb7c19671a55913b717f067a5844d42ab493;hb=81af631d07084074c6c0346435e7011a6b4582f7;hpb=7936ec229c7f62cfbf5a0e4ff5933149967e7c8f diff --git a/lustre/mdt/mdt_reint.c b/lustre/mdt/mdt_reint.c index 8cb8cb7..62308a8 100644 --- a/lustre/mdt/mdt_reint.c +++ b/lustre/mdt/mdt_reint.c @@ -15,11 +15,7 @@ * * You should have received a copy of the GNU General Public License * version 2 along with this program; If not, see - * http://www.sun.com/software/products/lustre/docs/GPLv2.pdf - * - * Please contact Sun Microsystems, Inc., 4150 Network Circle, Santa Clara, - * CA 95054 USA or visit www.sun.com if you need additional information or - * have any questions. + * http://www.gnu.org/licenses/gpl-2.0.html * * GPL HEADER END */ @@ -27,7 +23,7 @@ * Copyright (c) 2007, 2010, Oracle and/or its affiliates. All rights reserved. * Use is subject to license terms. * - * Copyright (c) 2011, 2013, Intel Corporation. + * Copyright (c) 2011, 2017, Intel Corporation. */ /* * This file is part of Lustre, http://www.lustre.org/ @@ -46,6 +42,7 @@ #define DEBUG_SUBSYSTEM S_MDS +#include #include "mdt_internal.h" #include @@ -56,33 +53,6 @@ static inline void mdt_reint_init_ma(struct mdt_thread_info *info, ma->ma_valid = 0; } -static int mdt_create_pack_capa(struct mdt_thread_info *info, int rc, - struct mdt_object *object, - struct mdt_body *repbody) -{ - ENTRY; - - /* for cross-ref mkdir, mds capa has been fetched from remote obj, then - * we won't go to below*/ - if (repbody->valid & OBD_MD_FLMDSCAPA) - RETURN(rc); - - if (rc == 0 && info->mti_mdt->mdt_lut.lut_mds_capa && - exp_connect_flags(info->mti_exp) & OBD_CONNECT_MDS_CAPA) { - struct lustre_capa *capa; - - capa = req_capsule_server_get(info->mti_pill, &RMF_CAPA1); - LASSERT(capa); - capa->lc_opc = CAPA_OPC_MDS_DEFAULT; - rc = mo_capa_get(info->mti_env, mdt_object_child(object), capa, - 0); - if (rc == 0) - repbody->valid |= OBD_MD_FLMDSCAPA; - } - - RETURN(rc); -} - /** * Get version of object by fid. * @@ -91,14 +61,15 @@ static int mdt_create_pack_capa(struct mdt_thread_info *info, int rc, static void mdt_obj_version_get(struct mdt_thread_info *info, struct mdt_object *o, __u64 *version) { - LASSERT(o); + LASSERT(o); + if (mdt_object_exists(o) && !mdt_object_remote(o) && !fid_is_obf(mdt_object_fid(o))) - *version = dt_version_get(info->mti_env, mdt_obj2dt(o)); - else - *version = ENOENT_VERSION; - CDEBUG(D_INODE, "FID "DFID" version is "LPX64"\n", - PFID(mdt_object_fid(o)), *version); + *version = dt_version_get(info->mti_env, mdt_obj2dt(o)); + else + *version = ENOENT_VERSION; + CDEBUG(D_INODE, "FID "DFID" version is %#llx\n", + PFID(mdt_object_fid(o)), *version); } /** @@ -126,7 +97,7 @@ static int mdt_version_check(struct ptlrpc_request *req, spin_unlock(&req->rq_export->exp_lock); RETURN(-EOVERFLOW); } else if (pre_ver[idx] != version) { - CDEBUG(D_INODE, "Version mismatch "LPX64" != "LPX64"\n", + CDEBUG(D_INODE, "Version mismatch %#llx != %#llx\n", pre_ver[idx], version); spin_lock(&req->rq_export->exp_lock); req->rq_export->exp_vbr_failed = 1; @@ -219,9 +190,10 @@ int mdt_version_get_check_save(struct mdt_thread_info *info, * This checks version of 'name'. Many reint functions uses 'name' for child not * FID, therefore we need to get object by name and check its version. */ -int mdt_lookup_version_check(struct mdt_thread_info *info, - struct mdt_object *p, const struct lu_name *lname, - struct lu_fid *fid, int idx) +static int mdt_lookup_version_check(struct mdt_thread_info *info, + struct mdt_object *p, + const struct lu_name *lname, + struct lu_fid *fid, int idx) { int rc, vbrc; @@ -245,79 +217,195 @@ int mdt_lookup_version_check(struct mdt_thread_info *info, } +static inline int mdt_remote_permission_check(struct mdt_thread_info *info) +{ + struct lu_ucred *uc = mdt_ucred(info); + struct mdt_device *mdt = info->mti_mdt; + + if (!md_capable(uc, CFS_CAP_SYS_ADMIN)) { + if (uc->uc_gid != mdt->mdt_enable_remote_dir_gid && + mdt->mdt_enable_remote_dir_gid != -1) + return -EPERM; + } + + return 0; +} + /** * mdt_remote_permission: Check whether the remote operation is permitted, * - * Before we implement async cross-MDT updates (DNE phase 2). There are a few - * limitations here: - * - * 1.Only sysadmin can create remote directory and striped directory and - * migrate directory now, unless - * lctl set_param mdt.*.enable_remote_dir_gid=allow_gid. - * 2.Remote directory can only be created on MDT0, unless - * lctl set_param mdt.*.enable_remote_dir = 1 - * 3.Only new clients can access remote dir( >= 2.4) and striped dir(>= 2.6), - * old client will return -ENOTSUPP. + * Only sysadmin can create remote directory / striped directory, + * migrate directory and set default stripedEA on directory, unless * - * XXX these check are only needed for remote synchronization, once async - * update is supported, these check will be removed. + * lctl set_param mdt.*.enable_remote_dir_gid=allow_gid. * - * param[in]info: execution environment. - * param[in]parent: the directory of this operation. - * param[in]child: the child of this operation. + * param[in] info: mdt_thread_info. * * retval = 0 remote operation is allowed. * < 0 remote operation is denied. */ -static int mdt_remote_permission(struct mdt_thread_info *info, - struct mdt_object *parent, - struct mdt_object *child) +static int mdt_remote_permission(struct mdt_thread_info *info) { - struct mdt_device *mdt = info->mti_mdt; - struct lu_ucred *uc = mdt_ucred(info); - struct md_op_spec *spec = &info->mti_spec; - struct lu_attr *attr = &info->mti_attr.ma_attr; - struct obd_export *exp = mdt_info_req(info)->rq_export; - - /* Only check create remote directory, striped directory and - * migration */ - if (mdt_object_remote(parent) == 0 && mdt_object_remote(child) == 0 && - !(S_ISDIR(attr->la_mode) && spec->u.sp_ea.eadata != NULL && - spec->u.sp_ea.eadatalen != 0) && - info->mti_rr.rr_opcode != REINT_MIGRATE) - return 0; + struct md_op_spec *spec = &info->mti_spec; + struct lu_attr *attr = &info->mti_attr.ma_attr; + struct obd_export *exp = mdt_info_req(info)->rq_export; + int rc; - if (!md_capable(uc, CFS_CAP_SYS_ADMIN)) { - if (uc->uc_gid != mdt->mdt_enable_remote_dir_gid && - mdt->mdt_enable_remote_dir_gid != -1) - return -EPERM; + if (info->mti_rr.rr_opcode == REINT_MIGRATE) { + rc = mdt_remote_permission_check(info); + if (rc != 0) + return rc; } - if (mdt->mdt_enable_remote_dir == 0) { - struct seq_server_site *ss = mdt_seq_site(mdt); - struct lu_seq_range range = { 0 }; - int rc; + if (info->mti_rr.rr_opcode == REINT_CREATE && + (S_ISDIR(attr->la_mode) && spec->u.sp_ea.eadata != NULL && + spec->u.sp_ea.eadatalen != 0)) { + const struct lmv_user_md *lum = spec->u.sp_ea.eadata; - fld_range_set_type(&range, LU_SEQ_RANGE_MDT); - rc = fld_server_lookup(info->mti_env, ss->ss_server_fld, - fid_seq(mdt_object_fid(parent)), &range); + /* Only new clients can create remote dir( >= 2.4) and + * striped dir(>= 2.6), old client will return -ENOTSUPP */ + if (!mdt_is_dne_client(exp)) + return -ENOTSUPP; + + if (le32_to_cpu(lum->lum_stripe_count) > 1 && + !mdt_is_striped_client(exp)) + return -ENOTSUPP; + + rc = mdt_remote_permission_check(info); if (rc != 0) return rc; - - if (range.lsr_index != 0) - return -EPERM; } - if (!mdt_is_dne_client(exp)) - return -ENOTSUPP; + if (info->mti_rr.rr_opcode == REINT_SETATTR) { + struct md_attr *ma = &info->mti_attr; - if (S_ISDIR(attr->la_mode) && spec->u.sp_ea.eadata != NULL && - spec->u.sp_ea.eadatalen != 0 && !mdt_is_striped_client(exp)) - return -ENOTSUPP; + if ((ma->ma_valid & MA_LMV)) { + rc = mdt_remote_permission_check(info); + if (rc != 0) + return rc; + } + } return 0; } +static int mdt_unlock_slaves(struct mdt_thread_info *mti, + struct mdt_object *obj, + struct ldlm_enqueue_info *einfo, + int decref) +{ + union ldlm_policy_data *policy = &mti->mti_policy; + struct mdt_lock_handle *lh = &mti->mti_lh[MDT_LH_LOCAL]; + struct lustre_handle_array *slave_locks = einfo->ei_cbdata; + int i; + + LASSERT(S_ISDIR(obj->mot_header.loh_attr)); + LASSERT(slave_locks); + + memset(policy, 0, sizeof(*policy)); + policy->l_inodebits.bits = einfo->ei_inodebits; + mdt_lock_handle_init(lh); + mdt_lock_reg_init(lh, einfo->ei_mode); + for (i = 0; i < slave_locks->ha_count; i++) { + if (test_bit(i, (void *)slave_locks->ha_map)) + lh->mlh_rreg_lh = slave_locks->ha_handles[i]; + else + lh->mlh_reg_lh = slave_locks->ha_handles[i]; + mdt_object_unlock(mti, NULL, lh, decref); + slave_locks->ha_handles[i].cookie = 0ull; + } + + return mo_object_unlock(mti->mti_env, mdt_object_child(obj), einfo, + policy); +} + +static inline int mdt_object_striped(struct mdt_thread_info *mti, + struct mdt_object *obj) +{ + int rc; + + if (!S_ISDIR(obj->mot_header.loh_attr)) + return 0; + + rc = mo_xattr_get(mti->mti_env, mdt_object_child(obj), &LU_BUF_NULL, + XATTR_NAME_LMV); + if (rc <= 0) + return rc == -ENODATA ? 0 : rc; + + return 1; +} + +/** + * Lock slave stripes if necessary, the lock handles of slave stripes + * will be stored in einfo->ei_cbdata. + **/ +static int mdt_lock_slaves(struct mdt_thread_info *mti, struct mdt_object *obj, + enum ldlm_mode mode, __u64 ibits, + struct ldlm_enqueue_info *einfo) +{ + union ldlm_policy_data *policy = &mti->mti_policy; + + LASSERT(S_ISDIR(obj->mot_header.loh_attr)); + + einfo->ei_type = LDLM_IBITS; + einfo->ei_mode = mode; + einfo->ei_cb_bl = mdt_remote_blocking_ast; + einfo->ei_cb_local_bl = mdt_blocking_ast; + einfo->ei_cb_cp = ldlm_completion_ast; + einfo->ei_enq_slave = 1; + einfo->ei_namespace = mti->mti_mdt->mdt_namespace; + einfo->ei_inodebits = ibits; + memset(policy, 0, sizeof(*policy)); + policy->l_inodebits.bits = ibits; + + return mo_object_lock(mti->mti_env, mdt_object_child(obj), NULL, einfo, + policy); +} + +static inline int mdt_reint_striped_lock(struct mdt_thread_info *info, + struct mdt_object *o, + struct mdt_lock_handle *lh, + __u64 ibits, + struct ldlm_enqueue_info *einfo, + bool cos_incompat) +{ + int rc; + + LASSERT(!mdt_object_remote(o)); + + memset(einfo, 0, sizeof(*einfo)); + + rc = mdt_reint_object_lock(info, o, lh, ibits, cos_incompat); + if (rc) + return rc; + + rc = mdt_object_striped(info, o); + if (rc != 1) { + if (rc < 0) + mdt_object_unlock(info, o, lh, rc); + return rc; + } + + rc = mdt_lock_slaves(info, o, lh->mlh_reg_mode, ibits, einfo); + if (rc) { + mdt_object_unlock(info, o, lh, rc); + if (rc == -EIO && OBD_FAIL_CHECK(OBD_FAIL_LFSCK_BAD_SLAVE_NAME)) + rc = 0; + } + + return rc; +} + +static inline void +mdt_reint_striped_unlock(struct mdt_thread_info *info, struct mdt_object *o, + struct mdt_lock_handle *lh, + struct ldlm_enqueue_info *einfo, int decref) +{ + if (einfo->ei_cbdata) + mdt_unlock_slaves(info, o, einfo, decref); + mdt_object_unlock(info, o, lh, decref); +} + /* * VBR: we save three versions in reply: * 0 - parent. Check that parent version is the same during replay. @@ -326,17 +414,17 @@ static int mdt_remote_permission(struct mdt_thread_info *info, * 2 - child. Version of child by FID. Must be ENOENT. It is mostly sanity * check. */ -static int mdt_md_create(struct mdt_thread_info *info) +static int mdt_create(struct mdt_thread_info *info) { - struct mdt_device *mdt = info->mti_mdt; - struct mdt_object *parent; - struct mdt_object *child; - struct mdt_lock_handle *lh; - struct mdt_body *repbody; - struct md_attr *ma = &info->mti_attr; - struct mdt_reint_record *rr = &info->mti_rr; - int rc; - ENTRY; + struct mdt_device *mdt = info->mti_mdt; + struct mdt_object *parent; + struct mdt_object *child; + struct mdt_lock_handle *lh; + struct mdt_body *repbody; + struct md_attr *ma = &info->mti_attr; + struct mdt_reint_record *rr = &info->mti_rr; + int rc; + ENTRY; DEBUG_REQ(D_INODE, mdt_info_req(info), "Create ("DNAME"->"DFID") " "in "DFID, @@ -356,8 +444,7 @@ static int mdt_md_create(struct mdt_thread_info *info) lh = &info->mti_lh[MDT_LH_PARENT]; mdt_lock_pdo_init(lh, LCK_PW, &rr->rr_name); - rc = mdt_object_lock(info, parent, lh, MDS_INODELOCK_UPDATE, - MDT_CROSS_LOCK); + rc = mdt_object_lock(info, parent, lh, MDS_INODELOCK_UPDATE); if (rc) GOTO(put_parent, rc); @@ -384,173 +471,119 @@ static int mdt_md_create(struct mdt_thread_info *info) mdt_enoent_version_save(info, 1); child = mdt_object_new(info->mti_env, mdt, rr->rr_fid2); - if (likely(!IS_ERR(child))) { - struct md_object *next = mdt_object_child(parent); - - rc = mdt_remote_permission(info, parent, child); - if (rc != 0) - GOTO(out_put_child, rc); - - ma->ma_need = MA_INODE; - ma->ma_valid = 0; - /* capa for cross-ref will be stored here */ - ma->ma_capa = req_capsule_server_get(info->mti_pill, - &RMF_CAPA1); - LASSERT(ma->ma_capa); - - mdt_fail_write(info->mti_env, info->mti_mdt->mdt_bottom, - OBD_FAIL_MDS_REINT_CREATE_WRITE); - - /* Version of child will be updated on disk. */ - tgt_vbr_obj_set(info->mti_env, mdt_obj2dt(child)); - rc = mdt_version_get_check_save(info, child, 2); - if (rc) - GOTO(out_put_child, rc); - - /* Let lower layer know current lock mode. */ - info->mti_spec.sp_cr_mode = - mdt_dlm_mode2mdl_mode(lh->mlh_pdo_mode); - - /* - * Do not perform lookup sanity check. We know that name does - * not exist. - */ - info->mti_spec.sp_cr_lookup = 0; - info->mti_spec.sp_feat = &dt_directory_features; - - rc = mdo_create(info->mti_env, next, &rr->rr_name, - mdt_object_child(child), &info->mti_spec, ma); - if (rc == 0) - rc = mdt_attr_get_complex(info, child, ma); - - if (rc == 0) { - /* Return fid & attr to client. */ - if (ma->ma_valid & MA_INODE) - mdt_pack_attr2body(info, repbody, &ma->ma_attr, - mdt_object_fid(child)); - } -out_put_child: - mdt_object_put(info->mti_env, child); - } else { - rc = PTR_ERR(child); - } - mdt_create_pack_capa(info, rc, child, repbody); -unlock_parent: - mdt_object_unlock(info, parent, lh, rc); -put_parent: - mdt_object_put(info->mti_env, parent); - RETURN(rc); -} + if (unlikely(IS_ERR(child))) + GOTO(unlock_parent, rc = PTR_ERR(child)); -static int mdt_unlock_slaves(struct mdt_thread_info *mti, - struct mdt_object *obj, __u64 ibits, - struct mdt_lock_handle *s0_lh, - struct mdt_object *s0_obj, - struct ldlm_enqueue_info *einfo) -{ - ldlm_policy_data_t *policy = &mti->mti_policy; - int rc; - ENTRY; + rc = mdt_remote_permission(info); + if (rc != 0) + GOTO(put_child, rc); - if (!S_ISDIR(obj->mot_header.loh_attr)) - RETURN(0); + ma->ma_need = MA_INODE; + ma->ma_valid = 0; - /* Unlock stripe 0 */ - if (s0_lh != NULL && lustre_handle_is_used(&s0_lh->mlh_reg_lh)) { - LASSERT(s0_obj != NULL); - mdt_object_unlock_put(mti, s0_obj, s0_lh, 1); - } + mdt_fail_write(info->mti_env, info->mti_mdt->mdt_bottom, + OBD_FAIL_MDS_REINT_CREATE_WRITE); - memset(policy, 0, sizeof(*policy)); - policy->l_inodebits.bits = ibits; + /* Version of child will be updated on disk. */ + tgt_vbr_obj_set(info->mti_env, mdt_obj2dt(child)); + rc = mdt_version_get_check_save(info, child, 2); + if (rc) + GOTO(put_child, rc); - rc = mo_object_unlock(mti->mti_env, mdt_object_child(obj), einfo, - policy); - RETURN(rc); -} + /* Let lower layer know current lock mode. */ + info->mti_spec.sp_cr_mode = mdt_dlm_mode2mdl_mode(lh->mlh_pdo_mode); -/** - * Lock slave stripes if necessary, the lock handles of slave stripes - * will be stored in einfo->ei_cbdata. - **/ -static int mdt_lock_slaves(struct mdt_thread_info *mti, struct mdt_object *obj, - ldlm_mode_t mode, __u64 ibits, - struct mdt_lock_handle *s0_lh, - struct mdt_object **s0_objp, - struct ldlm_enqueue_info *einfo) -{ - ldlm_policy_data_t *policy = &mti->mti_policy; - struct lu_buf *buf = &mti->mti_buf; - struct lmv_mds_md_v1 *lmv; - struct lu_fid *fid = &mti->mti_tmp_fid1; - int rc; - ENTRY; + /* + * Do not perform lookup sanity check. We know that name does + * not exist. + */ + info->mti_spec.sp_cr_lookup = 0; + info->mti_spec.sp_feat = &dt_directory_features; - if (!S_ISDIR(obj->mot_header.loh_attr)) - RETURN(0); + rc = mdo_create(info->mti_env, mdt_object_child(parent), &rr->rr_name, + mdt_object_child(child), &info->mti_spec, ma); + if (rc == 0) + rc = mdt_attr_get_complex(info, child, ma); - buf->lb_buf = mti->mti_xattr_buf; - buf->lb_len = sizeof(mti->mti_xattr_buf); - rc = mo_xattr_get(mti->mti_env, mdt_object_child(obj), buf, - XATTR_NAME_LMV); - if (rc == -ERANGE) { - rc = mdt_big_xattr_get(mti, obj, XATTR_NAME_LMV); - if (rc > 0) { - buf->lb_buf = mti->mti_big_lmm; - buf->lb_len = mti->mti_big_lmmsize; - } - } + if (rc < 0) + GOTO(put_child, rc); - if (rc == -ENODATA || rc == -ENOENT) - RETURN(0); + /* + * On DNE, we need to eliminate dependey between 'mkdir a' and + * 'mkdir a/b' if b is a striped directory, to achieve this, two + * things are done below: + * 1. save child and slaves lock. + * 2. if the child is a striped directory, relock parent so to + * compare against with COS locks to ensure parent was + * committed to disk. + */ + if (mdt_slc_is_enabled(mdt) && S_ISDIR(ma->ma_attr.la_mode)) { + struct mdt_lock_handle *lhc; + struct ldlm_enqueue_info *einfo = &info->mti_einfo; + bool cos_incompat; - if (rc <= 0) - RETURN(rc); + rc = mdt_object_striped(info, child); + if (rc < 0) + GOTO(put_child, rc); - lmv = buf->lb_buf; - if (le32_to_cpu(lmv->lmv_magic) != LMV_MAGIC_V1) - RETURN(-EINVAL); + cos_incompat = rc; + if (cos_incompat) { + if (!mdt_object_remote(parent)) { + mdt_object_unlock(info, parent, lh, 1); + mdt_lock_pdo_init(lh, LCK_PW, &rr->rr_name); + rc = mdt_reint_object_lock(info, parent, lh, + MDS_INODELOCK_UPDATE, + true); + if (rc) + GOTO(put_child, rc); + } + } - /* Sigh, 0_stripe and master object are different - * object, though they are in the same MDT, to avoid - * adding osd_object_lock here, so we will enqueue the - * stripe0 lock in MDT0 for now */ - fid_le_to_cpu(fid, &lmv->lmv_stripe_fids[0]); - *s0_objp = mdt_object_find_lock(mti, fid, s0_lh, ibits); - if (IS_ERR(*s0_objp)) - RETURN(PTR_ERR(*s0_objp)); + lhc = &info->mti_lh[MDT_LH_CHILD]; + mdt_lock_handle_init(lhc); + mdt_lock_reg_init(lhc, LCK_PW); + rc = mdt_reint_striped_lock(info, child, lhc, + MDS_INODELOCK_UPDATE, einfo, + cos_incompat); + if (rc) + GOTO(put_child, rc); - memset(einfo, 0, sizeof(*einfo)); - einfo->ei_type = LDLM_IBITS; - einfo->ei_mode = mode; - einfo->ei_cb_bl = mdt_remote_blocking_ast; - einfo->ei_cb_cp = ldlm_completion_ast; - einfo->ei_enq_slave = 1; - memset(policy, 0, sizeof(*policy)); - policy->l_inodebits.bits = ibits; + mdt_reint_striped_unlock(info, child, lhc, einfo, rc); + } - rc = mo_object_lock(mti->mti_env, mdt_object_child(obj), NULL, einfo, - policy); + /* Return fid & attr to client. */ + if (ma->ma_valid & MA_INODE) + mdt_pack_attr2body(info, repbody, &ma->ma_attr, + mdt_object_fid(child)); +put_child: + mdt_object_put(info->mti_env, child); +unlock_parent: + mdt_object_unlock(info, parent, lh, rc); +put_parent: + mdt_object_put(info->mti_env, parent); RETURN(rc); } -int mdt_attr_set(struct mdt_thread_info *info, struct mdt_object *mo, - struct md_attr *ma, int flags) +static int mdt_attr_set(struct mdt_thread_info *info, struct mdt_object *mo, + struct md_attr *ma) { struct mdt_lock_handle *lh; - int do_vbr = ma->ma_attr.la_valid & (LA_MODE|LA_UID|LA_GID|LA_FLAGS); + int do_vbr = ma->ma_attr.la_valid & + (LA_MODE | LA_UID | LA_GID | LA_PROJID | LA_FLAGS); __u64 lockpart = MDS_INODELOCK_UPDATE; struct ldlm_enqueue_info *einfo = &info->mti_einfo; - struct mdt_lock_handle *s0_lh; - struct mdt_object *s0_obj = NULL; + bool cos_incompat; int rc; ENTRY; - /* attr shouldn't be set on remote object */ - LASSERT(!mdt_object_remote(mo)); + rc = mdt_object_striped(info, mo); + if (rc < 0) + RETURN(rc); - lh = &info->mti_lh[MDT_LH_PARENT]; - mdt_lock_reg_init(lh, LCK_PW); + cos_incompat = rc; + + lh = &info->mti_lh[MDT_LH_PARENT]; + mdt_lock_reg_init(lh, LCK_PW); /* Even though the new MDT will grant PERM lock to the old * client, but the old client will almost ignore that during @@ -559,54 +592,41 @@ int mdt_attr_set(struct mdt_thread_info *info, struct mdt_object *mo, if (ma->ma_attr.la_valid & (LA_MODE|LA_UID|LA_GID)) lockpart |= MDS_INODELOCK_LOOKUP | MDS_INODELOCK_PERM; - rc = mdt_object_lock(info, mo, lh, lockpart, MDT_LOCAL_LOCK); - if (rc != 0) - RETURN(rc); - - s0_lh = &info->mti_lh[MDT_LH_LOCAL]; - mdt_lock_reg_init(s0_lh, LCK_EX); - rc = mdt_lock_slaves(info, mo, LCK_PW, lockpart, s0_lh, &s0_obj, einfo); + rc = mdt_reint_striped_lock(info, mo, lh, lockpart, einfo, + cos_incompat); if (rc != 0) - GOTO(out_unlock, rc); - - if (mdt_object_exists(mo) == 0) - GOTO(out_unlock, rc = -ENOENT); - - /* all attrs are packed into mti_attr in unpack_setattr */ - mdt_fail_write(info->mti_env, info->mti_mdt->mdt_bottom, - OBD_FAIL_MDS_REINT_SETATTR_WRITE); + RETURN(rc); - /* This is only for set ctime when rename's source is on remote MDS. */ - if (unlikely(ma->ma_attr.la_valid == LA_CTIME)) - ma->ma_attr_flags |= MDS_VTX_BYPASS; + /* all attrs are packed into mti_attr in unpack_setattr */ + mdt_fail_write(info->mti_env, info->mti_mdt->mdt_bottom, + OBD_FAIL_MDS_REINT_SETATTR_WRITE); - /* VBR: update version if attr changed are important for recovery */ - if (do_vbr) { - /* update on-disk version of changed object */ + /* VBR: update version if attr changed are important for recovery */ + if (do_vbr) { + /* update on-disk version of changed object */ tgt_vbr_obj_set(info->mti_env, mdt_obj2dt(mo)); - rc = mdt_version_get_check_save(info, mo, 0); - if (rc) - GOTO(out_unlock, rc); - } + rc = mdt_version_get_check_save(info, mo, 0); + if (rc) + GOTO(out_unlock, rc); + } /* Ensure constant striping during chown(). See LU-2789. */ - if (ma->ma_attr.la_valid & (LA_UID|LA_GID)) + if (ma->ma_attr.la_valid & (LA_UID|LA_GID|LA_PROJID)) mutex_lock(&mo->mot_lov_mutex); - /* all attrs are packed into mti_attr in unpack_setattr */ - rc = mo_attr_set(info->mti_env, mdt_object_child(mo), ma); + /* all attrs are packed into mti_attr in unpack_setattr */ + rc = mo_attr_set(info->mti_env, mdt_object_child(mo), ma); - if (ma->ma_attr.la_valid & (LA_UID|LA_GID)) + if (ma->ma_attr.la_valid & (LA_UID|LA_GID|LA_PROJID)) mutex_unlock(&mo->mot_lov_mutex); - if (rc != 0) - GOTO(out_unlock, rc); - - EXIT; + if (rc != 0) + GOTO(out_unlock, rc); + mdt_dom_obj_lvb_update(info->mti_env, mo, false); + EXIT; out_unlock: - mdt_unlock_slaves(info, mo, lockpart, s0_lh, s0_obj, einfo); - mdt_object_unlock(info, mo, lh, rc); - return rc; + mdt_reint_striped_unlock(info, mo, lh, einfo, rc); + return rc; } /** @@ -633,21 +653,12 @@ int mdt_add_dirty_flag(struct mdt_thread_info *info, struct mdt_object *mo, /* If an up2date copy exists in the backend, add dirty flag */ if ((ma->ma_valid & MA_HSM) && (ma->ma_hsm.mh_flags & HS_EXISTS) && !(ma->ma_hsm.mh_flags & (HS_DIRTY|HS_RELEASED))) { - struct mdt_lock_handle *lh = &info->mti_lh[MDT_LH_CHILD]; - ma->ma_hsm.mh_flags |= HS_DIRTY; - mdt_lock_reg_init(lh, LCK_PW); - rc = mdt_object_lock(info, mo, lh, MDS_INODELOCK_XATTR, - MDT_LOCAL_LOCK); - if (rc != 0) - RETURN(rc); - rc = mdt_hsm_attr_set(info, mo, &ma->ma_hsm); if (rc) CERROR("file attribute change error for "DFID": %d\n", PFID(mdt_object_fid(mo)), rc); - mdt_object_unlock(info, mo, lh, rc); } RETURN(rc); @@ -659,111 +670,98 @@ static int mdt_reint_setattr(struct mdt_thread_info *info, struct md_attr *ma = &info->mti_attr; struct mdt_reint_record *rr = &info->mti_rr; struct ptlrpc_request *req = mdt_info_req(info); - struct mdt_export_data *med = &req->rq_export->exp_mdt_data; - struct mdt_file_data *mfd; struct mdt_object *mo; struct mdt_body *repbody; - int som_au, rc, rc2; + int rc, rc2; ENTRY; DEBUG_REQ(D_INODE, req, "setattr "DFID" %x", PFID(rr->rr_fid1), (unsigned int)ma->ma_attr.la_valid); - if (info->mti_dlm_req) - ldlm_request_cancel(req, info->mti_dlm_req, 0); + if (info->mti_dlm_req) + ldlm_request_cancel(req, info->mti_dlm_req, 0, LATF_SKIP); repbody = req_capsule_server_get(info->mti_pill, &RMF_MDT_BODY); mo = mdt_object_find(info->mti_env, info->mti_mdt, rr->rr_fid1); if (IS_ERR(mo)) GOTO(out, rc = PTR_ERR(mo)); - /* start a log jounal handle if needed */ - if (!(mdt_conn_flags(info) & OBD_CONNECT_SOM)) { - if ((ma->ma_attr.la_valid & LA_SIZE) || - (rr->rr_flags & MRF_OPEN_TRUNC)) { - /* Check write access for the O_TRUNC case */ - if (mdt_write_read(mo) < 0) - GOTO(out_put, rc = -ETXTBSY); - } - } else if (info->mti_ioepoch && - (info->mti_ioepoch->flags & MF_EPOCH_OPEN)) { - /* Truncate case. IOEpoch is opened. */ - rc = mdt_write_get(mo); - if (rc) - GOTO(out_put, rc); - - mfd = mdt_mfd_new(med); - if (mfd == NULL) { - mdt_write_put(mo); - GOTO(out_put, rc = -ENOMEM); - } + if (!mdt_object_exists(mo)) + GOTO(out_put, rc = -ENOENT); - mdt_ioepoch_open(info, mo, 0); - repbody->ioepoch = mo->mot_ioepoch; + if (mdt_object_remote(mo)) + GOTO(out_put, rc = -EREMOTE); - mdt_object_get(info->mti_env, mo); - mdt_mfd_set_mode(mfd, MDS_FMODE_TRUNC); - mfd->mfd_object = mo; - mfd->mfd_xid = req->rq_xid; + if (ma->ma_attr.la_valid & LA_SIZE || rr->rr_flags & MRF_OPEN_TRUNC) { + /* Check write access for the O_TRUNC case */ + if (mdt_write_read(mo) < 0) + GOTO(out_put, rc = -ETXTBSY); - spin_lock(&med->med_open_lock); - cfs_list_add(&mfd->mfd_list, &med->med_open_head); - spin_unlock(&med->med_open_lock); - repbody->handle.cookie = mfd->mfd_handle.h_cookie; - } + /* LU-10286: compatibility check for FLR. + * Please check the comment in mdt_finish_open() for details */ + if (!exp_connect_flr(info->mti_exp)) { + rc = mdt_big_xattr_get(info, mo, XATTR_NAME_LOV); + if (rc < 0 && rc != -ENODATA) + GOTO(out_put, rc); - som_au = info->mti_ioepoch && info->mti_ioepoch->flags & MF_SOM_CHANGE; - if (som_au) { - /* SOM Attribute update case. Find the proper mfd and update - * SOM attributes on the proper object. */ - LASSERT(mdt_conn_flags(info) & OBD_CONNECT_SOM); - LASSERT(info->mti_ioepoch); - - spin_lock(&med->med_open_lock); - mfd = mdt_handle2mfd(med, &info->mti_ioepoch->handle, - req_is_replay(req)); - if (mfd == NULL) { - spin_unlock(&med->med_open_lock); - CDEBUG(D_INODE, "no handle for file close: " - "fid = "DFID": cookie = "LPX64"\n", - PFID(info->mti_rr.rr_fid1), - info->mti_ioepoch->handle.cookie); - GOTO(out_put, rc = -ESTALE); - } - LASSERT(mfd->mfd_mode == MDS_FMODE_SOM); - LASSERT(!(info->mti_ioepoch->flags & MF_EPOCH_CLOSE)); - - class_handle_unhash(&mfd->mfd_handle); - cfs_list_del_init(&mfd->mfd_list); - spin_unlock(&med->med_open_lock); - - mdt_mfd_close(info, mfd); - } else if ((ma->ma_valid & MA_INODE) && ma->ma_attr.la_valid) { - LASSERT((ma->ma_valid & MA_LOV) == 0); - rc = mdt_attr_set(info, mo, ma, rr->rr_flags); - if (rc) - GOTO(out_put, rc); - } else if ((ma->ma_valid & MA_LOV) && (ma->ma_valid & MA_INODE)) { - struct lu_buf *buf = &info->mti_buf; - LASSERT(ma->ma_attr.la_valid == 0); - buf->lb_buf = ma->ma_lmm; - buf->lb_len = ma->ma_lmm_size; - rc = mo_xattr_set(info->mti_env, mdt_object_child(mo), - buf, XATTR_NAME_LOV, 0); + if (rc > 0 && mdt_lmm_is_flr(info->mti_big_lmm)) + GOTO(out_put, rc = -EOPNOTSUPP); + } + + /* For truncate, the file size sent from client + * is believable, but the blocks are incorrect, + * which makes the block size in LSOM attribute + * inconsisent with the real block size. + */ + rc = mdt_lsom_update(info, mo, true); + if (rc) + GOTO(out_put, rc); + } + + if ((ma->ma_valid & MA_INODE) && ma->ma_attr.la_valid) { + if (ma->ma_valid & MA_LOV) + GOTO(out_put, rc = -EPROTO); + + rc = mdt_attr_set(info, mo, ma); if (rc) GOTO(out_put, rc); - } else if ((ma->ma_valid & MA_LMV) && (ma->ma_valid & MA_INODE)) { + } else if ((ma->ma_valid & (MA_LOV | MA_LMV)) && + (ma->ma_valid & MA_INODE)) { struct lu_buf *buf = &info->mti_buf; + struct mdt_lock_handle *lh; + + rc = mdt_remote_permission(info); + if (rc < 0) + GOTO(out_put, rc); + + if (ma->ma_attr.la_valid != 0) + GOTO(out_put, rc = -EPROTO); + + lh = &info->mti_lh[MDT_LH_PARENT]; + mdt_lock_reg_init(lh, LCK_PW); + + rc = mdt_object_lock(info, mo, lh, MDS_INODELOCK_XATTR); + if (rc != 0) + GOTO(out_put, rc); + + if (ma->ma_valid & MA_LOV) { + buf->lb_buf = ma->ma_lmm; + buf->lb_len = ma->ma_lmm_size; + } else { + buf->lb_buf = ma->ma_lmv; + buf->lb_len = ma->ma_lmv_size; + } + rc = mo_xattr_set(info->mti_env, mdt_object_child(mo), buf, + (ma->ma_valid & MA_LOV) ? + XATTR_NAME_LOV : XATTR_NAME_DEFAULT_LMV, + 0); - LASSERT(ma->ma_attr.la_valid == 0); - buf->lb_buf = ma->ma_lmv; - buf->lb_len = ma->ma_lmv_size; - rc = mo_xattr_set(info->mti_env, mdt_object_child(mo), - buf, XATTR_NAME_DEFAULT_LMV, 0); + mdt_object_unlock(info, mo, lh, rc); if (rc) GOTO(out_put, rc); - } else - LBUG(); + } else { + GOTO(out_put, rc = -EPROTO); + } /* If file data is modified, add the dirty flag */ if (ma->ma_attr_flags & MDS_DATA_MODIFIED) @@ -777,26 +775,11 @@ static int mdt_reint_setattr(struct mdt_thread_info *info, mdt_pack_attr2body(info, repbody, &ma->ma_attr, mdt_object_fid(mo)); - if (info->mti_mdt->mdt_lut.lut_oss_capa && - exp_connect_flags(info->mti_exp) & OBD_CONNECT_OSS_CAPA && - S_ISREG(lu_object_attr(&mo->mot_obj)) && - (ma->ma_attr.la_valid & LA_SIZE) && !som_au) { - struct lustre_capa *capa; - - capa = req_capsule_server_get(info->mti_pill, &RMF_CAPA2); - LASSERT(capa); - capa->lc_opc = CAPA_OPC_OSS_DEFAULT | CAPA_OPC_OSS_TRUNC; - rc = mo_capa_get(info->mti_env, mdt_object_child(mo), capa, 0); - if (rc) - GOTO(out_put, rc); - repbody->valid |= OBD_MD_FLOSSCAPA; - } - - EXIT; + EXIT; out_put: - mdt_object_put(info->mti_env, mo); + mdt_object_put(info->mti_env, mo); out: - if (rc == 0) + if (rc == 0) mdt_counter_incr(req, LPROC_MDT_SETATTR); mdt_client_compatibility(info); @@ -816,8 +799,9 @@ static int mdt_reint_create(struct mdt_thread_info *info, if (OBD_FAIL_CHECK(OBD_FAIL_MDS_REINT_CREATE)) RETURN(err_serious(-ESTALE)); - if (info->mti_dlm_req) - ldlm_request_cancel(mdt_info_req(info), info->mti_dlm_req, 0); + if (info->mti_dlm_req) + ldlm_request_cancel(mdt_info_req(info), + info->mti_dlm_req, 0, LATF_SKIP); if (!lu_name_is_valid(&info->mti_rr.rr_name)) RETURN(-EPROTO); @@ -842,7 +826,7 @@ static int mdt_reint_create(struct mdt_thread_info *info, RETURN(err_serious(-EOPNOTSUPP)); } - rc = mdt_md_create(info); + rc = mdt_create(info); RETURN(rc); } @@ -851,58 +835,56 @@ static int mdt_reint_create(struct mdt_thread_info *info, * Version of child is getting and checking during its lookup. If */ static int mdt_reint_unlink(struct mdt_thread_info *info, - struct mdt_lock_handle *lhc) + struct mdt_lock_handle *lhc) { - struct mdt_reint_record *rr = &info->mti_rr; - struct ptlrpc_request *req = mdt_info_req(info); - struct md_attr *ma = &info->mti_attr; - struct lu_fid *child_fid = &info->mti_tmp_fid1; - struct mdt_object *mp; - struct mdt_object *mc; - struct mdt_lock_handle *parent_lh; - struct mdt_lock_handle *child_lh; + struct mdt_reint_record *rr = &info->mti_rr; + struct ptlrpc_request *req = mdt_info_req(info); + struct md_attr *ma = &info->mti_attr; + struct lu_fid *child_fid = &info->mti_tmp_fid1; + struct mdt_object *mp; + struct mdt_object *mc; + struct mdt_lock_handle *parent_lh; + struct mdt_lock_handle *child_lh; struct ldlm_enqueue_info *einfo = &info->mti_einfo; - struct mdt_lock_handle *s0_lh = NULL; - struct mdt_object *s0_obj = NULL; - int rc; - int no_name = 0; + __u64 lock_ibits; + bool cos_incompat = false, discard = false; + int no_name = 0; + int rc; + ENTRY; DEBUG_REQ(D_INODE, req, "unlink "DFID"/"DNAME"", PFID(rr->rr_fid1), PNAME(&rr->rr_name)); - if (info->mti_dlm_req) - ldlm_request_cancel(req, info->mti_dlm_req, 0); + if (info->mti_dlm_req) + ldlm_request_cancel(req, info->mti_dlm_req, 0, LATF_SKIP); - if (OBD_FAIL_CHECK(OBD_FAIL_MDS_REINT_UNLINK)) - RETURN(err_serious(-ENOENT)); + if (OBD_FAIL_CHECK(OBD_FAIL_MDS_REINT_UNLINK)) + RETURN(err_serious(-ENOENT)); if (!fid_is_md_operative(rr->rr_fid1)) RETURN(-EPERM); - /* - * step 1: Found the parent. - */ mp = mdt_object_find(info->mti_env, info->mti_mdt, rr->rr_fid1); - if (IS_ERR(mp)) { - rc = PTR_ERR(mp); - GOTO(out, rc); + if (IS_ERR(mp)) + RETURN(PTR_ERR(mp)); + + if (mdt_object_remote(mp)) { + cos_incompat = true; + } else { + rc = mdt_version_get_check_save(info, mp, 0); + if (rc) + GOTO(put_parent, rc); } +relock: parent_lh = &info->mti_lh[MDT_LH_PARENT]; mdt_lock_pdo_init(parent_lh, LCK_PW, &rr->rr_name); - rc = mdt_object_lock(info, mp, parent_lh, MDS_INODELOCK_UPDATE, - MDT_CROSS_LOCK); + rc = mdt_reint_object_lock(info, mp, parent_lh, MDS_INODELOCK_UPDATE, + cos_incompat); if (rc != 0) GOTO(put_parent, rc); - if (!mdt_object_remote(mp)) { - rc = mdt_version_get_check_save(info, mp, 0); - if (rc) - GOTO(unlock_parent, rc); - } - - /* step 2: find & lock the child */ /* lookup child object along with version checking */ fid_zero(child_fid); rc = mdt_lookup_version_check(info, mp, &rr->rr_name, child_fid, 1); @@ -939,8 +921,38 @@ static int mdt_reint_unlink(struct mdt_thread_info *info, if (IS_ERR(mc)) GOTO(unlock_parent, rc = PTR_ERR(mc)); - child_lh = &info->mti_lh[MDT_LH_CHILD]; - mdt_lock_reg_init(child_lh, LCK_EX); + if (!cos_incompat) { + rc = mdt_object_striped(info, mc); + if (rc < 0) + GOTO(unlock_parent, rc = PTR_ERR(mc)); + + cos_incompat = rc; + if (cos_incompat) { + mdt_object_put(info->mti_env, mc); + mdt_object_unlock(info, mp, parent_lh, -EAGAIN); + goto relock; + } + } + + child_lh = &info->mti_lh[MDT_LH_CHILD]; + mdt_lock_reg_init(child_lh, LCK_EX); + if (info->mti_spec.sp_rm_entry) { + struct lu_ucred *uc = mdt_ucred(info); + + if (!mdt_is_dne_client(req->rq_export)) + /* Return -ENOTSUPP for old client */ + GOTO(put_child, rc = -ENOTSUPP); + + if (!md_capable(uc, CFS_CAP_SYS_ADMIN)) + GOTO(put_child, rc = -EPERM); + + ma->ma_need = MA_INODE; + ma->ma_valid = 0; + rc = mdo_unlink(info->mti_env, mdt_object_child(mp), + NULL, &rr->rr_name, ma, no_name); + GOTO(put_child, rc); + } + if (mdt_object_remote(mc)) { struct mdt_body *repbody; @@ -958,52 +970,39 @@ static int mdt_reint_unlink(struct mdt_thread_info *info, /* Return -ENOTSUPP for old client */ GOTO(put_child, rc = -ENOTSUPP); - if (info->mti_spec.sp_rm_entry) { - struct lu_ucred *uc = mdt_ucred(info); - - if (!md_capable(uc, CFS_CAP_SYS_ADMIN)) { - CERROR("%s: unlink remote entry is only " - "permitted for administrator: rc = %d\n", - mdt_obd_name(info->mti_mdt), - -EPERM); - GOTO(put_child, rc = -EPERM); - } - - ma->ma_need = MA_INODE; - ma->ma_valid = 0; - mdt_set_capainfo(info, 1, child_fid, BYPASS_CAPA); - rc = mdo_unlink(info->mti_env, mdt_object_child(mp), - NULL, &rr->rr_name, ma, no_name); - GOTO(put_child, rc); - } /* Revoke the LOOKUP lock of the remote object granted by * this MDT. Since the unlink will happen on another MDT, * it will release the LOOKUP lock right away. Then What * would happen if another client try to grab the LOOKUP * lock at the same time with unlink XXX */ - mdt_object_lock(info, mc, child_lh, MDS_INODELOCK_LOOKUP, - MDT_CROSS_LOCK); + mdt_object_lock(info, mc, child_lh, MDS_INODELOCK_LOOKUP); repbody = req_capsule_server_get(info->mti_pill, &RMF_MDT_BODY); LASSERT(repbody != NULL); - repbody->fid1 = *mdt_object_fid(mc); - repbody->valid |= (OBD_MD_FLID | OBD_MD_MDS); + repbody->mbo_fid1 = *mdt_object_fid(mc); + repbody->mbo_valid |= (OBD_MD_FLID | OBD_MD_MDS); GOTO(unlock_child, rc = -EREMOTE); - } else if (info->mti_spec.sp_rm_entry) { - rc = -EPERM; - CDEBUG(D_INFO, "%s: no rm_entry on local dir '"DNAME"': " - "rc = %d\n", - mdt_obd_name(info->mti_mdt), PNAME(&rr->rr_name), rc); - GOTO(put_child, rc); } - /* We used to acquire MDS_INODELOCK_FULL here but we can't do * this now because a running HSM restore on the child (unlink * victim) will hold the layout lock. See LU-4002. */ - rc = mdt_object_lock(info, mc, child_lh, - MDS_INODELOCK_LOOKUP | MDS_INODELOCK_UPDATE, - MDT_CROSS_LOCK); + lock_ibits = MDS_INODELOCK_LOOKUP | MDS_INODELOCK_UPDATE; + if (mdt_object_remote(mp)) { + /* Enqueue lookup lock from parent MDT */ + rc = mdt_remote_object_lock(info, mp, mdt_object_fid(mc), + &child_lh->mlh_rreg_lh, + child_lh->mlh_rreg_mode, + MDS_INODELOCK_LOOKUP, false); + if (rc != ELDLM_OK) + GOTO(put_child, rc); + + lock_ibits &= ~MDS_INODELOCK_LOOKUP; + } + + rc = mdt_reint_striped_lock(info, mc, child_lh, lock_ibits, einfo, + cos_incompat); if (rc != 0) GOTO(put_child, rc); + /* * Now we can only make sure we need MA_INODE, in mdd layer, will check * whether need MA_LOV and MA_COOKIE. @@ -1011,63 +1010,61 @@ static int mdt_reint_unlink(struct mdt_thread_info *info, ma->ma_need = MA_INODE; ma->ma_valid = 0; - s0_lh = &info->mti_lh[MDT_LH_LOCAL]; - mdt_lock_reg_init(s0_lh, LCK_EX); - rc = mdt_lock_slaves(info, mc, LCK_EX, MDS_INODELOCK_UPDATE, s0_lh, - &s0_obj, einfo); - if (rc != 0) - GOTO(unlock_child, rc); - mdt_fail_write(info->mti_env, info->mti_mdt->mdt_bottom, OBD_FAIL_MDS_REINT_UNLINK_WRITE); /* save version when object is locked */ mdt_version_get_save(info, mc, 1); - mdt_set_capainfo(info, 1, child_fid, BYPASS_CAPA); - mutex_lock(&mc->mot_lov_mutex); rc = mdo_unlink(info->mti_env, mdt_object_child(mp), mdt_object_child(mc), &rr->rr_name, ma, no_name); mutex_unlock(&mc->mot_lov_mutex); + if (rc != 0) + GOTO(unlock_child, rc); - if (rc == 0 && !lu_object_is_dying(&mc->mot_header)) + if (!lu_object_is_dying(&mc->mot_header)) { rc = mdt_attr_get_complex(info, mc, ma); - if (rc == 0) - mdt_handle_last_unlink(info, mc, ma); + if (rc) + GOTO(out_stat, rc); + } else { + discard = true; + } + mdt_handle_last_unlink(info, mc, ma); - if (ma->ma_valid & MA_INODE) { - switch (ma->ma_attr.la_mode & S_IFMT) { - case S_IFDIR: +out_stat: + if (ma->ma_valid & MA_INODE) { + switch (ma->ma_attr.la_mode & S_IFMT) { + case S_IFDIR: mdt_counter_incr(req, LPROC_MDT_RMDIR); - break; - case S_IFREG: - case S_IFLNK: - case S_IFCHR: - case S_IFBLK: - case S_IFIFO: - case S_IFSOCK: + break; + case S_IFREG: + case S_IFLNK: + case S_IFCHR: + case S_IFBLK: + case S_IFIFO: + case S_IFSOCK: mdt_counter_incr(req, LPROC_MDT_UNLINK); - break; - default: - LASSERTF(0, "bad file type %o unlinking\n", - ma->ma_attr.la_mode); - } - } + break; + default: + LASSERTF(0, "bad file type %o unlinking\n", + ma->ma_attr.la_mode); + } + } - EXIT; + EXIT; unlock_child: - mdt_unlock_slaves(info, mc, MDS_INODELOCK_UPDATE, s0_lh, s0_obj, einfo); - mdt_object_unlock(info, mc, child_lh, rc); + mdt_reint_striped_unlock(info, mc, child_lh, einfo, rc); put_child: + if (discard) + mdt_dom_check_and_discard(info, mc); mdt_object_put(info->mti_env, mc); unlock_parent: mdt_object_unlock(info, mp, parent_lh, rc); put_parent: mdt_object_put(info->mti_env, mp); -out: return rc; } @@ -1076,115 +1073,117 @@ out: * name. */ static int mdt_reint_link(struct mdt_thread_info *info, - struct mdt_lock_handle *lhc) + struct mdt_lock_handle *lhc) { - struct mdt_reint_record *rr = &info->mti_rr; - struct ptlrpc_request *req = mdt_info_req(info); - struct md_attr *ma = &info->mti_attr; - struct mdt_object *ms; - struct mdt_object *mp; - struct mdt_lock_handle *lhs; - struct mdt_lock_handle *lhp; - int rc; - ENTRY; + struct mdt_reint_record *rr = &info->mti_rr; + struct ptlrpc_request *req = mdt_info_req(info); + struct md_attr *ma = &info->mti_attr; + struct mdt_object *ms; + struct mdt_object *mp; + struct mdt_lock_handle *lhs; + struct mdt_lock_handle *lhp; + bool cos_incompat; + int rc; + ENTRY; DEBUG_REQ(D_INODE, req, "link "DFID" to "DFID"/"DNAME, PFID(rr->rr_fid1), PFID(rr->rr_fid2), PNAME(&rr->rr_name)); - if (OBD_FAIL_CHECK(OBD_FAIL_MDS_REINT_LINK)) - RETURN(err_serious(-ENOENT)); + if (OBD_FAIL_CHECK(OBD_FAIL_MDS_REINT_LINK)) + RETURN(err_serious(-ENOENT)); - if (info->mti_dlm_req) - ldlm_request_cancel(req, info->mti_dlm_req, 0); + if (info->mti_dlm_req) + ldlm_request_cancel(req, info->mti_dlm_req, 0, LATF_SKIP); - /* Invalid case so return error immediately instead of - * processing it */ - if (lu_fid_eq(rr->rr_fid1, rr->rr_fid2)) - RETURN(-EPERM); + /* Invalid case so return error immediately instead of + * processing it */ + if (lu_fid_eq(rr->rr_fid1, rr->rr_fid2)) + RETURN(-EPERM); if (!fid_is_md_operative(rr->rr_fid1) || !fid_is_md_operative(rr->rr_fid2)) RETURN(-EPERM); - /* step 1: find & lock the target parent dir */ - lhp = &info->mti_lh[MDT_LH_PARENT]; - mdt_lock_pdo_init(lhp, LCK_PW, &rr->rr_name); - mp = mdt_object_find_lock(info, rr->rr_fid2, lhp, - MDS_INODELOCK_UPDATE); - if (IS_ERR(mp)) - RETURN(PTR_ERR(mp)); - - rc = mdt_version_get_check_save(info, mp, 0); - if (rc) - GOTO(out_unlock_parent, rc); + /* step 1: find target parent dir */ + mp = mdt_object_find(info->mti_env, info->mti_mdt, rr->rr_fid2); + if (IS_ERR(mp)) + RETURN(PTR_ERR(mp)); - /* step 2: find & lock the source */ - lhs = &info->mti_lh[MDT_LH_CHILD]; - mdt_lock_reg_init(lhs, LCK_EX); + rc = mdt_version_get_check_save(info, mp, 0); + if (rc) + GOTO(put_parent, rc); - ms = mdt_object_find(info->mti_env, info->mti_mdt, rr->rr_fid1); - if (IS_ERR(ms)) - GOTO(out_unlock_parent, rc = PTR_ERR(ms)); + /* step 2: find source */ + ms = mdt_object_find(info->mti_env, info->mti_mdt, rr->rr_fid1); + if (IS_ERR(ms)) + GOTO(put_parent, rc = PTR_ERR(ms)); if (!mdt_object_exists(ms)) { - mdt_object_put(info->mti_env, ms); CDEBUG(D_INFO, "%s: "DFID" does not exist.\n", mdt_obd_name(info->mti_mdt), PFID(rr->rr_fid1)); - GOTO(out_unlock_parent, rc = -ENOENT); + GOTO(put_source, rc = -ENOENT); } - if (mdt_object_remote(ms)) { - mdt_object_put(info->mti_env, ms); - CERROR("%s: source inode "DFID" on remote MDT from "DFID"\n", - mdt_obd_name(info->mti_mdt), PFID(rr->rr_fid1), - PFID(rr->rr_fid2)); - GOTO(out_unlock_parent, rc = -EXDEV); - } + cos_incompat = (mdt_object_remote(mp) || mdt_object_remote(ms)); + + lhp = &info->mti_lh[MDT_LH_PARENT]; + mdt_lock_pdo_init(lhp, LCK_PW, &rr->rr_name); + rc = mdt_reint_object_lock(info, mp, lhp, MDS_INODELOCK_UPDATE, + cos_incompat); + if (rc != 0) + GOTO(put_source, rc); + + OBD_FAIL_TIMEOUT(OBD_FAIL_MDS_RENAME3, 5); - rc = mdt_object_lock(info, ms, lhs, MDS_INODELOCK_UPDATE | - MDS_INODELOCK_XATTR, MDT_LOCAL_LOCK); - if (rc != 0) { - mdt_object_put(info->mti_env, ms); - GOTO(out_unlock_parent, rc); - } + lhs = &info->mti_lh[MDT_LH_CHILD]; + mdt_lock_reg_init(lhs, LCK_EX); + rc = mdt_reint_object_lock(info, ms, lhs, + MDS_INODELOCK_UPDATE | MDS_INODELOCK_XATTR, + cos_incompat); + if (rc != 0) + GOTO(unlock_parent, rc); - /* step 3: link it */ - mdt_fail_write(info->mti_env, info->mti_mdt->mdt_bottom, - OBD_FAIL_MDS_REINT_LINK_WRITE); + /* step 3: link it */ + mdt_fail_write(info->mti_env, info->mti_mdt->mdt_bottom, + OBD_FAIL_MDS_REINT_LINK_WRITE); tgt_vbr_obj_set(info->mti_env, mdt_obj2dt(ms)); - rc = mdt_version_get_check_save(info, ms, 1); - if (rc) - GOTO(out_unlock_child, rc); + rc = mdt_version_get_check_save(info, ms, 1); + if (rc) + GOTO(unlock_source, rc); - /** check target version by name during replay */ + /** check target version by name during replay */ rc = mdt_lookup_version_check(info, mp, &rr->rr_name, &info->mti_tmp_fid1, 2); - if (rc != 0 && rc != -ENOENT) - GOTO(out_unlock_child, rc); - /* save version of file name for replay, it must be ENOENT here */ - if (!req_is_replay(mdt_info_req(info))) { + if (rc != 0 && rc != -ENOENT) + GOTO(unlock_source, rc); + /* save version of file name for replay, it must be ENOENT here */ + if (!req_is_replay(mdt_info_req(info))) { if (rc != -ENOENT) { CDEBUG(D_INFO, "link target "DNAME" existed!\n", PNAME(&rr->rr_name)); - GOTO(out_unlock_child, rc = -EEXIST); + GOTO(unlock_source, rc = -EEXIST); } - info->mti_ver[2] = ENOENT_VERSION; - mdt_version_save(mdt_info_req(info), info->mti_ver[2], 2); - } + info->mti_ver[2] = ENOENT_VERSION; + mdt_version_save(mdt_info_req(info), info->mti_ver[2], 2); + } rc = mdo_link(info->mti_env, mdt_object_child(mp), - mdt_object_child(ms), &rr->rr_name, ma); + mdt_object_child(ms), &rr->rr_name, ma); - if (rc == 0) + if (rc == 0) mdt_counter_incr(req, LPROC_MDT_LINK); - EXIT; -out_unlock_child: - mdt_object_unlock_put(info, ms, lhs, rc); -out_unlock_parent: - mdt_object_unlock_put(info, mp, lhp, rc); - return rc; + EXIT; +unlock_source: + mdt_object_unlock(info, ms, lhs, rc); +unlock_parent: + mdt_object_unlock(info, mp, lhp, rc); +put_source: + mdt_object_put(info->mti_env, ms); +put_parent: + mdt_object_put(info->mti_env, mp); + return rc; } /** * lock the part of the directory according to the hash of the name @@ -1192,11 +1191,13 @@ out_unlock_parent: */ static int mdt_pdir_hash_lock(struct mdt_thread_info *info, struct mdt_lock_handle *lh, - struct mdt_object *obj, __u64 ibits) + struct mdt_object *obj, __u64 ibits, + bool cos_incompat) { struct ldlm_res_id *res = &info->mti_res_id; struct ldlm_namespace *ns = info->mti_mdt->mdt_namespace; - ldlm_policy_data_t *policy = &info->mti_policy; + union ldlm_policy_data *policy = &info->mti_policy; + __u64 dlmflags = LDLM_FL_LOCAL_ONLY | LDLM_FL_ATOMIC_CB; int rc; /* @@ -1207,40 +1208,29 @@ static int mdt_pdir_hash_lock(struct mdt_thread_info *info, fid_build_pdo_res_name(mdt_object_fid(obj), lh->mlh_pdo_hash, res); memset(policy, 0, sizeof(*policy)); policy->l_inodebits.bits = ibits; + if (cos_incompat && + (lh->mlh_reg_mode == LCK_PW || lh->mlh_reg_mode == LCK_EX)) + dlmflags |= LDLM_FL_COS_INCOMPAT; /* * Use LDLM_FL_LOCAL_ONLY for this lock. We do not know yet if it is * going to be sent to client. If it is - mdt_intent_policy() path will * fix it up and turn FL_LOCAL flag off. */ rc = mdt_fid_lock(ns, &lh->mlh_reg_lh, lh->mlh_reg_mode, policy, - res, LDLM_FL_LOCAL_ONLY | LDLM_FL_ATOMIC_CB, - &info->mti_exp->exp_handle.h_cookie); + res, dlmflags, &info->mti_exp->exp_handle.h_cookie); return rc; } -enum mdt_rename_lock { - MRL_RENAME, - MRL_MIGRATE, -}; - /** - * Get BFL lock for rename or migrate process, right now, it does not support - * cross-MDT rename, so we only need global rename lock during migration. + * Get BFL lock for rename or migrate process. **/ static int mdt_rename_lock(struct mdt_thread_info *info, - struct lustre_handle *lh, - enum mdt_rename_lock rename_lock) + struct lustre_handle *lh) { - struct ldlm_namespace *ns = info->mti_mdt->mdt_namespace; - ldlm_policy_data_t *policy = &info->mti_policy; - struct ldlm_res_id *res_id = &info->mti_res_id; - __u64 flags = 0; - int rc; + int rc; ENTRY; - /* XXX only do global rename lock for migration */ - if (mdt_seq_site(info->mti_mdt)->ss_node_id != 0 && - rename_lock == MRL_MIGRATE) { + if (mdt_seq_site(info->mti_mdt)->ss_node_id != 0) { struct lu_fid *fid = &info->mti_tmp_fid1; struct mdt_object *obj; @@ -1252,25 +1242,29 @@ static int mdt_rename_lock(struct mdt_thread_info *info, if (IS_ERR(obj)) RETURN(PTR_ERR(obj)); - LASSERT(mdt_object_remote(obj)); rc = mdt_remote_object_lock(info, obj, &LUSTRE_BFL_FID, lh, LCK_EX, - MDS_INODELOCK_UPDATE); + MDS_INODELOCK_UPDATE, false); mdt_object_put(info->mti_env, obj); } else { + struct ldlm_namespace *ns = info->mti_mdt->mdt_namespace; + union ldlm_policy_data *policy = &info->mti_policy; + struct ldlm_res_id *res_id = &info->mti_res_id; + __u64 flags = 0; + fid_build_reg_res_name(&LUSTRE_BFL_FID, res_id); memset(policy, 0, sizeof *policy); policy->l_inodebits.bits = MDS_INODELOCK_UPDATE; flags = LDLM_FL_LOCAL_ONLY | LDLM_FL_ATOMIC_CB; rc = ldlm_cli_enqueue_local(ns, res_id, LDLM_IBITS, policy, - LCK_EX, &flags, ldlm_blocking_ast, - ldlm_completion_ast, NULL, NULL, 0, - LVB_T_NONE, - &info->mti_exp->exp_handle.h_cookie, - lh); + LCK_EX, &flags, ldlm_blocking_ast, + ldlm_completion_ast, NULL, NULL, 0, + LVB_T_NONE, + &info->mti_exp->exp_handle.h_cookie, + lh); + RETURN(rc); } - RETURN(rc); } @@ -1283,54 +1277,6 @@ static void mdt_rename_unlock(struct lustre_handle *lh) EXIT; } -/* - * This is is_subdir() variant, it is CMD if cmm forwards it to correct - * target. Source should not be ancestor of target dir. May be other rename - * checks can be moved here later. - */ -static int mdt_rename_sanity(struct mdt_thread_info *info, - const struct lu_fid *dir_fid, - const struct lu_fid *fid) -{ - struct mdt_object *dst; - struct lu_fid dst_fid = *dir_fid; - int rc = 0; - ENTRY; - - /* If the source and target are in the same directory, they can not - * be parent/child relationship, so subdir check is not needed */ - if (lu_fid_eq(dir_fid, fid)) - return 0; - - do { - LASSERT(fid_is_sane(&dst_fid)); - dst = mdt_object_find(info->mti_env, info->mti_mdt, &dst_fid); - if (!IS_ERR(dst)) { - rc = mdo_is_subdir(info->mti_env, - mdt_object_child(dst), fid, - &dst_fid); - mdt_object_put(info->mti_env, dst); - if (rc != -EREMOTE && rc < 0) { - CERROR("%s: failed subdir check in "DFID" for " - DFID": rc = %d\n", - mdt_obd_name(info->mti_mdt), - PFID(dir_fid), PFID(fid), rc); - /* Return EINVAL only if a parent is the @fid */ - if (rc == -EINVAL) - rc = -EIO; - } else { - /* check the found fid */ - if (lu_fid_eq(&dst_fid, fid)) - rc = -EINVAL; - } - } else { - rc = PTR_ERR(dst); - } - } while (rc == -EREMOTE); - - RETURN(rc); -} - /* Update object linkEA */ struct mdt_lock_list { struct mdt_object *mll_obj; @@ -1357,8 +1303,9 @@ static int mdt_lock_objects_in_linkea(struct mdt_thread_info *info, struct list_head *lock_list) { struct lu_buf *buf = &info->mti_big_buf; - struct linkea_data ldata = { 0 }; + struct linkea_data ldata = { NULL }; int count; + int retry_count; int rc; ENTRY; @@ -1377,6 +1324,10 @@ static int mdt_lock_objects_in_linkea(struct mdt_thread_info *info, RETURN(rc); } + /* ignore the migrating parent(@pobj) */ + retry_count = ldata.ld_leh->leh_reccount - 1; + +again: LASSERT(ldata.ld_leh != NULL); ldata.ld_lee = (struct link_ea_entry *)(ldata.ld_leh + 1); for (count = 0; count < ldata.ld_leh->leh_reccount; count++) { @@ -1385,30 +1336,37 @@ static int mdt_lock_objects_in_linkea(struct mdt_thread_info *info, struct mdt_lock_list *mll; struct lu_name name; struct lu_fid fid; + __u64 ibits; linkea_entry_unpack(ldata.ld_lee, &ldata.ld_reclen, &name, &fid); - ldata.ld_lee = (struct link_ea_entry *)((char *)ldata.ld_lee + - ldata.ld_reclen); mdt_pobj = mdt_object_find(info->mti_env, mdt, &fid); if (IS_ERR(mdt_pobj)) { CWARN("%s: cannot find obj "DFID": rc = %ld\n", mdt_obd_name(mdt), PFID(&fid), PTR_ERR(mdt_pobj)); - continue; + goto next; } if (!mdt_object_exists(mdt_pobj)) { CDEBUG(D_INFO, "%s: obj "DFID" does not exist\n", mdt_obd_name(mdt), PFID(&fid)); mdt_object_put(info->mti_env, mdt_pobj); - continue; + goto next; + } + + /* Check if the object already exists in the list */ + list_for_each_entry(mll, lock_list, mll_list) { + if (mll->mll_obj == mdt_pobj) { + mdt_object_put(info->mti_env, mdt_pobj); + goto next; + } } if (mdt_pobj == pobj) { CDEBUG(D_INFO, "%s: skipping parent obj "DFID"\n", mdt_obd_name(mdt), PFID(&fid)); mdt_object_put(info->mti_env, mdt_pobj); - continue; + goto next; } OBD_ALLOC_PTR(mll); @@ -1417,21 +1375,64 @@ static int mdt_lock_objects_in_linkea(struct mdt_thread_info *info, GOTO(out, rc = -ENOMEM); } + /* Since this needs to lock all of objects in linkea, to avoid + * deadlocks, because it does not follow parent-child order as + * other MDT operation, let's use try_lock here and if the lock + * cannot be gotten because of conflicting locks, then drop all + * current locks, send an AST to the client, and start again. */ mdt_lock_pdo_init(&mll->mll_lh, LCK_PW, &name); - rc = mdt_object_lock(info, mdt_pobj, &mll->mll_lh, - MDS_INODELOCK_UPDATE, - MDT_CROSS_LOCK); - if (rc != 0) { - CERROR("%s: cannot lock "DFID": rc =%d\n", - mdt_obd_name(mdt), PFID(&fid), rc); - mdt_object_put(info->mti_env, mdt_pobj); + ibits = 0; + rc = mdt_object_lock_try(info, mdt_pobj, &mll->mll_lh, &ibits, + MDS_INODELOCK_UPDATE, true); + if (!(ibits & MDS_INODELOCK_UPDATE)) { + mdt_unlock_list(info, lock_list, 0); + + CDEBUG(D_INFO, "%s: busy lock on "DFID" %s retry %d\n", + mdt_obd_name(mdt), PFID(&fid), name.ln_name, + retry_count); + + if (retry_count == 0) { + mdt_object_put(info->mti_env, mdt_pobj); + OBD_FREE_PTR(mll); + GOTO(out, rc = -EBUSY); + } + + mdt_lock_pdo_init(&mll->mll_lh, LCK_PW, &name); + rc = mdt_object_lock(info, mdt_pobj, &mll->mll_lh, + MDS_INODELOCK_UPDATE); + if (rc != 0) { + mdt_object_put(info->mti_env, mdt_pobj); + OBD_FREE_PTR(mll); + GOTO(out, rc); + } + + if (mdt_object_remote(mdt_pobj)) { + struct ldlm_lock *lock; + + /* For remote object, Set lock to cb_atomic, + * so lock can be released in blocking_ast() + * immediately, then the next try_lock will + * have better chance to succeds */ + lock = + ldlm_handle2lock(&mll->mll_lh.mlh_rreg_lh); + LASSERT(lock != NULL); + lock_res_and_lock(lock); + ldlm_set_atomic_cb(lock); + unlock_res_and_lock(lock); + LDLM_LOCK_PUT(lock); + } + mdt_object_unlock_put(info, mdt_pobj, &mll->mll_lh, rc); OBD_FREE_PTR(mll); - GOTO(out, rc); + retry_count--; + goto again; } - - CFS_INIT_LIST_HEAD(&mll->mll_list); + rc = 0; + INIT_LIST_HEAD(&mll->mll_list); mll->mll_obj = mdt_pobj; list_add_tail(&mll->mll_list, lock_list); +next: + ldata.ld_lee = (struct link_ea_entry *)((char *)ldata.ld_lee + + ldata.ld_reclen); } out: if (rc != 0) @@ -1453,6 +1454,9 @@ static int mdt_reint_migrate_internal(struct mdt_thread_info *info, struct mdt_lock_handle *lh_tgtp = NULL; struct lu_fid *old_fid = &info->mti_tmp_fid1; struct list_head lock_list; + __u64 lock_ibits; + struct ldlm_lock *lease = NULL; + bool lock_open_sem = false; int rc; ENTRY; @@ -1462,7 +1466,7 @@ static int mdt_reint_migrate_internal(struct mdt_thread_info *info, /* 1: lock the source dir. */ msrcdir = mdt_object_find(info->mti_env, info->mti_mdt, rr->rr_fid1); if (IS_ERR(msrcdir)) { - CERROR("%s: cannot find source dir "DFID" : rc = %d\n", + CDEBUG(D_OTHER, "%s: cannot find source dir "DFID" : rc = %d\n", mdt_obd_name(info->mti_mdt), PFID(rr->rr_fid1), (int)PTR_ERR(msrcdir)); RETURN(PTR_ERR(msrcdir)); @@ -1470,9 +1474,8 @@ static int mdt_reint_migrate_internal(struct mdt_thread_info *info, lh_dirp = &info->mti_lh[MDT_LH_PARENT]; mdt_lock_pdo_init(lh_dirp, LCK_PW, &rr->rr_name); - rc = mdt_object_lock(info, msrcdir, lh_dirp, - MDS_INODELOCK_UPDATE, - MDT_CROSS_LOCK); + rc = mdt_reint_object_lock(info, msrcdir, lh_dirp, MDS_INODELOCK_UPDATE, + true); if (rc) GOTO(out_put_parent, rc); @@ -1494,43 +1497,129 @@ static int mdt_reint_migrate_internal(struct mdt_thread_info *info, if (!fid_is_md_operative(old_fid)) GOTO(out_unlock_parent, rc = -EPERM); + if (lu_fid_eq(old_fid, &info->mti_mdt->mdt_md_root_fid)) + GOTO(out_unlock_parent, rc = -EPERM); + mold = mdt_object_find(info->mti_env, info->mti_mdt, old_fid); if (IS_ERR(mold)) GOTO(out_unlock_parent, rc = PTR_ERR(mold)); + if (!mdt_object_exists(mold)) { + LU_OBJECT_DEBUG(D_INODE, info->mti_env, + &mold->mot_obj, + "object does not exist"); + GOTO(out_put_child, rc = -ENOENT); + } + if (mdt_object_remote(mold)) { - CERROR("%s: source "DFID" is on the remote MDT\n", + CDEBUG(D_OTHER, "%s: source "DFID" is on the remote MDT\n", mdt_obd_name(info->mti_mdt), PFID(old_fid)); GOTO(out_put_child, rc = -EREMOTE); } if (S_ISREG(lu_object_attr(&mold->mot_obj)) && !mdt_object_remote(msrcdir)) { - CERROR("%s: parent "DFID" is still on the same" + CDEBUG(D_OTHER, "%s: parent "DFID" is still on the same" " MDT, which should be migrated first:" " rc = %d\n", mdt_obd_name(info->mti_mdt), PFID(mdt_object_fid(msrcdir)), -EPERM); GOTO(out_put_child, rc = -EPERM); } - rc = mdt_remote_permission(info, msrcdir, mold); + rc = mdt_remote_permission(info); if (rc != 0) GOTO(out_put_child, rc); /* 3: iterate the linkea of the object and lock all of the objects */ - CFS_INIT_LIST_HEAD(&lock_list); + INIT_LIST_HEAD(&lock_list); rc = mdt_lock_objects_in_linkea(info, mold, msrcdir, &lock_list); if (rc != 0) GOTO(out_put_child, rc); + if (info->mti_spec.sp_migrate_close) { + struct close_data *data; + struct mdt_body *repbody; + bool lease_broken = false; + + if (!req_capsule_field_present(info->mti_pill, &RMF_MDT_EPOCH, + RCL_CLIENT) || + !req_capsule_field_present(info->mti_pill, &RMF_CLOSE_DATA, + RCL_CLIENT)) + GOTO(out_lease, rc = -EPROTO); + + data = req_capsule_client_get(info->mti_pill, &RMF_CLOSE_DATA); + if (data == NULL) + GOTO(out_lease, rc = -EPROTO); + + lease = ldlm_handle2lock(&data->cd_handle); + if (lease == NULL) + GOTO(out_lease, rc = -ESTALE); + + /* try to hold open_sem so that nobody else can open the file */ + if (!down_write_trylock(&mold->mot_open_sem)) { + ldlm_lock_cancel(lease); + GOTO(out_lease, rc = -EBUSY); + } + + lock_open_sem = true; + /* Check if the lease open lease has already canceled */ + lock_res_and_lock(lease); + lease_broken = ldlm_is_cancel(lease); + unlock_res_and_lock(lease); + + LDLM_DEBUG(lease, DFID " lease broken? %d", + PFID(mdt_object_fid(mold)), lease_broken); + + /* Cancel server side lease. Client side counterpart should + * have been cancelled. It's okay to cancel it now as we've + * held mot_open_sem. */ + ldlm_lock_cancel(lease); + + if (lease_broken) + GOTO(out_lease, rc = -EAGAIN); +out_lease: + rc = mdt_close_internal(info, mdt_info_req(info), NULL); + repbody = req_capsule_server_get(info->mti_pill, &RMF_MDT_BODY); + repbody->mbo_valid |= OBD_MD_CLOSE_INTENT_EXECED; + if (rc != 0) + GOTO(out_unlock_list, rc); + } + /* 4: lock of the object migrated object */ lh_childp = &info->mti_lh[MDT_LH_OLD]; mdt_lock_reg_init(lh_childp, LCK_EX); - rc = mdt_object_lock(info, mold, lh_childp, - MDS_INODELOCK_LOOKUP | MDS_INODELOCK_UPDATE | - MDS_INODELOCK_LAYOUT, MDT_CROSS_LOCK); + lock_ibits = MDS_INODELOCK_LOOKUP | MDS_INODELOCK_UPDATE | + MDS_INODELOCK_LAYOUT; + if (mdt_object_remote(msrcdir)) { + /* Enqueue lookup lock from the parent MDT */ + rc = mdt_remote_object_lock(info, msrcdir, mdt_object_fid(mold), + &lh_childp->mlh_rreg_lh, + lh_childp->mlh_rreg_mode, + MDS_INODELOCK_LOOKUP, false); + if (rc != ELDLM_OK) + GOTO(out_unlock_list, rc); + + lock_ibits &= ~MDS_INODELOCK_LOOKUP; + } + + rc = mdt_reint_object_lock(info, mold, lh_childp, lock_ibits, true); + if (rc != 0) + GOTO(out_unlock_child, rc); + + /* Migration is incompatible with HSM. */ + ma->ma_need = MA_HSM; + ma->ma_valid = 0; + rc = mdt_attr_get_complex(info, mold, ma); if (rc != 0) - GOTO(out_unlock_list, rc); + GOTO(out_unlock_child, rc); + + if ((ma->ma_valid & MA_HSM) && ma->ma_hsm.mh_flags != 0) { + rc = -ENOSYS; + CDEBUG(D_OTHER, + "%s: cannot migrate HSM archived file "DFID": rc = %d\n", + mdt_obd_name(info->mti_mdt), PFID(old_fid), rc); + GOTO(out_unlock_child, rc); + } ma->ma_need = MA_LMV; ma->ma_valid = 0; @@ -1538,7 +1627,7 @@ static int mdt_reint_migrate_internal(struct mdt_thread_info *info, ma->ma_lmv_size = sizeof(info->mti_xattr_buf); rc = mdt_stripe_get(info, mold, ma, XATTR_NAME_LMV); if (rc != 0) - GOTO(out_unlock_list, rc); + GOTO(out_unlock_child, rc); if ((ma->ma_valid & MA_LMV)) { struct lmv_mds_md_v1 *lmm1; @@ -1546,7 +1635,7 @@ static int mdt_reint_migrate_internal(struct mdt_thread_info *info, lmv_le_to_cpu(ma->ma_lmv, ma->ma_lmv); lmm1 = &ma->ma_lmv->lmv_md_v1; if (!(lmm1->lmv_hash_type & LMV_HASH_FLAG_MIGRATION)) { - CERROR("%s: can not migrate striped dir "DFID + CDEBUG(D_OTHER, "%s: can not migrate striped dir "DFID ": rc = %d\n", mdt_obd_name(info->mti_mdt), PFID(mdt_object_fid(mold)), -EPERM); GOTO(out_unlock_child, rc = -EPERM); @@ -1561,7 +1650,8 @@ static int mdt_reint_migrate_internal(struct mdt_thread_info *info, GOTO(out_unlock_child, rc = PTR_ERR(mnew)); if (!mdt_object_remote(mnew)) { - CERROR("%s: "DFID" being migrated is on this MDT:" + CDEBUG(D_OTHER, + "%s: "DFID" being migrated is on this MDT:" " rc = %d\n", mdt_obd_name(info->mti_mdt), PFID(rr->rr_fid2), -EPERM); GOTO(out_put_new, rc = -EPERM); @@ -1573,7 +1663,7 @@ static int mdt_reint_migrate_internal(struct mdt_thread_info *info, mdt_object_fid(mnew), &lh_tgtp->mlh_rreg_lh, lh_tgtp->mlh_rreg_mode, - MDS_INODELOCK_UPDATE); + MDS_INODELOCK_UPDATE, false); if (rc != 0) { lh_tgtp = NULL; GOTO(out_put_new, rc); @@ -1584,7 +1674,7 @@ static int mdt_reint_migrate_internal(struct mdt_thread_info *info, if (IS_ERR(mnew)) GOTO(out_unlock_child, rc = PTR_ERR(mnew)); if (!mdt_object_remote(mnew)) { - CERROR("%s: Migration "DFID" is on this MDT:" + CDEBUG(D_OTHER, "%s: Migration "DFID" is on this MDT:" " rc = %d\n", mdt_obd_name(info->mti_mdt), PFID(rr->rr_fid2), -EXDEV); GOTO(out_put_new, rc = -EXDEV); @@ -1602,6 +1692,7 @@ static int mdt_reint_migrate_internal(struct mdt_thread_info *info, mdt_object_child(mnew), ma); if (rc != 0) GOTO(out_unlock_new, rc); + out_unlock_new: if (lh_tgtp != NULL) mdt_object_unlock(info, mnew, lh_tgtp, rc); @@ -1611,7 +1702,17 @@ out_put_new: out_unlock_child: mdt_object_unlock(info, mold, lh_childp, rc); out_unlock_list: - mdt_unlock_list(info, &lock_list, rc); + /* we don't really modify linkea objects, so we can safely decref these + * locks, and this can avoid saving them as COS locks, which may prevent + * subsequent migrate. */ + mdt_unlock_list(info, &lock_list, 1); + if (lease != NULL) { + ldlm_reprocess_all(lease->l_resource); + LDLM_LOCK_PUT(lease); + } + + if (lock_open_sem) + up_write(&mold->mot_open_sem); out_put_child: mdt_object_put(info->mti_env, mold); out_unlock_parent: @@ -1648,13 +1749,13 @@ out_put: static int mdt_object_lock_save(struct mdt_thread_info *info, struct mdt_object *dir, struct mdt_lock_handle *lh, - int idx) + int idx, bool cos_incompat) { int rc; /* we lock the target dir if it is local */ - rc = mdt_object_lock(info, dir, lh, MDS_INODELOCK_UPDATE, - MDT_LOCAL_LOCK); + rc = mdt_reint_object_lock(info, dir, lh, MDS_INODELOCK_UPDATE, + cos_incompat); if (rc != 0) return rc; @@ -1663,107 +1764,102 @@ static int mdt_object_lock_save(struct mdt_thread_info *info, return 0; } - -static int mdt_rename_parents_lock(struct mdt_thread_info *info, - struct mdt_object **srcp, - struct mdt_object **tgtp) +/* + * determine lock order of sobj and tobj + * + * there are two situations we need to lock tobj before sobj: + * 1. sobj is child of tobj + * 2. sobj and tobj are stripes of a directory, and stripe index of sobj is + * larger than that of tobj + * + * \retval 1 lock tobj before sobj + * \retval 0 lock sobj before tobj + * \retval -ev negative errno upon error + */ +static int mdt_rename_determine_lock_order(struct mdt_thread_info *info, + struct mdt_object *sobj, + struct mdt_object *tobj) { - struct mdt_reint_record *rr = &info->mti_rr; - const struct lu_fid *fid_src = rr->rr_fid1; - const struct lu_fid *fid_tgt = rr->rr_fid2; - struct mdt_lock_handle *lh_src = &info->mti_lh[MDT_LH_PARENT]; - struct mdt_lock_handle *lh_tgt = &info->mti_lh[MDT_LH_CHILD]; - struct mdt_object *src; - struct mdt_object *tgt; - int reverse = 0; - int rc; - ENTRY; + struct md_attr *ma = &info->mti_attr; + struct lu_fid *spfid = &info->mti_tmp_fid1; + struct lu_fid *tpfid = &info->mti_tmp_fid2; + struct lmv_mds_md_v1 *lmv; + __u32 sindex; + __u32 tindex; + int rc; - /* Check if the @src is not a child of the @tgt, otherwise a - * reverse locking must take place. - * - * Note: cannot be called after object_find, because if the object - * is destroyed in between it gets stuck in lu_object_find_at(), - * waiting for the last ref. */ - rc = mdt_rename_sanity(info, fid_src, fid_tgt); - if (rc == -EINVAL) - reverse = 1; - else if (rc) - RETURN(rc); + /* sobj and tobj are the same */ + if (sobj == tobj) + return 0; - /* find both parents. */ - src = mdt_object_find_check(info, fid_src, 0); - if (IS_ERR(src)) - RETURN(PTR_ERR(src)); + if (fid_is_root(mdt_object_fid(sobj))) + return 0; - OBD_FAIL_TIMEOUT(OBD_FAIL_MDS_RENAME3, 5); + if (fid_is_root(mdt_object_fid(tobj))) + return 1; - if (lu_fid_eq(fid_src, fid_tgt)) { - tgt = src; - mdt_object_get(info->mti_env, tgt); - } else { - tgt = mdt_object_find_check(info, fid_tgt, 1); - if (IS_ERR(tgt)) - GOTO(err_src_put, rc = PTR_ERR(tgt)); - - if (unlikely(mdt_object_remote(tgt))) { - CDEBUG(D_INFO, "Source dir "DFID" target dir "DFID - "on different MDTs\n", PFID(fid_src), - PFID(fid_tgt)); - GOTO(err_tgt_put, rc = -EXDEV); - } - } + /* check whether sobj is child of tobj */ + rc = mdo_is_subdir(info->mti_env, mdt_object_child(sobj), + mdt_object_fid(tobj)); + if (rc < 0) + return rc; - /* lock parents in the proper order. */ - if (reverse) { - rc = mdt_object_lock_save(info, tgt, lh_tgt, 1); - if (rc) - GOTO(err_tgt_put, rc); + if (rc == 1) + return 1; - OBD_FAIL_TIMEOUT(OBD_FAIL_MDS_RENAME, 5); + /* check whether sobj and tobj are children of the same parent */ + rc = mdt_attr_get_pfid(info, sobj, spfid); + if (rc) + return rc; - rc = mdt_object_lock_save(info, src, lh_src, 0); - } else { - rc = mdt_object_lock_save(info, src, lh_src, 0); - if (rc) - GOTO(err_tgt_put, rc); + rc = mdt_attr_get_pfid(info, tobj, tpfid); + if (rc) + return rc; - OBD_FAIL_TIMEOUT(OBD_FAIL_MDS_RENAME, 5); + if (!lu_fid_eq(spfid, tpfid)) + return 0; - if (tgt != src) - rc = mdt_object_lock_save(info, tgt, lh_tgt, 1); - else if (lh_src->mlh_pdo_hash != lh_tgt->mlh_pdo_hash) { - rc = mdt_pdir_hash_lock(info, lh_tgt, tgt, - MDS_INODELOCK_UPDATE); - OBD_FAIL_TIMEOUT(OBD_FAIL_MDS_PDO_LOCK2, 10); - } - } + /* check whether sobj and tobj are sibling stripes */ + ma->ma_need = MA_LMV; + ma->ma_valid = 0; + ma->ma_lmv = (union lmv_mds_md *)info->mti_xattr_buf; + ma->ma_lmv_size = sizeof(info->mti_xattr_buf); + rc = mdt_stripe_get(info, sobj, ma, XATTR_NAME_LMV); if (rc) - GOTO(err_unlock, rc); + return rc; - if (lu_object_is_dying(&tgt->mot_header)) - GOTO(err_unlock, rc = -ENOENT); + if (!(ma->ma_valid & MA_LMV)) + return 0; - *srcp = src; - *tgtp = tgt; - RETURN(0); + lmv = &ma->ma_lmv->lmv_md_v1; + if (!(le32_to_cpu(lmv->lmv_magic) & LMV_MAGIC_STRIPE)) + return 0; + sindex = le32_to_cpu(lmv->lmv_master_mdt_index); -err_unlock: - /* The order does not matter as the handle is checked inside, - * as well as not used handle. */ - mdt_object_unlock(info, src, lh_src, rc); - mdt_object_unlock(info, tgt, lh_tgt, rc); -err_tgt_put: - mdt_object_put(info->mti_env, tgt); -err_src_put: - mdt_object_put(info->mti_env, src); - RETURN(rc); + ma->ma_valid = 0; + rc = mdt_stripe_get(info, tobj, ma, XATTR_NAME_LMV); + if (rc) + return rc; + + if (!(ma->ma_valid & MA_LMV)) + return -ENODATA; + + lmv = &ma->ma_lmv->lmv_md_v1; + if (!(le32_to_cpu(lmv->lmv_magic) & LMV_MAGIC_STRIPE)) + return -EINVAL; + tindex = le32_to_cpu(lmv->lmv_master_mdt_index); + + /* check stripe index of sobj and tobj */ + if (sindex == tindex) + return -EINVAL; + + return sindex < tindex ? 0 : 1; } /* - * VBR: rename versions in reply: 0 - src parent; 1 - tgt parent; - * 2 - src child; 3 - tgt child. - * Update on disk version of src child. + * VBR: rename versions in reply: 0 - srcdir parent; 1 - tgtdir parent; + * 2 - srcdir child; 3 - tgtdir child. + * Update on disk version of srcdir child. */ /** * For DNE phase I, only these renames are allowed @@ -1780,38 +1876,109 @@ static int mdt_reint_rename_internal(struct mdt_thread_info *info, struct mdt_lock_handle *lhc) { struct mdt_reint_record *rr = &info->mti_rr; - struct md_attr *ma = &info->mti_attr; - struct ptlrpc_request *req = mdt_info_req(info); - struct mdt_object *msrcdir = NULL; - struct mdt_object *mtgtdir = NULL; - struct mdt_object *mold; - struct mdt_object *mnew = NULL; - struct mdt_lock_handle *lh_srcdirp; - struct mdt_lock_handle *lh_tgtdirp; - struct mdt_lock_handle *lh_oldp = NULL; - struct mdt_lock_handle *lh_newp = NULL; - struct lu_fid *old_fid = &info->mti_tmp_fid1; - struct lu_fid *new_fid = &info->mti_tmp_fid2; - int rc; + struct md_attr *ma = &info->mti_attr; + struct ptlrpc_request *req = mdt_info_req(info); + struct mdt_object *msrcdir = NULL; + struct mdt_object *mtgtdir = NULL; + struct mdt_object *mold; + struct mdt_object *mnew = NULL; + struct mdt_lock_handle *lh_srcdirp; + struct mdt_lock_handle *lh_tgtdirp; + struct mdt_lock_handle *lh_oldp = NULL; + struct mdt_lock_handle *lh_newp = NULL; + struct lu_fid *old_fid = &info->mti_tmp_fid1; + struct lu_fid *new_fid = &info->mti_tmp_fid2; + __u64 lock_ibits; + bool reverse = false; + bool cos_incompat, discard = false; + int rc; ENTRY; DEBUG_REQ(D_INODE, req, "rename "DFID"/"DNAME" to "DFID"/"DNAME, PFID(rr->rr_fid1), PNAME(&rr->rr_name), PFID(rr->rr_fid2), PNAME(&rr->rr_tgt_name)); + /* find both parents. */ + msrcdir = mdt_object_find_check(info, rr->rr_fid1, 0); + if (IS_ERR(msrcdir)) + RETURN(PTR_ERR(msrcdir)); + + OBD_FAIL_TIMEOUT(OBD_FAIL_MDS_RENAME3, 5); + + if (lu_fid_eq(rr->rr_fid1, rr->rr_fid2)) { + mtgtdir = msrcdir; + mdt_object_get(info->mti_env, mtgtdir); + } else { + mtgtdir = mdt_object_find_check(info, rr->rr_fid2, 1); + if (IS_ERR(mtgtdir)) + GOTO(out_put_srcdir, rc = PTR_ERR(mtgtdir)); + } + + rc = mdt_rename_determine_lock_order(info, msrcdir, mtgtdir); + if (rc < 0) + GOTO(out_put_tgtdir, rc); + + reverse = rc; + + /* source needs to be looked up after locking source parent, otherwise + * this rename may race with unlink source, and cause rename hang, see + * sanityn.sh 55b, so check parents first, if later we found source is + * remote, relock parents. */ + cos_incompat = (mdt_object_remote(msrcdir) || + mdt_object_remote(mtgtdir)); + + OBD_FAIL_TIMEOUT(OBD_FAIL_MDS_RENAME4, 5); + + /* lock parents in the proper order. */ lh_srcdirp = &info->mti_lh[MDT_LH_PARENT]; - mdt_lock_pdo_init(lh_srcdirp, LCK_PW, &rr->rr_name); lh_tgtdirp = &info->mti_lh[MDT_LH_CHILD]; + +relock: + mdt_lock_pdo_init(lh_srcdirp, LCK_PW, &rr->rr_name); mdt_lock_pdo_init(lh_tgtdirp, LCK_PW, &rr->rr_tgt_name); - /* step 1&2: lock the source and target dirs. */ - rc = mdt_rename_parents_lock(info, &msrcdir, &mtgtdir); - if (rc) - RETURN(rc); + if (reverse) { + rc = mdt_object_lock_save(info, mtgtdir, lh_tgtdirp, 1, + cos_incompat); + if (rc) + GOTO(out_put_tgtdir, rc); + + OBD_FAIL_TIMEOUT(OBD_FAIL_MDS_RENAME, 5); + + rc = mdt_object_lock_save(info, msrcdir, lh_srcdirp, 0, + cos_incompat); + if (rc != 0) { + mdt_object_unlock(info, mtgtdir, lh_tgtdirp, rc); + GOTO(out_put_tgtdir, rc); + } + } else { + rc = mdt_object_lock_save(info, msrcdir, lh_srcdirp, 0, + cos_incompat); + if (rc) + GOTO(out_put_tgtdir, rc); + + OBD_FAIL_TIMEOUT(OBD_FAIL_MDS_RENAME, 5); + + if (mtgtdir != msrcdir) { + rc = mdt_object_lock_save(info, mtgtdir, lh_tgtdirp, 1, + cos_incompat); + } else if (lh_srcdirp->mlh_pdo_hash != + lh_tgtdirp->mlh_pdo_hash) { + rc = mdt_pdir_hash_lock(info, lh_tgtdirp, mtgtdir, + MDS_INODELOCK_UPDATE, + cos_incompat); + OBD_FAIL_TIMEOUT(OBD_FAIL_MDS_PDO_LOCK2, 10); + } + if (rc != 0) { + mdt_object_unlock(info, msrcdir, lh_srcdirp, rc); + GOTO(out_put_tgtdir, rc); + } + } + OBD_FAIL_TIMEOUT(OBD_FAIL_MDS_RENAME4, 5); OBD_FAIL_TIMEOUT(OBD_FAIL_MDS_RENAME2, 5); - /* step 3: find & lock the old object. */ + /* find mold object. */ fid_zero(old_fid); rc = mdt_lookup_version_check(info, msrcdir, &rr->rr_name, old_fid, 2); if (rc != 0) @@ -1827,20 +1994,40 @@ static int mdt_reint_rename_internal(struct mdt_thread_info *info, if (IS_ERR(mold)) GOTO(out_unlock_parents, rc = PTR_ERR(mold)); + if (!mdt_object_exists(mold)) { + LU_OBJECT_DEBUG(D_INODE, info->mti_env, + &mold->mot_obj, + "object does not exist"); + GOTO(out_put_old, rc = -ENOENT); + } + /* Check if @mtgtdir is subdir of @mold, before locking child * to avoid reverse locking. */ - rc = mdt_rename_sanity(info, rr->rr_fid2, old_fid); - if (rc) - GOTO(out_put_old, rc); + if (mtgtdir != msrcdir) { + rc = mdo_is_subdir(info->mti_env, mdt_object_child(mtgtdir), + old_fid); + if (rc) { + if (rc == 1) + rc = -EINVAL; + GOTO(out_put_old, rc); + } + } tgt_vbr_obj_set(info->mti_env, mdt_obj2dt(mold)); /* save version after locking */ mdt_version_get_save(info, mold, 2); - mdt_set_capainfo(info, 2, old_fid, BYPASS_CAPA); - /* step 4: find & lock the new object. */ - /* new target object may not exist now */ - /* lookup with version checking */ + if (!cos_incompat && mdt_object_remote(mold)) { + cos_incompat = true; + mdt_object_put(info->mti_env, mold); + mdt_object_unlock(info, mtgtdir, lh_tgtdirp, -EAGAIN); + mdt_object_unlock(info, msrcdir, lh_srcdirp, -EAGAIN); + goto relock; + } + + /* find mnew object: + * mnew target object may not exist now + * lookup with version checking */ fid_zero(new_fid); rc = mdt_lookup_version_check(info, mtgtdir, &rr->rr_tgt_name, new_fid, 3); @@ -1856,34 +2043,69 @@ static int mdt_reint_rename_internal(struct mdt_thread_info *info, if (!fid_is_md_operative(new_fid)) GOTO(out_put_old, rc = -EPERM); - if (mdt_object_remote(mold)) { - CDEBUG(D_INFO, "Src child "DFID" is on another MDT\n", - PFID(old_fid)); - GOTO(out_put_old, rc = -EXDEV); - } - mnew = mdt_object_find(info->mti_env, info->mti_mdt, new_fid); if (IS_ERR(mnew)) GOTO(out_put_old, rc = PTR_ERR(mnew)); + if (!mdt_object_exists(mnew)) { + LU_OBJECT_DEBUG(D_INODE, info->mti_env, + &mnew->mot_obj, + "object does not exist"); + GOTO(out_put_new, rc = -ENOENT); + } + if (mdt_object_remote(mnew)) { - CDEBUG(D_INFO, "src child "DFID" is on another MDT\n", - PFID(new_fid)); + struct mdt_body *repbody; + + /* Always send rename req to the target child MDT */ + repbody = req_capsule_server_get(info->mti_pill, + &RMF_MDT_BODY); + LASSERT(repbody != NULL); + repbody->mbo_fid1 = *new_fid; + repbody->mbo_valid |= (OBD_MD_FLID | OBD_MD_MDS); GOTO(out_put_new, rc = -EXDEV); } + /* Before locking the target dir, check we do not replace + * a dir with a non-dir, otherwise it may deadlock with + * link op which tries to create a link in this dir + * back to this non-dir. */ + if (S_ISDIR(lu_object_attr(&mnew->mot_obj)) && + !S_ISDIR(lu_object_attr(&mold->mot_obj))) + GOTO(out_put_new, rc = -EISDIR); lh_oldp = &info->mti_lh[MDT_LH_OLD]; mdt_lock_reg_init(lh_oldp, LCK_EX); - rc = mdt_object_lock(info, mold, lh_oldp, MDS_INODELOCK_LOOKUP | - MDS_INODELOCK_XATTR, MDT_CROSS_LOCK); + lock_ibits = MDS_INODELOCK_LOOKUP | MDS_INODELOCK_XATTR; + if (mdt_object_remote(msrcdir)) { + /* Enqueue lookup lock from the parent MDT */ + rc = mdt_remote_object_lock(info, msrcdir, + mdt_object_fid(mold), + &lh_oldp->mlh_rreg_lh, + lh_oldp->mlh_rreg_mode, + MDS_INODELOCK_LOOKUP, + false); + if (rc != ELDLM_OK) + GOTO(out_put_new, rc); + + lock_ibits &= ~MDS_INODELOCK_LOOKUP; + } + + rc = mdt_reint_object_lock(info, mold, lh_oldp, lock_ibits, + cos_incompat); if (rc != 0) - GOTO(out_put_new, rc); + GOTO(out_unlock_old, rc); /* Check if @msrcdir is subdir of @mnew, before locking child * to avoid reverse locking. */ - rc = mdt_rename_sanity(info, rr->rr_fid1, new_fid); - if (rc) - GOTO(out_unlock_old, rc); + if (mtgtdir != msrcdir) { + rc = mdo_is_subdir(info->mti_env, + mdt_object_child(msrcdir), new_fid); + if (rc) { + if (rc == 1) + rc = -EINVAL; + GOTO(out_unlock_old, rc); + } + } /* We used to acquire MDS_INODELOCK_FULL here but we * can't do this now because a running HSM restore on @@ -1892,33 +2114,39 @@ static int mdt_reint_rename_internal(struct mdt_thread_info *info, lh_newp = &info->mti_lh[MDT_LH_NEW]; mdt_lock_reg_init(lh_newp, LCK_EX); - rc = mdt_object_lock(info, mnew, lh_newp, - MDS_INODELOCK_LOOKUP | - MDS_INODELOCK_UPDATE, - MDT_LOCAL_LOCK); + rc = mdt_reint_object_lock(info, mnew, lh_newp, + MDS_INODELOCK_LOOKUP | + MDS_INODELOCK_UPDATE, + cos_incompat); if (rc != 0) GOTO(out_unlock_old, rc); /* get and save version after locking */ mdt_version_get_save(info, mnew, 3); - mdt_set_capainfo(info, 3, new_fid, BYPASS_CAPA); } else if (rc != -EREMOTE && rc != -ENOENT) { GOTO(out_put_old, rc); } else { - /* If mnew does not exist and mold are remote directory, - * it only allows rename if they are under same directory */ - if (mtgtdir != msrcdir && mdt_object_remote(mold)) { - CDEBUG(D_INFO, "Src child "DFID" is on another MDT\n", - PFID(old_fid)); - GOTO(out_put_old, rc = -EXDEV); - } - lh_oldp = &info->mti_lh[MDT_LH_OLD]; mdt_lock_reg_init(lh_oldp, LCK_EX); - rc = mdt_object_lock(info, mold, lh_oldp, MDS_INODELOCK_LOOKUP | - MDS_INODELOCK_XATTR, MDT_CROSS_LOCK); + lock_ibits = MDS_INODELOCK_LOOKUP | MDS_INODELOCK_XATTR; + if (mdt_object_remote(msrcdir)) { + /* Enqueue lookup lock from the parent MDT */ + rc = mdt_remote_object_lock(info, msrcdir, + mdt_object_fid(mold), + &lh_oldp->mlh_rreg_lh, + lh_oldp->mlh_rreg_mode, + MDS_INODELOCK_LOOKUP, + false); + if (rc != ELDLM_OK) + GOTO(out_put_old, rc); + + lock_ibits &= ~MDS_INODELOCK_LOOKUP; + } + + rc = mdt_reint_object_lock(info, mold, lh_oldp, lock_ibits, + cos_incompat); if (rc != 0) - GOTO(out_put_old, rc); + GOTO(out_unlock_old, rc); mdt_enoent_version_save(info, 3); } @@ -1943,8 +2171,10 @@ static int mdt_reint_rename_internal(struct mdt_thread_info *info, /* handle last link of tgt object */ if (rc == 0) { mdt_counter_incr(req, LPROC_MDT_RENAME); - if (mnew) + if (mnew) { mdt_handle_last_unlink(info, mnew, ma); + discard = true; + } mdt_rename_counter_tally(info, info->mti_mdt, req, msrcdir, mtgtdir); @@ -1956,19 +2186,25 @@ static int mdt_reint_rename_internal(struct mdt_thread_info *info, out_unlock_old: mdt_object_unlock(info, mold, lh_oldp, rc); out_put_new: - if (mnew != NULL) + if (mnew != NULL) { + if (discard) + mdt_dom_check_and_discard(info, mnew); mdt_object_put(info->mti_env, mnew); + } out_put_old: mdt_object_put(info->mti_env, mold); out_unlock_parents: - mdt_object_unlock_put(info, mtgtdir, lh_tgtdirp, rc); - mdt_object_unlock_put(info, msrcdir, lh_srcdirp, rc); + mdt_object_unlock(info, mtgtdir, lh_tgtdirp, rc); + mdt_object_unlock(info, msrcdir, lh_srcdirp, rc); +out_put_tgtdir: + mdt_object_put(info->mti_env, mtgtdir); +out_put_srcdir: + mdt_object_put(info->mti_env, msrcdir); return rc; } static int mdt_reint_rename_or_migrate(struct mdt_thread_info *info, - struct mdt_lock_handle *lhc, - enum mdt_rename_lock rename_lock) + struct mdt_lock_handle *lhc, bool rename) { struct mdt_reint_record *rr = &info->mti_rr; struct ptlrpc_request *req = mdt_info_req(info); @@ -1977,20 +2213,26 @@ static int mdt_reint_rename_or_migrate(struct mdt_thread_info *info, ENTRY; if (info->mti_dlm_req) - ldlm_request_cancel(req, info->mti_dlm_req, 0); + ldlm_request_cancel(req, info->mti_dlm_req, 0, LATF_SKIP); - if (fid_is_obf(rr->rr_fid1) || fid_is_dot_lustre(rr->rr_fid1) || - fid_is_obf(rr->rr_fid2) || fid_is_dot_lustre(rr->rr_fid2)) + if (!fid_is_md_operative(rr->rr_fid1) || + !fid_is_md_operative(rr->rr_fid2)) RETURN(-EPERM); - rc = mdt_rename_lock(info, &rename_lh, rename_lock); - if (rc != 0) { - CERROR("%s: can't lock FS for rename: rc = %d\n", - mdt_obd_name(info->mti_mdt), rc); - RETURN(rc); + /* Note: do not enqueue rename lock for replay request, because + * if other MDT holds rename lock, but being blocked to wait for + * this MDT to finish its recovery, and the failover MDT can not + * get rename lock, which will cause deadlock. */ + if (!req_is_replay(req)) { + rc = mdt_rename_lock(info, &rename_lh); + if (rc != 0) { + CERROR("%s: can't lock FS for rename: rc = %d\n", + mdt_obd_name(info->mti_mdt), rc); + RETURN(rc); + } } - if (rename_lock == MRL_RENAME) + if (rename) rc = mdt_reint_rename_internal(info, lhc); else rc = mdt_reint_migrate_internal(info, lhc); @@ -2004,37 +2246,160 @@ static int mdt_reint_rename_or_migrate(struct mdt_thread_info *info, static int mdt_reint_rename(struct mdt_thread_info *info, struct mdt_lock_handle *lhc) { - return mdt_reint_rename_or_migrate(info, lhc, MRL_RENAME); + return mdt_reint_rename_or_migrate(info, lhc, true); } static int mdt_reint_migrate(struct mdt_thread_info *info, struct mdt_lock_handle *lhc) { - return mdt_reint_rename_or_migrate(info, lhc, MRL_MIGRATE); + return mdt_reint_rename_or_migrate(info, lhc, false); +} + +static int mdt_reint_resync(struct mdt_thread_info *info, + struct mdt_lock_handle *lhc) +{ + struct mdt_reint_record *rr = &info->mti_rr; + struct ptlrpc_request *req = mdt_info_req(info); + struct md_attr *ma = &info->mti_attr; + struct mdt_object *mo; + struct ldlm_lock *lease; + struct mdt_body *repbody; + struct md_layout_change layout = { 0 }; + bool lease_broken; + int rc, rc2; + ENTRY; + + DEBUG_REQ(D_INODE, req, DFID": FLR file resync\n", PFID(rr->rr_fid1)); + + if (info->mti_dlm_req) + ldlm_request_cancel(req, info->mti_dlm_req, 0, LATF_SKIP); + + mo = mdt_object_find(info->mti_env, info->mti_mdt, rr->rr_fid1); + if (IS_ERR(mo)) + GOTO(out, rc = PTR_ERR(mo)); + + if (!mdt_object_exists(mo)) + GOTO(out_obj, rc = -ENOENT); + + if (!S_ISREG(lu_object_attr(&mo->mot_obj))) + GOTO(out_obj, rc = -EINVAL); + + if (mdt_object_remote(mo)) + GOTO(out_obj, rc = -EREMOTE); + + lease = ldlm_handle2lock(rr->rr_handle); + if (lease == NULL) + GOTO(out_obj, rc = -ESTALE); + + /* It's really necessary to grab open_sem and check if the lease lock + * has been lost. There would exist a concurrent writer coming in and + * generating some dirty data in memory cache, the writeback would fail + * after the layout version is increased by MDS_REINT_RESYNC RPC. */ + if (!down_write_trylock(&mo->mot_open_sem)) + GOTO(out_put_lease, rc = -EBUSY); + + lock_res_and_lock(lease); + lease_broken = ldlm_is_cancel(lease); + unlock_res_and_lock(lease); + if (lease_broken) + GOTO(out_unlock, rc = -EBUSY); + + /* the file has yet opened by anyone else after we took the lease. */ + layout.mlc_opc = MD_LAYOUT_RESYNC; + rc = mdt_layout_change(info, mo, &layout); + if (rc) + GOTO(out_unlock, rc); + + ma->ma_need = MA_INODE; + ma->ma_valid = 0; + rc = mdt_attr_get_complex(info, mo, ma); + if (rc != 0) + GOTO(out_unlock, rc); + + repbody = req_capsule_server_get(info->mti_pill, &RMF_MDT_BODY); + mdt_pack_attr2body(info, repbody, &ma->ma_attr, mdt_object_fid(mo)); + + EXIT; +out_unlock: + up_write(&mo->mot_open_sem); +out_put_lease: + LDLM_LOCK_PUT(lease); +out_obj: + mdt_object_put(info->mti_env, mo); +out: + mdt_client_compatibility(info); + rc2 = mdt_fix_reply(info); + if (rc == 0) + rc = rc2; + return rc; } -typedef int (*mdt_reinter)(struct mdt_thread_info *info, - struct mdt_lock_handle *lhc); - -static mdt_reinter reinters[REINT_MAX] = { - [REINT_SETATTR] = mdt_reint_setattr, - [REINT_CREATE] = mdt_reint_create, - [REINT_LINK] = mdt_reint_link, - [REINT_UNLINK] = mdt_reint_unlink, - [REINT_RENAME] = mdt_reint_rename, - [REINT_OPEN] = mdt_reint_open, - [REINT_SETXATTR] = mdt_reint_setxattr, - [REINT_RMENTRY] = mdt_reint_unlink, - [REINT_MIGRATE] = mdt_reint_migrate, +struct mdt_reinter { + int (*mr_handler)(struct mdt_thread_info *, struct mdt_lock_handle *); + enum lprocfs_extra_opc mr_extra_opc; +}; + +static const struct mdt_reinter mdt_reinters[] = { + [REINT_SETATTR] = { + .mr_handler = &mdt_reint_setattr, + .mr_extra_opc = MDS_REINT_SETATTR, + }, + [REINT_CREATE] = { + .mr_handler = &mdt_reint_create, + .mr_extra_opc = MDS_REINT_CREATE, + }, + [REINT_LINK] = { + .mr_handler = &mdt_reint_link, + .mr_extra_opc = MDS_REINT_LINK, + }, + [REINT_UNLINK] = { + .mr_handler = &mdt_reint_unlink, + .mr_extra_opc = MDS_REINT_UNLINK, + }, + [REINT_RENAME] = { + .mr_handler = &mdt_reint_rename, + .mr_extra_opc = MDS_REINT_RENAME, + }, + [REINT_OPEN] = { + .mr_handler = &mdt_reint_open, + .mr_extra_opc = MDS_REINT_OPEN, + }, + [REINT_SETXATTR] = { + .mr_handler = &mdt_reint_setxattr, + .mr_extra_opc = MDS_REINT_SETXATTR, + }, + [REINT_RMENTRY] = { + .mr_handler = &mdt_reint_unlink, + .mr_extra_opc = MDS_REINT_UNLINK, + }, + [REINT_MIGRATE] = { + .mr_handler = &mdt_reint_migrate, + .mr_extra_opc = MDS_REINT_RENAME, + }, + [REINT_RESYNC] = { + .mr_handler = &mdt_reint_resync, + .mr_extra_opc = MDS_REINT_RESYNC, + }, }; int mdt_reint_rec(struct mdt_thread_info *info, - struct mdt_lock_handle *lhc) + struct mdt_lock_handle *lhc) { - int rc; - ENTRY; + const struct mdt_reinter *mr; + int rc; + ENTRY; + + if (!(info->mti_rr.rr_opcode < ARRAY_SIZE(mdt_reinters))) + RETURN(-EPROTO); - rc = reinters[info->mti_rr.rr_opcode](info, lhc); + mr = &mdt_reinters[info->mti_rr.rr_opcode]; + if (mr->mr_handler == NULL) + RETURN(-EPROTO); + + rc = (*mr->mr_handler)(info, lhc); - RETURN(rc); + lprocfs_counter_incr(ptlrpc_req2svc(mdt_info_req(info))->srv_stats, + PTLRPC_LAST_CNTR + mr->mr_extra_opc); + + RETURN(rc); }