From c0f8f6b5001ec1a58493cde790da50dd2876030c Mon Sep 17 00:00:00 2001 From: Bobi Jam Date: Wed, 28 Apr 2021 13:16:05 +0800 Subject: [PATCH] LU-14642 flr: transfer layout version on layout change After layout changed (mirror extend/split), the file's layout version needs to transfer to OST ASAP so that following IO won't be blocked since OFD will check its layout version stored in the xattr XATTR_NAME_FID and find that the layout version from the client IO is bigger (ofd_verify_layout_version()). Signed-off-by: Bobi Jam Change-Id: I353800e868eaf13e3c795926b0d76fb1eb45c535 Reviewed-on: https://review.whamcloud.com/43472 Tested-by: jenkins Reviewed-by: Andreas Dilger Tested-by: Maloo Reviewed-by: Yingjin Qian Reviewed-by: Oleg Drokin --- lustre/lod/lod_object.c | 109 ++++++++++++++++++++++++++++++++++++++++++------ lustre/mdd/mdd_object.c | 6 +-- lustre/osp/osp_object.c | 5 +++ lustre/osp/osp_sync.c | 3 +- 4 files changed, 107 insertions(+), 16 deletions(-) diff --git a/lustre/lod/lod_object.c b/lustre/lod/lod_object.c index 38d04ff..858831f 100644 --- a/lustre/lod/lod_object.c +++ b/lustre/lod/lod_object.c @@ -1262,9 +1262,11 @@ static bool lod_obj_attr_set_comp_skip_cb(const struct lu_env *env, } break; } + case LCM_FL_RDONLY: + case LCM_FL_SYNC_PENDING: + break; default: LASSERTF(0, "impossible: %d\n", lo->ldo_flr_state); - case LCM_FL_SYNC_PENDING: break; } @@ -3270,13 +3272,14 @@ static int lod_declare_layout_merge(const struct lu_env *env, struct dt_object *dt, const struct lu_buf *mbuf, struct thandle *th) { - struct lod_thread_info *info = lod_env_info(env); - struct lu_buf *buf = &info->lti_buf; - struct lod_object *lo = lod_dt_obj(dt); - struct lov_comp_md_v1 *lcm; - struct lov_comp_md_v1 *cur_lcm; - struct lov_comp_md_v1 *merge_lcm; - struct lov_comp_md_entry_v1 *lcme; + struct lod_thread_info *info = lod_env_info(env); + struct lu_attr *layout_attr = &info->lti_layout_attr; + struct lu_buf *buf = &info->lti_buf; + struct lod_object *lo = lod_dt_obj(dt); + struct lov_comp_md_v1 *lcm; + struct lov_comp_md_v1 *cur_lcm; + struct lov_comp_md_v1 *merge_lcm; + struct lov_comp_md_entry_v1 *lcme; struct lov_mds_md_v1 *lmm; size_t size = 0; size_t offset; @@ -3285,7 +3288,7 @@ static int lod_declare_layout_merge(const struct lu_env *env, __u32 id = 0; __u16 mirror_id = 0; __u32 mirror_count; - int rc, i; + int rc, i; bool merge_has_dom; ENTRY; @@ -3404,8 +3407,6 @@ static int lod_declare_layout_merge(const struct lu_env *env, } /* fixup layout information */ - lod_obj_inc_layout_gen(lo); - lcm->lcm_layout_gen = cpu_to_le32(lo->ldo_layout_gen); lcm->lcm_size = cpu_to_le32(size); lcm->lcm_entry_count = cpu_to_le16(cur_entry_count + merge_entry_count); lcm->lcm_mirror_count = cpu_to_le16(mirror_count); @@ -3416,6 +3417,23 @@ static int lod_declare_layout_merge(const struct lu_env *env, if (rc) GOTO(out, rc); + lod_obj_inc_layout_gen(lo); + lcm->lcm_layout_gen = cpu_to_le32(lo->ldo_layout_gen); + + /* transfer layout version to OST objects. */ + if (lo->ldo_mirror_count > 1) { + struct lod_obj_stripe_cb_data data = { {0} }; + + layout_attr->la_valid = LA_LAYOUT_VERSION; + layout_attr->la_layout_version = 0; + data.locd_attr = layout_attr; + data.locd_declare = true; + data.locd_stripe_cb = lod_obj_stripe_attr_set_cb; + rc = lod_obj_for_each_stripe(env, lo, th, &data); + if (rc) + GOTO(out, rc); + } + rc = lod_sub_declare_xattr_set(env, dt_object_child(dt), buf, XATTR_NAME_LOV, LU_XATTR_REPLACE, th); @@ -3431,6 +3449,8 @@ static int lod_declare_layout_split(const struct lu_env *env, struct dt_object *dt, const struct lu_buf *mbuf, struct thandle *th) { + struct lod_thread_info *info = lod_env_info(env); + struct lu_attr *layout_attr = &info->lti_layout_attr; struct lod_object *lo = lod_dt_obj(dt); struct lov_comp_md_v1 *lcm = mbuf->lb_buf; int rc; @@ -3444,6 +3464,21 @@ static int lod_declare_layout_split(const struct lu_env *env, /* fix on-disk layout gen */ lcm->lcm_layout_gen = cpu_to_le32(lo->ldo_layout_gen); + + /* transfer layout version to OST objects. */ + if (lo->ldo_mirror_count > 1) { + struct lod_obj_stripe_cb_data data = { {0} }; + + layout_attr->la_valid = LA_LAYOUT_VERSION; + layout_attr->la_layout_version = 0; + data.locd_attr = layout_attr; + data.locd_declare = true; + data.locd_stripe_cb = lod_obj_stripe_attr_set_cb; + rc = lod_obj_for_each_stripe(env, lo, th, &data); + if (rc) + RETURN(rc); + } + rc = lod_sub_declare_xattr_set(env, dt_object_child(dt), mbuf, XATTR_NAME_LOV, LU_XATTR_REPLACE, th); RETURN(rc); @@ -4738,7 +4773,10 @@ static int lod_xattr_set(const struct lu_env *env, const char *name, int fl, struct thandle *th) { struct dt_object *next = dt_object_child(dt); - int rc; + struct lu_attr *layout_attr = &lod_env_info(env)->lti_layout_attr; + struct lod_object *lo = lod_dt_obj(dt); + struct lod_obj_stripe_cb_data data = { {0} }; + int rc = 0; ENTRY; @@ -4832,6 +4870,29 @@ static int lod_xattr_set(const struct lu_env *env, lod_striping_free(env, lod_dt_obj(dt)); rc = lod_sub_xattr_set(env, next, buf, name, fl, th); + } else if (fl & LU_XATTR_SPLIT) { + rc = lod_sub_xattr_set(env, next, buf, name, fl, th); + if (rc) + RETURN(rc); + + rc = lod_striping_reload(env, lo, buf); + if (rc) + RETURN(rc); + + if (lo->ldo_mirror_count > 1 && + layout_attr->la_valid & LA_LAYOUT_VERSION) { + /* mirror split */ + layout_attr->la_layout_version = + lo->ldo_layout_gen; + data.locd_attr = layout_attr; + data.locd_declare = false; + data.locd_stripe_cb = + lod_obj_stripe_attr_set_cb; + rc = lod_obj_for_each_stripe(env, lo, th, + &data); + if (rc) + RETURN(rc); + } } else if (fl & LU_XATTR_PURGE) { rc = lod_layout_purge(env, dt, buf, th); } else if (dt_object_remote(dt)) { @@ -4862,6 +4923,23 @@ static int lod_xattr_set(const struct lu_env *env, !lod_dt_obj(dt)->ldo_comp_cached)); rc = lod_striped_create(env, dt, NULL, NULL, th); + if (rc) + RETURN(rc); + + if (fl & LU_XATTR_MERGE && lo->ldo_mirror_count > 1 && + layout_attr->la_valid & LA_LAYOUT_VERSION) { + /* mirror merge exec phase */ + layout_attr->la_layout_version = + lo->ldo_layout_gen; + data.locd_attr = layout_attr; + data.locd_declare = false; + data.locd_stripe_cb = + lod_obj_stripe_attr_set_cb; + rc = lod_obj_for_each_stripe(env, lo, th, + &data); + if (rc) + RETURN(rc); + } } RETURN(rc); } else if (strcmp(name, XATTR_NAME_FID) == 0) { @@ -7832,6 +7910,7 @@ static int lod_declare_update_sync_pending(const struct lu_env *env, struct thandle *th) { struct lod_thread_info *info = lod_env_info(env); + struct lu_attr *layout_attr = &info->lti_layout_attr; unsigned sync_components = 0; unsigned resync_components = 0; int i; @@ -7904,6 +7983,12 @@ static int lod_declare_update_sync_pending(const struct lu_env *env, lo->ldo_flr_state = LCM_FL_RDONLY; lod_obj_inc_layout_gen(lo); + layout_attr->la_valid = LA_LAYOUT_VERSION; + layout_attr->la_layout_version = 0; /* set current version */ + rc = lod_declare_attr_set(env, &lo->ldo_obj, layout_attr, th); + if (rc) + GOTO(out, rc); + info->lti_buf.lb_len = lod_comp_md_size(lo, false); rc = lod_sub_declare_xattr_set(env, lod_object_child(lo), &info->lti_buf, XATTR_NAME_LOV, 0, th); diff --git a/lustre/mdd/mdd_object.c b/lustre/mdd/mdd_object.c index 0ed2820..3689a90 100644 --- a/lustre/mdd/mdd_object.c +++ b/lustre/mdd/mdd_object.c @@ -1588,7 +1588,7 @@ static int mdd_xattr_merge(const struct lu_env *env, struct md_object *md_obj, if (rc) GOTO(out, rc); - rc = mdd_declare_xattr_set(env, mdd, obj, buf_vic, XATTR_LUSTRE_LOV, + rc = mdd_declare_xattr_set(env, mdd, obj, buf_vic, XATTR_NAME_LOV, LU_XATTR_MERGE, handle); if (rc) GOTO(out, rc); @@ -1597,7 +1597,7 @@ static int mdd_xattr_merge(const struct lu_env *env, struct md_object *md_obj, if (rc != 0) GOTO(out, rc); - rc = mdo_xattr_set(env, obj, buf_vic, XATTR_LUSTRE_LOV, LU_XATTR_MERGE, + rc = mdo_xattr_set(env, obj, buf_vic, XATTR_NAME_LOV, LU_XATTR_MERGE, handle); if (rc) GOTO(out, rc); @@ -1876,7 +1876,7 @@ static int mdd_xattr_split(const struct lu_env *env, struct md_object *md_obj, } /* set obj's layout in @buf */ - rc = mdo_xattr_set(env, obj, buf, XATTR_NAME_LOV, LU_XATTR_REPLACE, + rc = mdo_xattr_set(env, obj, buf, XATTR_NAME_LOV, LU_XATTR_SPLIT, handle); if (rc) GOTO(unlock, rc); diff --git a/lustre/osp/osp_object.c b/lustre/osp/osp_object.c index 554f74a..0139379 100644 --- a/lustre/osp/osp_object.c +++ b/lustre/osp/osp_object.c @@ -753,7 +753,12 @@ static int osp_attr_set(const struct lu_env *env, struct dt_object *dt, osp_update_request_destroy(env, update); } else { + struct osp_device *osp = lu2osp_dev(dt->do_lu.lo_dev); + rc = osp_sync_add(env, o, MDS_SETATTR64_REC, th, attr); + /* send layout version to OST ASAP */ + if (attr->la_valid & LA_LAYOUT_VERSION) + wake_up(&osp->opd_sync_waitq); /* XXX: send new uid/gid to OST ASAP? */ } } else { diff --git a/lustre/osp/osp_sync.c b/lustre/osp/osp_sync.c index 2ea884c..d270453 100644 --- a/lustre/osp/osp_sync.c +++ b/lustre/osp/osp_sync.c @@ -761,7 +761,8 @@ static int osp_sync_new_setattr_job(struct osp_device *d, if (body->oa.o_valid & OBD_MD_LAYOUT_VERSION) { OBD_FAIL_TIMEOUT(OBD_FAIL_FLR_LV_DELAY, cfs_fail_val); if (unlikely(OBD_FAIL_CHECK(OBD_FAIL_FLR_LV_INC))) - ++body->oa.o_layout_version; + body->oa.o_layout_version = LU_LAYOUT_RESYNC | + (body->oa.o_layout_version + 1); } osp_sync_send_new_rpc(d, llh, h, req); -- 1.8.3.1