LU-17334 lmv: handle object created on newly added MDT

When a new MDT is added to a filesystem without no_create, then a new
object is created on the MDT relatively quickly after it is added to
the filesystem, in particular because the new MDT would be preferred
by QOS space balancing due to lots of free space. However, it might
take a few seconds for the addition of the new MDT to be propagated
across all of the clients, so there is a risk that one client creates
a directory on an MDT that a client is not yet aware of, which returns
an error to the application immediately.

This patch fixes the issue by adding lmv_tgt_retry() that will retry
to use the MDT and wait for some number of seconds for the filesystem
layout to be updated if the MDT index an existing file/directory is
not found.

Commands that depend on user input, like 'lfs mkdir -i' and 'lfs df'
and round-robin MDT allocation will continue to use lmv_tgt() which
doesn't retry in case user specifies wrong MDT index, otherwise it can
hang the command for an extended period of time.

Lustre-change: https://review.whamcloud.com/53363
Lustre-commit: 94a4663db95656ade6b6e695b849cd7763f0bd49

Signed-off-by: Lai Siyao <lai.siyao@whamcloud.com>
Change-Id: Idb0cf65e95f665628d6799298732b7a06cde4a86
Reviewed-on: https://review.whamcloud.com/c/ex/lustre-release/+/54018
Reviewed-by: Jian Yu <yujian@whamcloud.com>
Reviewed-by: Andreas Dilger <adilger@whamcloud.com>
Tested-by: Andreas Dilger <adilger@whamcloud.com>
Tested-by: Maloo <maloo@whamcloud.com>

diff --git a/lustre/lmv/lmv_intent.c b/lustre/lmv/lmv_intent.c
index dc7e38f..152cdb9 100644 (file)
--- a/lustre/lmv/lmv_intent.c
+++ b/lustre/lmv/lmv_intent.c
@@ -209,7 +209,7 @@ int lmv_revalidate_slaves(struct obd_export *exp,
                op_data->op_bias = MDS_CROSS_REF;
                op_data->op_cli_flags = CLI_NO_SLOT;
 
-               tgt = lmv_tgt(lmv, lsm->lsm_md_oinfo[i].lmo_mds);
+               tgt = lmv_tgt_retry(lmv, lsm->lsm_md_oinfo[i].lmo_mds);
                if (!tgt)
                        GOTO(cleanup, rc = -ENODEV);
 

diff --git a/lustre/lmv/lmv_internal.h b/lustre/lmv/lmv_internal.h
index 88e44b5..2166ad8 100644 (file)
--- a/lustre/lmv/lmv_internal.h
+++ b/lustre/lmv/lmv_internal.h
@@ -69,15 +69,14 @@ static inline struct obd_device *lmv2obd_dev(struct lmv_obd *lmv)
        return container_of_safe(lmv, struct obd_device, u.lmv);
 }
 
-static inline struct lu_tgt_desc *
-lmv_tgt(struct lmv_obd *lmv, __u32 index)
+static inline struct lu_tgt_desc *lmv_tgt(struct lmv_obd *lmv, __u32 index)
 {
        return index < lmv->lmv_mdt_descs.ltd_tgts_size ?
                LTD_TGT(&lmv->lmv_mdt_descs, index) : NULL;
 }
+struct lu_tgt_desc *lmv_tgt_retry(struct lmv_obd *lmv, __u32 index);
 
-static inline bool
-lmv_mdt0_inited(struct lmv_obd *lmv)
+static inline bool lmv_mdt0_inited(struct lmv_obd *lmv)
 {
        return lmv->lmv_mdt_descs.ltd_tgts_size > 0 &&
               test_bit(0, lmv->lmv_mdt_descs.ltd_tgt_bitmap);
@@ -137,11 +136,14 @@ lmv_fid2tgt(struct lmv_obd *lmv, const struct lu_fid *fid)
        struct lu_tgt_desc *tgt;
        int index;
 
+       if (!fid_is_sane(fid))
+               return ERR_PTR(-EINVAL);
+
        index = lmv_fid2tgt_index(lmv, fid);
        if (index < 0)
                return ERR_PTR(index);
 
-       tgt = lmv_tgt(lmv, index);
+       tgt = lmv_tgt_retry(lmv, index);
 
        return tgt ? tgt : ERR_PTR(-ENODEV);
 }

diff --git a/lustre/lmv/lmv_obd.c b/lustre/lmv/lmv_obd.c
index 6f36d42..d468410 100644 (file)
--- a/lustre/lmv/lmv_obd.c
+++ b/lustre/lmv/lmv_obd.c
@@ -130,6 +130,49 @@ static int lmv_set_mdc_active(struct lmv_obd *lmv,
        return rc;
 }
 
+struct lu_tgt_desc *lmv_tgt_retry(struct lmv_obd *lmv, __u32 index)
+{
+       struct obd_device *obd = lmv2obd_dev(lmv);
+       struct lu_tgt_desc *tgt;
+       static time64_t next_print;
+       time64_t retry_limit = 0;
+       time64_t now;
+       unsigned int level;
+       int rc;
+
+       might_sleep();
+retry:
+       tgt = lmv_tgt(lmv, index);
+       if (likely(tgt && tgt->ltd_exp))
+               return tgt;
+
+       now = ktime_get_seconds();
+       if (retry_limit == 0) {
+               level = now > next_print ? D_WARNING : D_INFO;
+               retry_limit = now + RECONNECT_DELAY_MAX;
+       } else if (now > retry_limit) {
+               level = D_ERROR;
+       } else {
+               level = D_INFO;
+       }
+       CDEBUG_LIMIT(level, index < lmv->lmv_mdt_count ?
+                    "%s: MDT index %u/%u not configured\n" :
+                    "%s: MDT index %u more than MDT count %u\n",
+                    obd->obd_name, index, lmv->lmv_mdt_count);
+       if (now > next_print) {
+               LCONSOLE_INFO("%s: wait %ds while client connects to new MDT\n",
+                             obd->obd_name, (int)(retry_limit - now));
+               next_print = retry_limit + 600;
+       }
+       if (now < retry_limit) {
+               rc = schedule_timeout_interruptible(cfs_time_seconds(1));
+               if (rc == 0)
+                       goto retry;
+       }
+
+       return NULL;
+}
+
 static struct obd_uuid *lmv_get_uuid(struct obd_export *exp)
 {
        struct lmv_obd *lmv = &exp->exp_obd->u.lmv;
@@ -1727,7 +1770,7 @@ lmv_locate_tgt_by_name(struct lmv_obd *lmv, struct lmv_stripe_object *lso,
 
        *fid = oinfo->lmo_fid;
        *mds = oinfo->lmo_mds;
-       tgt = lmv_tgt(lmv, oinfo->lmo_mds);
+       tgt = lmv_tgt_retry(lmv, oinfo->lmo_mds);
 
        CDEBUG(D_INODE, "locate MDT %u parent "DFID"\n", *mds, PFID(fid));
 
@@ -2358,7 +2401,7 @@ static int lmv_migrate(struct obd_export *exp, struct md_op_data *op_data,
 
                /* save source stripe FID in fid4 temporarily for ELC */
                op_data->op_fid4 = oinfo->lmo_fid;
-               sp_tgt = lmv_tgt(lmv, oinfo->lmo_mds);
+               sp_tgt = lmv_tgt_retry(lmv, oinfo->lmo_mds);
                if (!sp_tgt)
                        RETURN(-ENODEV);
 
@@ -2373,7 +2416,7 @@ static int lmv_migrate(struct obd_export *exp, struct md_op_data *op_data,
                                RETURN(PTR_ERR(oinfo));
 
                        op_data->op_fid2 = oinfo->lmo_fid;
-                       tp_tgt = lmv_tgt(lmv, oinfo->lmo_mds);
+                       tp_tgt = lmv_tgt_retry(lmv, oinfo->lmo_mds);
                        if (!tp_tgt)
                                RETURN(-ENODEV);
 
@@ -2810,7 +2853,7 @@ static struct lu_dirent *stripe_dirent_load(struct lmv_dir_ctxt *ctxt,
                        break;
                }
 
-               tgt = lmv_tgt(ctxt->ldc_lmv, oinfo->lmo_mds);
+               tgt = lmv_tgt_retry(ctxt->ldc_lmv, oinfo->lmo_mds);
                if (!tgt) {
                        rc = -ENODEV;
                        break;