LU-17334 lmv: handle object created on newly added MDT

When a new MDT is added to a filesystem without no_create, then a new
object is created on the MDT relatively quickly after it is added to
the filesystem, in particular because the new MDT would be preferred
by QOS space balancing due to lots of free space. However, it might
take a few seconds for the addition of the new MDT to be propagated
across all of the clients, so there is a risk that one client creates
a directory on an MDT that a client is not yet aware of, which returns
an error to the application immediately.

This patch fixes the issue by adding lmv_tgt_retry() that will retry
to use the MDT and wait for some number of seconds for the filesystem
layout to be updated if the MDT index an existing file/directory is
not found.

Commands that depend on user input, like 'lfs mkdir -i' and 'lfs df'
and round-robin MDT allocation will continue to use lmv_tgt() which
doesn't retry in case user specifies wrong MDT index, otherwise it can
hang the command for an extended period of time.

Signed-off-by: Lai Siyao <lai.siyao@whamcloud.com>
Change-Id: Idb0cf65e95f665628d6799298732b7a06cde4a86
Reviewed-on: https://review.whamcloud.com/c/fs/lustre-release/+/53363
Tested-by: jenkins <devops@whamcloud.com>
Tested-by: Maloo <maloo@whamcloud.com>
Reviewed-by: Andreas Dilger <adilger@whamcloud.com>
Reviewed-by: Jian Yu <yujian@whamcloud.com>
Reviewed-by: Oleg Drokin <green@whamcloud.com>

diff --git a/lustre/lmv/lmv_intent.c b/lustre/lmv/lmv_intent.c
index 1acdd84..7329a26 100644 (file)
--- a/lustre/lmv/lmv_intent.c
+++ b/lustre/lmv/lmv_intent.c
@@ -208,7 +208,7 @@ int lmv_revalidate_slaves(struct obd_export *exp,
                op_data->op_bias = MDS_CROSS_REF;
                op_data->op_cli_flags = CLI_NO_SLOT;
 
-               tgt = lmv_tgt(lmv, lsm->lsm_md_oinfo[i].lmo_mds);
+               tgt = lmv_tgt_retry(lmv, lsm->lsm_md_oinfo[i].lmo_mds);
                if (!tgt)
                        GOTO(cleanup, rc = -ENODEV);
 

diff --git a/lustre/lmv/lmv_internal.h b/lustre/lmv/lmv_internal.h
index 6340605..b6c1591 100644 (file)
--- a/lustre/lmv/lmv_internal.h
+++ b/lustre/lmv/lmv_internal.h
@@ -80,15 +80,14 @@ static inline struct obd_device *lmv2obd_dev(struct lmv_obd *lmv)
        return container_of_safe(lmv, struct obd_device, u.lmv);
 }
 
-static inline struct lu_tgt_desc *
-lmv_tgt(struct lmv_obd *lmv, __u32 index)
+static inline struct lu_tgt_desc *lmv_tgt(struct lmv_obd *lmv, __u32 index)
 {
        return index < lmv->lmv_mdt_descs.ltd_tgts_size ?
                LTD_TGT(&lmv->lmv_mdt_descs, index) : NULL;
 }
+struct lu_tgt_desc *lmv_tgt_retry(struct lmv_obd *lmv, __u32 index);
 
-static inline bool
-lmv_mdt0_inited(struct lmv_obd *lmv)
+static inline bool lmv_mdt0_inited(struct lmv_obd *lmv)
 {
        return lmv->lmv_mdt_descs.ltd_tgts_size > 0 &&
               test_bit(0, lmv->lmv_mdt_descs.ltd_tgt_bitmap);
@@ -148,11 +147,14 @@ lmv_fid2tgt(struct lmv_obd *lmv, const struct lu_fid *fid)
        struct lu_tgt_desc *tgt;
        int index;
 
+       if (!fid_is_sane(fid))
+               return ERR_PTR(-EINVAL);
+
        index = lmv_fid2tgt_index(lmv, fid);
        if (index < 0)
                return ERR_PTR(index);
 
-       tgt = lmv_tgt(lmv, index);
+       tgt = lmv_tgt_retry(lmv, index);
 
        return tgt ? tgt : ERR_PTR(-ENODEV);
 }

diff --git a/lustre/lmv/lmv_obd.c b/lustre/lmv/lmv_obd.c
index f88cb73..2fd65db 100644 (file)
--- a/lustre/lmv/lmv_obd.c
+++ b/lustre/lmv/lmv_obd.c
@@ -130,6 +130,49 @@ static int lmv_set_mdc_active(struct lmv_obd *lmv,
        return rc;
 }
 
+struct lu_tgt_desc *lmv_tgt_retry(struct lmv_obd *lmv, __u32 index)
+{
+       struct obd_device *obd = lmv2obd_dev(lmv);
+       struct lu_tgt_desc *tgt;
+       static time64_t next_print;
+       time64_t retry_limit = 0;
+       time64_t now;
+       unsigned int level;
+       int rc;
+
+       might_sleep();
+retry:
+       tgt = lmv_tgt(lmv, index);
+       if (likely(tgt && tgt->ltd_exp))
+               return tgt;
+
+       now = ktime_get_seconds();
+       if (retry_limit == 0) {
+               level = now > next_print ? D_WARNING : D_INFO;
+               retry_limit = now + RECONNECT_DELAY_MAX;
+       } else if (now > retry_limit) {
+               level = D_ERROR;
+       } else {
+               level = D_INFO;
+       }
+       CDEBUG_LIMIT(level, index < lmv->lmv_mdt_count ?
+                    "%s: MDT index %u/%u not configured\n" :
+                    "%s: MDT index %u more than MDT count %u\n",
+                    obd->obd_name, index, lmv->lmv_mdt_count);
+       if (now > next_print) {
+               LCONSOLE_INFO("%s: wait %ds while client connects to new MDT\n",
+                             obd->obd_name, (int)(retry_limit - now));
+               next_print = retry_limit + 600;
+       }
+       if (now < retry_limit) {
+               rc = schedule_timeout_interruptible(cfs_time_seconds(1));
+               if (rc == 0)
+                       goto retry;
+       }
+
+       return NULL;
+}
+
 static struct obd_uuid *lmv_get_uuid(struct obd_export *exp)
 {
        struct lmv_obd *lmv = &exp->exp_obd->u.lmv;
@@ -1747,7 +1790,7 @@ lmv_locate_tgt_by_name(struct lmv_obd *lmv, struct lmv_stripe_object *lso,
 
        *fid = oinfo->lmo_fid;
        *mds = oinfo->lmo_mds;
-       tgt = lmv_tgt(lmv, oinfo->lmo_mds);
+       tgt = lmv_tgt_retry(lmv, oinfo->lmo_mds);
 
        CDEBUG(D_INODE, "locate MDT %u parent "DFID"\n", *mds, PFID(fid));
 
@@ -2367,7 +2410,7 @@ static int lmv_migrate(struct obd_export *exp, struct md_op_data *op_data,
 
                /* save source stripe FID in fid4 temporarily for ELC */
                op_data->op_fid4 = oinfo->lmo_fid;
-               sp_tgt = lmv_tgt(lmv, oinfo->lmo_mds);
+               sp_tgt = lmv_tgt_retry(lmv, oinfo->lmo_mds);
                if (!sp_tgt)
                        RETURN(-ENODEV);
 
@@ -2382,7 +2425,7 @@ static int lmv_migrate(struct obd_export *exp, struct md_op_data *op_data,
                                RETURN(PTR_ERR(oinfo));
 
                        op_data->op_fid2 = oinfo->lmo_fid;
-                       tp_tgt = lmv_tgt(lmv, oinfo->lmo_mds);
+                       tp_tgt = lmv_tgt_retry(lmv, oinfo->lmo_mds);
                        if (!tp_tgt)
                                RETURN(-ENODEV);
 
@@ -2819,7 +2862,7 @@ static struct lu_dirent *stripe_dirent_load(struct lmv_dir_ctxt *ctxt,
                        break;
                }
 
-               tgt = lmv_tgt(ctxt->ldc_lmv, oinfo->lmo_mds);
+               tgt = lmv_tgt_retry(ctxt->ldc_lmv, oinfo->lmo_mds);
                if (!tgt) {
                        rc = -ENODEV;
                        break;