LU-14490 lmv: striped directory as subdirectory mount

lmv_intent_lookup() will replace fid1 with stripe FID, but if striped
directory is mounted as subdirectory mount, it should be handled
differently. Because fid2 is directory master object, if stripe is
located on different MDT as master object, it will be treated as
remote object by server, thus server won't reply LOOKUP lock back,
therefore each file access needs to lookup "/".

And remote directory (either plain or striped) shouldn't be used for
subdirectory mount, because remote object can't get LOOKUP lock.
Add an option "mdt_enable_remote_subdir_mount" (1 by default for
backward compatibility), mdt_get_root() will return -EREMOTE if
user specified subdir is a remote directory and this option is
disabled.

Add sanity 247g, updated 247f.

Lustre-change: https://review.whamcloud.com/41893
Lustre-commit: 503917278a8f1dd7dd578fea6551de6c5dc4ebb9

Signed-off-by: Lai Siyao <lai.siyao@whamcloud.com>
Change-Id: I5e8f95ee95c4155336098e55b7569ed7a43865c1
Reviewed-on: https://review.whamcloud.com/42046
Tested-by: jenkins <devops@whamcloud.com>
Reviewed-by: Andreas Dilger <adilger@whamcloud.com>
Tested-by: Maloo <maloo@whamcloud.com>

diff --git a/lustre/lmv/lmv_intent.c b/lustre/lmv/lmv_intent.c
index 83a28bf..24c616b 100644 (file)
--- a/lustre/lmv/lmv_intent.c
+++ b/lustre/lmv/lmv_intent.c
@@ -445,11 +445,20 @@ lmv_intent_lookup(struct obd_export *exp, struct md_op_data *op_data,
 
 retry:
        if (op_data->op_flags & MF_GETATTR_BY_FID) {
-               /* getattr by FID, replace fid1 with stripe FID */
+               /* getattr by FID, replace fid1 with stripe FID,
+                * NB, don't replace if name is "/", because it may be a subtree
+                * mount, and if it's a striped directory, fid1 will be replaced
+                * to stripe FID by hash, while fid2 is master object FID, which
+                * will be treated as a remote object if the two FIDs are
+                * located on different MDTs, and LOOKUP lock can't be fetched.
+                */
                LASSERT(op_data->op_name);
-               tgt = lmv_locate_tgt(lmv, op_data, &op_data->op_fid1);
-               if (IS_ERR(tgt))
-                       RETURN(PTR_ERR(tgt));
+               if (op_data->op_namelen != 1 ||
+                   strncmp(op_data->op_name, "/", 1) != 0) {
+                       tgt = lmv_locate_tgt(lmv, op_data, &op_data->op_fid1);
+                       if (IS_ERR(tgt))
+                               RETURN(PTR_ERR(tgt));
+               }
 
                /* name is used to locate stripe target, clear it here
                 * to avoid packing name in request, so that MDS knows

diff --git a/lustre/mdt/mdt_handler.c b/lustre/mdt/mdt_handler.c
index d6ef09c..0e62b20 100644 (file)
--- a/lustre/mdt/mdt_handler.c
+++ b/lustre/mdt/mdt_handler.c
@@ -268,6 +268,7 @@ static int mdt_lookup_fileset(struct mdt_thread_info *info, const char *fileset,
 {
        struct mdt_device *mdt = info->mti_mdt;
        struct lu_name *lname = &info->mti_name;
+       const char *start = fileset;
        char *filename = info->mti_filename;
        struct mdt_object *parent;
        u32 mode;
@@ -282,8 +283,8 @@ static int mdt_lookup_fileset(struct mdt_thread_info *info, const char *fileset,
         */
        *fid = mdt->mdt_md_root_fid;
 
-       while (rc == 0 && fileset != NULL && *fileset != '\0') {
-               const char *s1 = fileset;
+       while (rc == 0 && start != NULL && *start != '\0') {
+               const char *s1 = start;
                const char *s2;
 
                while (*++s1 == '/')
@@ -295,7 +296,7 @@ static int mdt_lookup_fileset(struct mdt_thread_info *info, const char *fileset,
                if (s2 == s1)
                        break;
 
-               fileset = s2;
+               start = s2;
 
                lname->ln_namelen = s2 - s1;
                if (lname->ln_namelen > NAME_MAX) {
@@ -331,9 +332,21 @@ static int mdt_lookup_fileset(struct mdt_thread_info *info, const char *fileset,
                        rc = PTR_ERR(parent);
                else {
                        mode = lu_object_attr(&parent->mot_obj);
-                       mdt_object_put(info->mti_env, parent);
-                       if (!S_ISDIR(mode))
+                       if (!S_ISDIR(mode)) {
                                rc = -ENOTDIR;
+                       } else if (mdt_is_remote_object(info, parent, parent)) {
+                               if (!mdt->mdt_enable_remote_subdir_mount) {
+                                       rc = -EREMOTE;
+                                       LCONSOLE_WARN("%s: subdir mount '%s' refused because 'enable_remote_subdir_mount=0': rc = %d\n",
+                                                     mdt_obd_name(mdt),
+                                                     fileset, rc);
+                               } else {
+                                       LCONSOLE_INFO("%s: subdir mount '%s' is remote and may be slow\n",
+                                                     mdt_obd_name(mdt),
+                                                     fileset);
+                               }
+                       }
+                       mdt_object_put(info->mti_env, parent);
                }
        }
 
@@ -5406,6 +5419,7 @@ static int mdt_init0(const struct lu_env *env, struct mdt_device *m,
        m->mdt_enable_remote_dir_gid = 0;
        m->mdt_enable_chprojid_gid = 0;
        m->mdt_enable_remote_rename = 1;
+       m->mdt_enable_remote_subdir_mount = 1;
 
        atomic_set(&m->mdt_mds_mds_conns, 0);
        atomic_set(&m->mdt_async_commit_count, 0);

diff --git a/lustre/mdt/mdt_internal.h b/lustre/mdt/mdt_internal.h
index 79a17e4..27344c9 100644 (file)
--- a/lustre/mdt/mdt_internal.h
+++ b/lustre/mdt/mdt_internal.h
@@ -252,7 +252,9 @@ struct mdt_device {
                                   mdt_enable_striped_dir:1,
                                   mdt_enable_dir_migration:1,
                                   mdt_enable_remote_rename:1,
-                                  mdt_skip_lfsck:1;
+                                  mdt_skip_lfsck:1,
+                                  /* subdirectory mount of remote dir */
+                                  mdt_enable_remote_subdir_mount:1;
 
                                   /* user with gid can create remote/striped
                                    * dir, and set default dir stripe */

diff --git a/lustre/mdt/mdt_lproc.c b/lustre/mdt/mdt_lproc.c
index 102b041..0941f95 100644 (file)
--- a/lustre/mdt/mdt_lproc.c
+++ b/lustre/mdt/mdt_lproc.c
@@ -1012,6 +1012,36 @@ mdt_enable_chprojid_gid_seq_write(struct file *file, const char __user *buffer,
 }
 LPROC_SEQ_FOPS(mdt_enable_chprojid_gid);
 
+static int
+mdt_enable_remote_subdir_mount_seq_show(struct seq_file *m, void *data)
+{
+       struct obd_device *obd = m->private;
+       struct mdt_device *mdt = mdt_dev(obd->obd_lu_dev);
+
+       seq_printf(m, "%u\n",  mdt->mdt_enable_remote_subdir_mount);
+       return 0;
+}
+
+static ssize_t
+mdt_enable_remote_subdir_mount_seq_write(struct file *file,
+                                        const char __user *buffer,
+                                        size_t count, loff_t *off)
+{
+       struct seq_file *m = file->private_data;
+       struct obd_device *obd = m->private;
+       struct mdt_device *mdt = mdt_dev(obd->obd_lu_dev);
+       bool val;
+       int rc;
+
+       rc = kstrtobool_from_user(buffer, count, &val);
+       if (rc)
+               return rc;
+
+       mdt->mdt_enable_remote_subdir_mount = val;
+       return count;
+}
+LPROC_SEQ_FOPS(mdt_enable_remote_subdir_mount);
+
 LPROC_SEQ_FOPS_RO_TYPE(mdt, recovery_status);
 LPROC_SEQ_FOPS_RO_TYPE(mdt, num_exports);
 LPROC_SEQ_FOPS_RO_TYPE(mdt, target_instance);
@@ -1101,6 +1131,8 @@ static struct lprocfs_vars lprocfs_mdt_obd_vars[] = {
          .fops =       &mdt_dom_read_open_fops                 },
        { .name =       "migrate_hsm_allowed",
          .fops =       &mdt_migrate_hsm_allowed_fops           },
+       { .name =       "enable_remote_subdir_mount",
+         .fops =       &mdt_enable_remote_subdir_mount_fops    },
        { NULL }
 };
 

diff --git a/lustre/tests/sanity.sh b/lustre/tests/sanity.sh
index dd83e11..3fbdbf4 100755 (executable)
--- a/lustre/tests/sanity.sh
+++ b/lustre/tests/sanity.sh
@@ -17103,8 +17103,10 @@ run_test 247e "mount .. as fileset"
 
 test_247f() {
        [ $MDSCOUNT -lt 2 ] && skip_env "needs >= 2 MDTs"
-       [ $MDS1_VERSION -lt $(version_code 2.13.52) ] &&
-               skip "Need at least version 2.13.52"
+       [ $MDS1_VERSION -lt $(version_code 2.12.6) ] &&
+               skip "Need at least version 2.12.6"
+       [ $CLIENT_VERSION -lt $(version_code 2.12.6) ] &&
+               skip "Need at least version 2.12.6"
        lctl get_param -n mdc.$FSNAME-MDT0000*.import |
                grep -q subtree ||
                skip "Fileset feature is not supported"
@@ -17113,26 +17115,78 @@ test_247f() {
        $LFS mkdir -i $((MDSCOUNT - 1)) $DIR/$tdir/remote ||
                error "mkdir remote failed"
        mkdir $DIR/$tdir/remote/subdir || error "mkdir remote/subdir failed"
-       $LFS mkdir -c $MDSCOUNT $DIR/$tdir/striped ||
+       $LFS mkdir -i 0 -c $MDSCOUNT $DIR/$tdir/striped ||
                error "mkdir striped failed"
        mkdir $DIR/$tdir/striped/subdir || error "mkdir striped/subdir failed"
 
        local submount=${MOUNT}_$tdir
 
        mkdir -p $submount || error "mkdir $submount failed"
+       stack_trap "rmdir $submount"
 
        local dir
+       local stat
        local fileset=$FILESET
+       local mdts=$(comma_list $(mdts_nodes))
+
+       stat=$(do_facet mds1 $LCTL get_param -n \
+               mdt.*MDT0000.enable_remote_subdir_mount)
+       stack_trap "do_nodes $mdts $LCTL set_param \
+               mdt.*.enable_remote_subdir_mount=$stat"
+
+       do_nodes $mdts "$LCTL set_param mdt.*.enable_remote_subdir_mount=0"
+       stack_trap "umount_client $submount"
+       FILESET="$fileset/$tdir/remote" mount_client $submount &&
+               error "mount remote dir $dir should fail"
 
-       for dir in $tdir/remote $tdir/remote/subdir \
-                  $tdir/striped $tdir/striped/subdir $tdir/striped/. ; do
+       for dir in $tdir/remote/subdir $tdir/striped $tdir/striped/subdir \
+               $tdir/striped/. ; do
                FILESET="$fileset/$dir" mount_client $submount ||
                        error "mount $dir failed"
                umount_client $submount
        done
+
+       do_nodes $mdts "$LCTL set_param mdt.*.enable_remote_subdir_mount=1"
+       FILESET="$fileset/$tdir/remote" mount_client $submount ||
+               error "mount $tdir/remote failed"
 }
 run_test 247f "mount striped or remote directory as fileset"
 
+test_247g() {
+       [ $MDSCOUNT -lt 4 ] && skip_env "needs >= 4 MDTs"
+       [ $CLIENT_VERSION -lt $(version_code 2.12.6) ] &&
+               skip "Need at least version 2.12.6"
+
+       $LFS mkdir -i 0 -c 4 -H fnv_1a_64 $DIR/$tdir ||
+               error "mkdir $tdir failed"
+       touch $DIR/$tdir/$tfile || error "touch $tfile failed"
+
+       local submount=${MOUNT}_$tdir
+
+       mkdir -p $submount || error "mkdir $submount failed"
+       stack_trap "rmdir $submount"
+
+       FILESET="$fileset/$tdir" mount_client $submount ||
+               error "mount $dir failed"
+       stack_trap "umount $submount"
+
+       local mdts=$(comma_list $(mdts_nodes))
+
+       local nrpcs
+
+       stat $submount > /dev/null
+       cancel_lru_locks $MDC
+       stat $submount > /dev/null
+       stat $submount/$tfile > /dev/null
+       do_nodes $mdts "$LCTL set_param mdt.*.md_stats=clear > /dev/null"
+       stat $submount/$tfile > /dev/null
+       nrpcs=$(do_nodes $mdts "lctl get_param -n mdt.*.md_stats" |
+               awk '/getattr/ {sum += $2} END {print sum}')
+
+       [ -z "$nrpcs" ] || error "$nrpcs extra getattr sent"
+}
+run_test 247g "mount striped directory as fileset caches ROOT lookup lock"
+
 test_248() {
        local fast_read_sav=$($LCTL get_param -n llite.*.fast_read 2>/dev/null)
        [ -z "$fast_read_sav" ] && skip "no fast read support"