From 775f88ed6c8b6235031268e258e15da405a5b955 Mon Sep 17 00:00:00 2001 From: Lai Siyao Date: Fri, 5 Mar 2021 17:07:34 +0800 Subject: [PATCH] LU-14490 lmv: striped directory as subdirectory mount lmv_intent_lookup() will replace fid1 with stripe FID, but if striped directory is mounted as subdirectory mount, it should be handled differently. Because fid2 is directory master object, if stripe is located on different MDT as master object, it will be treated as remote object by server, thus server won't reply LOOKUP lock back, therefore each file access needs to lookup "/". And remote directory (either plain or striped) shouldn't be used for subdirectory mount, because remote object can't get LOOKUP lock. Add an option "mdt_enable_remote_subdir_mount" (1 by default for backward compatibility), mdt_get_root() will return -EREMOTE if user specified subdir is a remote directory and this option is disabled. Add sanity 247g, updated 247f. Signed-off-by: Lai Siyao Change-Id: I5e8f95ee95c4155336098e55b7569ed7a43865c1 Reviewed-on: https://review.whamcloud.com/41893 Tested-by: jenkins Reviewed-by: Andreas Dilger Tested-by: Maloo Reviewed-by: Yingjin Qian Reviewed-by: Oleg Drokin --- lustre/lmv/lmv_intent.c | 17 ++++++++++---- lustre/mdt/mdt_handler.c | 24 +++++++++++++++---- lustre/mdt/mdt_internal.h | 4 +++- lustre/mdt/mdt_lproc.c | 33 ++++++++++++++++++++++++++ lustre/tests/sanity.sh | 60 ++++++++++++++++++++++++++++++++++++++++++++--- 5 files changed, 125 insertions(+), 13 deletions(-) diff --git a/lustre/lmv/lmv_intent.c b/lustre/lmv/lmv_intent.c index de2b425..7cc9ad8 100644 --- a/lustre/lmv/lmv_intent.c +++ b/lustre/lmv/lmv_intent.c @@ -446,11 +446,20 @@ lmv_intent_lookup(struct obd_export *exp, struct md_op_data *op_data, retry: if (op_data->op_flags & MF_GETATTR_BY_FID) { - /* getattr by FID, replace fid1 with stripe FID */ + /* getattr by FID, replace fid1 with stripe FID, + * NB, don't replace if name is "/", because it may be a subtree + * mount, and if it's a striped directory, fid1 will be replaced + * to stripe FID by hash, while fid2 is master object FID, which + * will be treated as a remote object if the two FIDs are + * located on different MDTs, and LOOKUP lock can't be fetched. + */ LASSERT(op_data->op_name); - tgt = lmv_locate_tgt(lmv, op_data); - if (IS_ERR(tgt)) - RETURN(PTR_ERR(tgt)); + if (op_data->op_namelen != 1 || + strncmp(op_data->op_name, "/", 1) != 0) { + tgt = lmv_locate_tgt(lmv, op_data); + if (IS_ERR(tgt)) + RETURN(PTR_ERR(tgt)); + } /* name is used to locate stripe target, clear it here * to avoid packing name in request, so that MDS knows diff --git a/lustre/mdt/mdt_handler.c b/lustre/mdt/mdt_handler.c index 0a6d779..462d522 100644 --- a/lustre/mdt/mdt_handler.c +++ b/lustre/mdt/mdt_handler.c @@ -276,6 +276,7 @@ static int mdt_lookup_fileset(struct mdt_thread_info *info, const char *fileset, { struct mdt_device *mdt = info->mti_mdt; struct lu_name *lname = &info->mti_name; + const char *start = fileset; char *filename = info->mti_filename; struct mdt_object *parent; u32 mode; @@ -290,8 +291,8 @@ static int mdt_lookup_fileset(struct mdt_thread_info *info, const char *fileset, */ *fid = mdt->mdt_md_root_fid; - while (rc == 0 && fileset != NULL && *fileset != '\0') { - const char *s1 = fileset; + while (rc == 0 && start != NULL && *start != '\0') { + const char *s1 = start; const char *s2; while (*++s1 == '/') @@ -303,7 +304,7 @@ static int mdt_lookup_fileset(struct mdt_thread_info *info, const char *fileset, if (s2 == s1) break; - fileset = s2; + start = s2; lname->ln_namelen = s2 - s1; if (lname->ln_namelen > NAME_MAX) { @@ -339,9 +340,21 @@ static int mdt_lookup_fileset(struct mdt_thread_info *info, const char *fileset, rc = PTR_ERR(parent); else { mode = lu_object_attr(&parent->mot_obj); - mdt_object_put(info->mti_env, parent); - if (!S_ISDIR(mode)) + if (!S_ISDIR(mode)) { rc = -ENOTDIR; + } else if (mdt_is_remote_object(info, parent, parent)) { + if (!mdt->mdt_enable_remote_subdir_mount) { + rc = -EREMOTE; + LCONSOLE_WARN("%s: subdir mount '%s' refused because 'enable_remote_subdir_mount=0': rc = %d\n", + mdt_obd_name(mdt), + fileset, rc); + } else { + LCONSOLE_INFO("%s: subdir mount '%s' is remote and may be slow\n", + mdt_obd_name(mdt), + fileset); + } + } + mdt_object_put(info->mti_env, parent); } } @@ -5700,6 +5713,7 @@ static int mdt_init0(const struct lu_env *env, struct mdt_device *m, m->mdt_enable_chprojid_gid = 0; m->mdt_enable_remote_rename = 1; m->mdt_dir_restripe_nsonly = 1; + m->mdt_enable_remote_subdir_mount = 1; atomic_set(&m->mdt_mds_mds_conns, 0); atomic_set(&m->mdt_async_commit_count, 0); diff --git a/lustre/mdt/mdt_internal.h b/lustre/mdt/mdt_internal.h index 58aadaa..64d5bef 100644 --- a/lustre/mdt/mdt_internal.h +++ b/lustre/mdt/mdt_internal.h @@ -293,7 +293,9 @@ struct mdt_device { mdt_skip_lfsck:1, mdt_readonly:1, /* dir restripe migrate dirent only */ - mdt_dir_restripe_nsonly:1; + mdt_dir_restripe_nsonly:1, + /* subdirectory mount of remote dir */ + mdt_enable_remote_subdir_mount:1; /* user with gid can create remote/striped * dir, and set default dir stripe */ diff --git a/lustre/mdt/mdt_lproc.c b/lustre/mdt/mdt_lproc.c index 60d6ec2..331042e 100644 --- a/lustre/mdt/mdt_lproc.c +++ b/lustre/mdt/mdt_lproc.c @@ -1231,6 +1231,38 @@ static ssize_t dir_restripe_nsonly_store(struct kobject *kobj, } LUSTRE_RW_ATTR(dir_restripe_nsonly); +static ssize_t enable_remote_subdir_mount_show(struct kobject *kobj, + struct attribute *attr, + char *buf) +{ + struct obd_device *obd = container_of(kobj, struct obd_device, + obd_kset.kobj); + struct mdt_device *mdt = mdt_dev(obd->obd_lu_dev); + + return scnprintf(buf, PAGE_SIZE, "%u\n", + mdt->mdt_enable_remote_subdir_mount); +} + +static ssize_t enable_remote_subdir_mount_store(struct kobject *kobj, + struct attribute *attr, + const char *buffer, + size_t count) +{ + struct obd_device *obd = container_of(kobj, struct obd_device, + obd_kset.kobj); + struct mdt_device *mdt = mdt_dev(obd->obd_lu_dev); + bool val; + int rc; + + rc = kstrtobool(buffer, &val); + if (rc) + return rc; + + mdt->mdt_enable_remote_subdir_mount = val; + return count; +} +LUSTRE_RW_ATTR(enable_remote_subdir_mount); + /** * Show if the OFD enforces T10PI checksum. * @@ -1350,6 +1382,7 @@ static struct attribute *mdt_attrs[] = { &lustre_attr_dir_split_delta.attr, &lustre_attr_dir_restripe_nsonly.attr, &lustre_attr_checksum_t10pi_enforce.attr, + &lustre_attr_enable_remote_subdir_mount.attr, NULL, }; diff --git a/lustre/tests/sanity.sh b/lustre/tests/sanity.sh index d49d244..7d68c91 100755 --- a/lustre/tests/sanity.sh +++ b/lustre/tests/sanity.sh @@ -19528,6 +19528,8 @@ test_247f() { [ $MDSCOUNT -lt 2 ] && skip_env "needs >= 2 MDTs" [ $MDS1_VERSION -lt $(version_code 2.13.52) ] && skip "Need at least version 2.13.52" + [ $CLIENT_VERSION -lt $(version_code 2.14.50) ] && + skip "Need at least version 2.14.50" lctl get_param -n mdc.$FSNAME-MDT0000*.import | grep -q subtree || skip "Fileset feature is not supported" @@ -19536,26 +19538,78 @@ test_247f() { $LFS mkdir -i $((MDSCOUNT - 1)) $DIR/$tdir/remote || error "mkdir remote failed" mkdir $DIR/$tdir/remote/subdir || error "mkdir remote/subdir failed" - $LFS mkdir -c $MDSCOUNT $DIR/$tdir/striped || + $LFS mkdir -i 0 -c $MDSCOUNT $DIR/$tdir/striped || error "mkdir striped failed" mkdir $DIR/$tdir/striped/subdir || error "mkdir striped/subdir failed" local submount=${MOUNT}_$tdir mkdir -p $submount || error "mkdir $submount failed" + stack_trap "rmdir $submount" local dir + local stat local fileset=$FILESET + local mdts=$(comma_list $(mdts_nodes)) + + stat=$(do_facet mds1 $LCTL get_param -n \ + mdt.*MDT0000.enable_remote_subdir_mount) + stack_trap "do_nodes $mdts $LCTL set_param \ + mdt.*.enable_remote_subdir_mount=$stat" - for dir in $tdir/remote $tdir/remote/subdir \ - $tdir/striped $tdir/striped/subdir $tdir/striped/. ; do + do_nodes $mdts "$LCTL set_param mdt.*.enable_remote_subdir_mount=0" + stack_trap "umount_client $submount" + FILESET="$fileset/$tdir/remote" mount_client $submount && + error "mount remote dir $dir should fail" + + for dir in $tdir/remote/subdir $tdir/striped $tdir/striped/subdir \ + $tdir/striped/. ; do FILESET="$fileset/$dir" mount_client $submount || error "mount $dir failed" umount_client $submount done + + do_nodes $mdts "$LCTL set_param mdt.*.enable_remote_subdir_mount=1" + FILESET="$fileset/$tdir/remote" mount_client $submount || + error "mount $tdir/remote failed" } run_test 247f "mount striped or remote directory as fileset" +test_247g() { + [ $MDSCOUNT -lt 4 ] && skip_env "needs >= 4 MDTs" + [ $CLIENT_VERSION -lt $(version_code 2.14.50) ] && + skip "Need at least version 2.14.50" + + $LFS mkdir -i 0 -c 4 -H fnv_1a_64 $DIR/$tdir || + error "mkdir $tdir failed" + touch $DIR/$tdir/$tfile || error "touch $tfile failed" + + local submount=${MOUNT}_$tdir + + mkdir -p $submount || error "mkdir $submount failed" + stack_trap "rmdir $submount" + + FILESET="$fileset/$tdir" mount_client $submount || + error "mount $dir failed" + stack_trap "umount $submount" + + local mdts=$(comma_list $(mdts_nodes)) + + local nrpcs + + stat $submount > /dev/null + cancel_lru_locks $MDC + stat $submount > /dev/null + stat $submount/$tfile > /dev/null + do_nodes $mdts "$LCTL set_param mdt.*.md_stats=clear > /dev/null" + stat $submount/$tfile > /dev/null + nrpcs=$(do_nodes $mdts "lctl get_param -n mdt.*.md_stats" | + awk '/getattr/ {sum += $2} END {print sum}') + + [ -z "$nrpcs" ] || error "$nrpcs extra getattr sent" +} +run_test 247g "mount striped directory as fileset caches ROOT lookup lock" + test_248a() { local fast_read_sav=$($LCTL get_param -n llite.*.fast_read 2>/dev/null) [ -z "$fast_read_sav" ] && skip "no fast read support" -- 1.8.3.1