LU-14975 dne: dir migration in non-recursive mode

Add an option "-d|--directory" option for "lfs migrate -m" to
migrate specified directory only, which is similar to "ls -d".

Add sanity 230w.

Signed-off-by: Lai Siyao <lai.siyao@whamcloud.com>
Change-Id: Ib97949e3840a3b49f7074b16e259582a9bf16e3b
Reviewed-on: https://review.whamcloud.com/44802
Tested-by: jenkins <devops@whamcloud.com>
Reviewed-by: Andreas Dilger <adilger@whamcloud.com>
Tested-by: Maloo <maloo@whamcloud.com>
Reviewed-by: Yingjin Qian <qian@ddn.com>
Reviewed-by: Oleg Drokin <green@whamcloud.com>

diff --git a/lustre/doc/lfs-migrate.1 b/lustre/doc/lfs-migrate.1
index 795b5d3..3991c9f 100644 (file)
--- a/lustre/doc/lfs-migrate.1
+++ b/lustre/doc/lfs-migrate.1
@@ -8,7 +8,7 @@ lfs migrate \- migrate files or directories between MDTs or OSTs.
 .IR FILE " ..."
 .br
 .B lfs migrate -m \fISTART_MDT_INDEX
-.RB [ -cHv ]
+.RB [ -cdHv ]
 .I DIRECTORY
 .br
 .SH DESCRIPTION
@@ -124,6 +124,11 @@ values are specified in a comma-separated list, then the number of specified
 values must match
 .IR COUNT .
 .TP
+.BR -d , --directory
+Only migrate the specified \fIDIRECTORY\fR and the non-directory inodes that are
+directly located within it.
+Similar to '\fBls -d\fR' and '\fBlfs getstripe -d\fR'.
+.TP
 .BR -H , --mdt-hash=\fIHASH_TYPE\fR
 Use
 .I HASH_TYPE
@@ -193,6 +198,11 @@ from its current MDT to MDT0000 and MDT0002.  The
 .B testremote
 directory and all of its subdirectories will be striped across both MDTs.
 .TP
+.B $ lfs migrate -m 0,2 -d ./testremote
+Move ./testremote and the first level of sub files from their current MDT
+to the MDT with index 0 and 2. Different from above case, the layout of
+subdirectories under ./testremote won't be changed.
+.TP
 .B # lfs setstripe -E 256M -c 1 -E 16G -c 4 -E eof -c 40 topdir
 Set a default PFL layout (without any DoM component) on the directory
 .BR topdir ,

diff --git a/lustre/include/uapi/linux/lustre/lustre_idl.h b/lustre/include/uapi/linux/lustre/lustre_idl.h
index 71a20f4..06bcb8d 100644 (file)
--- a/lustre/include/uapi/linux/lustre/lustre_idl.h
+++ b/lustre/include/uapi/linux/lustre/lustre_idl.h
@@ -1955,6 +1955,8 @@ enum mds_op_bias {
        /* setstripe create only, don't restripe if target exists */
        MDS_SETSTRIPE_CREATE    = 1 << 21,
        MDS_FID_OP              = 1 << 22,
+       /* migrate dirent only */
+       MDS_MIGRATE_NSONLY      = 1 << 23,
 };
 
 #define MDS_CLOSE_INTENT (MDS_HSM_RELEASE | MDS_CLOSE_LAYOUT_SWAP |         \

diff --git a/lustre/llite/dir.c b/lustre/llite/dir.c
index 0a2725c..ad94e5a 100644 (file)
--- a/lustre/llite/dir.c
+++ b/lustre/llite/dir.c
@@ -2144,6 +2144,7 @@ out_hur:
                int len;
                char *filename;
                int namelen = 0;
+               __u32 flags;
                int rc;
 
                rc = obd_ioctl_getdata(&data, &len, (void __user *)arg);
@@ -2156,6 +2157,7 @@ out_hur:
 
                filename = data->ioc_inlbuf1;
                namelen = data->ioc_inllen1;
+               flags = data->ioc_type;
 
                if (namelen < 1 || namelen != strlen(filename) + 1) {
                        CDEBUG(D_INFO, "IOC_MDC_LOOKUP missing filename\n");
@@ -2171,7 +2173,7 @@ out_hur:
                        GOTO(migrate_free, rc);
                }
 
-               rc = ll_migrate(inode, file, lum, filename);
+               rc = ll_migrate(inode, file, lum, filename, flags);
 migrate_free:
                OBD_FREE_LARGE(data, len);
 

diff --git a/lustre/llite/file.c b/lustre/llite/file.c
index 9e230b9..21fb0b4 100644 (file)
--- a/lustre/llite/file.c
+++ b/lustre/llite/file.c
@@ -4790,7 +4790,7 @@ out_req:
 }
 
 int ll_migrate(struct inode *parent, struct file *file, struct lmv_user_md *lum,
-              const char *name)
+              const char *name, __u32 flags)
 {
        struct dentry *dchild = NULL;
        struct inode *child_inode = NULL;
@@ -4888,6 +4888,11 @@ int ll_migrate(struct inode *parent, struct file *file, struct lmv_user_md *lum,
        op_data->op_data = lum;
        op_data->op_data_size = lumlen;
 
+       /* migrate dirent only for subdirs if MDS_MIGRATE_NSONLY set */
+       if (S_ISDIR(child_inode->i_mode) && (flags & MDS_MIGRATE_NSONLY) &&
+           lmv_dir_layout_changing(ll_i2info(parent)->lli_lsm_md))
+               op_data->op_bias |= MDS_MIGRATE_NSONLY;
+
 again:
        if (S_ISREG(child_inode->i_mode)) {
                och = ll_lease_open(child_inode, NULL, FMODE_WRITE, 0);

diff --git a/lustre/llite/llite_internal.h b/lustre/llite/llite_internal.h
index 3f06e46..1a8ff18 100644 (file)
--- a/lustre/llite/llite_internal.h
+++ b/lustre/llite/llite_internal.h
@@ -1125,7 +1125,7 @@ static inline int ll_inode_flags_to_xflags(int inode_flags)
 }
 
 int ll_migrate(struct inode *parent, struct file *file,
-              struct lmv_user_md *lum, const char *name);
+              struct lmv_user_md *lum, const char *name, __u32 flags);
 int ll_get_fid_by_name(struct inode *parent, const char *name,
                       int namelen, struct lu_fid *fid, struct inode **inode);
 int ll_inode_permission(struct inode *inode, int mask);

diff --git a/lustre/lmv/lmv_obd.c b/lustre/lmv/lmv_obd.c
index 64d19e3..3e876f5 100644 (file)
--- a/lustre/lmv/lmv_obd.c
+++ b/lustre/lmv/lmv_obd.c
@@ -2267,6 +2267,11 @@ static int lmv_migrate(struct obd_export *exp, struct md_op_data *op_data,
                        tp_tgt = lmv_tgt(lmv, oinfo->lmo_mds);
                        if (!tp_tgt)
                                RETURN(-ENODEV);
+
+                       /* parent unchanged and update namespace only */
+                       if (lu_fid_eq(&op_data->op_fid4, &op_data->op_fid2) &&
+                           op_data->op_bias & MDS_MIGRATE_NSONLY)
+                               RETURN(-EALREADY);
                }
        } else {
                sp_tgt = parent_tgt;

diff --git a/lustre/mdt/mdt_lib.c b/lustre/mdt/mdt_lib.c
index 9b5693d..89700bb 100644 (file)
--- a/lustre/mdt/mdt_lib.c
+++ b/lustre/mdt/mdt_lib.c
@@ -1557,7 +1557,8 @@ static int mdt_migrate_unpack(struct mdt_thread_info *info)
        } else {
                spec->sp_migrate_close = 0;
        }
-       spec->sp_migrate_nsonly = 0;
+
+       spec->sp_migrate_nsonly = !!(rec->rn_bias & MDS_MIGRATE_NSONLY);
 
        /* lustre version > 2.11 migration packs lum */
        if (req_capsule_has_field(pill, &RMF_EADATA, RCL_CLIENT)) {

diff --git a/lustre/ptlrpc/wiretest.c b/lustre/ptlrpc/wiretest.c
index 61389ec..fea0df8 100644 (file)
--- a/lustre/ptlrpc/wiretest.c
+++ b/lustre/ptlrpc/wiretest.c
@@ -2407,6 +2407,12 @@ void lustre_assert_wire_constants(void)
                (unsigned)MDS_PCC_ATTACH);
        LASSERTF(MDS_CLOSE_UPDATE_TIMES == 0x00100000UL, "found 0x%.8xUL\n",
                (unsigned)MDS_CLOSE_UPDATE_TIMES);
+       LASSERTF(MDS_SETSTRIPE_CREATE == 0x00200000UL, "found 0x%.8xUL\n",
+               (unsigned)MDS_SETSTRIPE_CREATE);
+       LASSERTF(MDS_FID_OP == 0x00400000UL, "found 0x%.8xUL\n",
+               (unsigned)MDS_FID_OP);
+       LASSERTF(MDS_MIGRATE_NSONLY == 0x00800000UL, "found 0x%.8xUL\n",
+               (unsigned)MDS_MIGRATE_NSONLY);
 
        /* Checks for struct mdt_body */
        LASSERTF((int)sizeof(struct mdt_body) == 216, "found %lld\n",

diff --git a/lustre/tests/sanity.sh b/lustre/tests/sanity.sh
index 094c3bd..6930566 100755 (executable)
--- a/lustre/tests/sanity.sh
+++ b/lustre/tests/sanity.sh
@@ -20389,6 +20389,24 @@ test_230v()
 }
 run_test 230v "subdir migrated to the MDT where its parent is located"
 
+test_230w() {
+       (( MDSCOUNT > 1 )) || skip "needs >= 2 MDTs"
+       (( MDS1_VERSION >= $(version_code 2.14.53) )) ||
+               skip "Need MDS version at least 2.14.53"
+
+       mkdir -p $DIR/$tdir/sub || error "mkdir failed"
+
+       $LFS migrate -m 1 -c $MDSCOUNT -d $DIR/$tdir ||
+               error "migrate failed"
+
+       (( $($LFS getdirstripe -c $DIR/$tdir) == MDSCOUNT )) ||
+               error "$tdir stripe count mismatch"
+
+       (( $($LFS getdirstripe -c $DIR/$tdir/sub) == 0 )) ||
+               error "$tdir/sub is striped"
+}
+run_test 230w "non-recursive mode dir migration"
+
 test_231a()
 {
        # For simplicity this test assumes that max_pages_per_rpc

diff --git a/lustre/tests/sanityn.sh b/lustre/tests/sanityn.sh
index 7a4cc88..70a6af7 100755 (executable)
--- a/lustre/tests/sanityn.sh
+++ b/lustre/tests/sanityn.sh
@@ -4516,6 +4516,7 @@ test_80a() {
        local file
        local pid
 
+       mkdir_on_mdt0 $DIR1/$tdir
        mkdir -p $DIR1/$tdir/dir
        createmany -o $DIR1/$tdir/dir/f 10 ||
                error "create files under remote dir failed $i"

diff --git a/lustre/utils/lfs.c b/lustre/utils/lfs.c
index 58fed51..3d22c9d 100644 (file)
--- a/lustre/utils/lfs.c
+++ b/lustre/utils/lfs.c
@@ -564,7 +564,7 @@ command_t cmdlist[] = {
         "usage: swap_layouts <path1> <path2>"},
        {"migrate", lfs_setstripe_migrate, 0,
         "migrate directories and their inodes between MDTs.\n"
-        "usage: migrate [--mdt-count|-c STRIPE_COUNT]\n"
+        "usage: migrate [--mdt-count|-c STRIPE_COUNT] [--directory|-d]\n"
         "               [--mdt-hash|-H HASH_TYPE]\n"
         "               [--mdt-index|-m START_MDT_INDEX] [--verbose|-v]\n"
         "               DIRECTORY\n"
@@ -3448,6 +3448,8 @@ static int lfs_setstripe_internal(int argc, char **argv,
                                                .has_arg = required_argument},
        { .val = 'd',   .name = "delete",       .has_arg = no_argument},
        { .val = 'd',   .name = "destroy",      .has_arg = no_argument},
+       /* used with "lfs migrate -m" */
+       { .val = 'd',   .name = "directory",    .has_arg = no_argument},
        /* --non-direct is only valid in migrate mode */
        { .val = 'D',   .name = "non-direct",   .has_arg = no_argument },
        { .val = 'E',   .name = "comp-end",     .has_arg = required_argument},
@@ -3479,7 +3481,6 @@ static int lfs_setstripe_internal(int argc, char **argv,
        { .val = 'p',   .name = "pool",         .has_arg = required_argument },
 /* find        { .val = 'P',   .name = "print",        .has_arg = no_argument }, */
 /* getstripe { .val = 'q', .name = "quiet",    .has_arg = no_argument }, */
-/* getstripe { .val = 'r', .name = "recursive",        .has_arg = no_argument }, */
 /* getstripe { .val = 'R', .name = "raw",      .has_arg = no_argument }, */
        { .val = 'S',   .name = "stripe-size",  .has_arg = required_argument },
        { .val = 'S',   .name = "stripe_size",  .has_arg = required_argument },
@@ -3681,16 +3682,20 @@ static int lfs_setstripe_internal(int argc, char **argv,
                                lsa.lsa_stripe_count = LLAPI_LAYOUT_WIDE;
                        break;
                case 'd':
-                       /* delete the default striping pattern */
-                       delete = 1;
-                       if (opc == SO_MIRROR_SPLIT) {
-                               if (has_m_file) {
-                                       fprintf(stderr,
-                                             "%s %s: -d cannot used with -f\n",
-                                               progname, argv[0]);
-                                       goto usage_error;
+                       if (migrate_mode) {
+                               migrate_mdt_param.fp_max_depth = 1;
+                       } else {
+                               /* delete the default striping pattern */
+                               delete = 1;
+                               if (opc == SO_MIRROR_SPLIT) {
+                                       if (has_m_file) {
+                                               fprintf(stderr,
+                                                     "%s %s: -d cannot used with -f\n",
+                                                       progname, argv[0]);
+                                               goto usage_error;
+                                       }
+                                       mirror_flags |= MF_DESTROY;
                                }
-                               mirror_flags |= MF_DESTROY;
                        }
                        break;
                case 'D':

diff --git a/lustre/utils/liblustreapi.c b/lustre/utils/liblustreapi.c
index 511b48f..b55e51c 100644 (file)
--- a/lustre/utils/liblustreapi.c
+++ b/lustre/utils/liblustreapi.c
@@ -5474,6 +5474,9 @@ static int cb_migrate_mdt_init(char *path, int p, int *dp,
        data.ioc_inlbuf2 = (char *)lmu;
        data.ioc_inllen2 = lmv_user_md_size(lmu->lum_stripe_count,
                                            lmu->lum_magic);
+       /* reach bottom? */
+       if (param->fp_depth == param->fp_max_depth)
+               data.ioc_type = MDS_MIGRATE_NSONLY;
        ret = llapi_ioctl_pack(&data, &rawbuf, sizeof(raw));
        if (ret != 0) {
                llapi_error(LLAPI_MSG_ERROR, ret,
@@ -5515,6 +5518,11 @@ migrate:
        }
 
 out:
+       /* Do not get down anymore? */
+       if (param->fp_depth == param->fp_max_depth)
+               ret = 1;
+       param->fp_depth++;
+
        if (dp != NULL) {
                /*
                 * If the directory is being migration, we need

diff --git a/lustre/utils/wirecheck.c b/lustre/utils/wirecheck.c
index 8206d8a..0a619d5 100644 (file)
--- a/lustre/utils/wirecheck.c
+++ b/lustre/utils/wirecheck.c
@@ -1130,6 +1130,9 @@ check_mds_op_bias(void)
        CHECK_VALUE_X(MDS_TRUNC_KEEP_LEASE);
        CHECK_VALUE_X(MDS_PCC_ATTACH);
        CHECK_VALUE_X(MDS_CLOSE_UPDATE_TIMES);
+       CHECK_VALUE_X(MDS_SETSTRIPE_CREATE);
+       CHECK_VALUE_X(MDS_FID_OP);
+       CHECK_VALUE_X(MDS_MIGRATE_NSONLY);
 }
 
 static void

diff --git a/lustre/utils/wiretest.c b/lustre/utils/wiretest.c
index 2e9c11b..576dbc9 100644 (file)
--- a/lustre/utils/wiretest.c
+++ b/lustre/utils/wiretest.c
@@ -2433,6 +2433,12 @@ void lustre_assert_wire_constants(void)
                (unsigned)MDS_PCC_ATTACH);
        LASSERTF(MDS_CLOSE_UPDATE_TIMES == 0x00100000UL, "found 0x%.8xUL\n",
                (unsigned)MDS_CLOSE_UPDATE_TIMES);
+       LASSERTF(MDS_SETSTRIPE_CREATE == 0x00200000UL, "found 0x%.8xUL\n",
+               (unsigned)MDS_SETSTRIPE_CREATE);
+       LASSERTF(MDS_FID_OP == 0x00400000UL, "found 0x%.8xUL\n",
+               (unsigned)MDS_FID_OP);
+       LASSERTF(MDS_MIGRATE_NSONLY == 0x00800000UL, "found 0x%.8xUL\n",
+               (unsigned)MDS_MIGRATE_NSONLY);
 
        /* Checks for struct mdt_body */
        LASSERTF((int)sizeof(struct mdt_body) == 216, "found %lld\n",