From: Feng Lei Date: Fri, 22 Nov 2024 01:18:53 +0000 (+0800) Subject: LU-18454 utils: 'lfs migrate' can read filenames from file X-Git-Tag: 2.16.54~76 X-Git-Url: https://git.whamcloud.com/?a=commitdiff_plain;h=b85d28309764e9988005a8e98cfd4ffd900c090c;p=fs%2Flustre-release.git LU-18454 utils: 'lfs migrate' can read filenames from file Enhance 'lfs migrate' and 'lfs mirror extend' command to be able to read filenames from a file or pipeline and handle all the files in one process. When -0 or --null is specified, read filenames from stdin by default. Each filename is ended with a NUL char. So it can work together with 'lfs find -0' very well. For example: # lfs find /mnt/lustre --ost 0 -0 | lfs migrate -0 --ost 1 When --files-from=LISTFILE is specified, read filenames from LISTFILE. One line for each filename. If LISTFILE is -, read from stdin. If --null is specified at the same time, filenames are separated by NUL char in LISTFILE. Filenames can be specified on command line directly as before. Signed-off-by: Lei Feng Test-Parameters: trivial Change-Id: Id12aaba2b52a8a541a4d552d37facf6fb0fadc57 Reviewed-on: https://review.whamcloud.com/c/fs/lustre-release/+/57104 Tested-by: jenkins Tested-by: Maloo Reviewed-by: Andreas Dilger Reviewed-by: Etienne AUJAMES Reviewed-by: Olaf Faaland Reviewed-by: Oleg Drokin --- diff --git a/lustre/doc/lfs-migrate.1 b/lustre/doc/lfs-migrate.1 index 201cffa..f39ef89 100644 --- a/lustre/doc/lfs-migrate.1 +++ b/lustre/doc/lfs-migrate.1 @@ -6,7 +6,7 @@ lfs-migrate \- migrate files or directories between MDTs or OSTs. .RB [ -h ] .RB [ -v ] .RI [ SETSTRIPE_OPTIONS " ... ]" -.IR FILE " ..." +.IR FILES_FROM .SY "lfs migrate" .B -m .I START_MDT_INDEX @@ -14,14 +14,33 @@ lfs-migrate \- migrate files or directories between MDTs or OSTs. .I DIRECTORY .YS .SH DESCRIPTION -Migrate OST objects between OSTs for the specified -.IR FILE , +Migrate OST objects between OSTs for the file(s) specified by \fIFILES_FROM\fR, or recursively migrate .I DIRECTORY and all inodes/directories therein between MDTs. + .SH OPTIONS .SS OST MIGRATE OPTIONS .P +.IR FILES_FROM +may be: +.TP +.IR FILENAME " [...]" +File names are listed on command line. +Multiple file names are separated by space char. +.TP +.BR -0 ", " --null +Read file names from stdin by default. Each file name is followed by a NUL char. +Usually is used after a pipeline from \fBlfs find --print0\fR command. +.TP +.BR --files-from = \fILIST_FILE +Read file names from file \fILIST_FILE\fR. One line for each file name. +If \fILIST_FILE\fR is \fB-\fR, read from stdin. +If \fB--null\fR is specified at the same time, +file names are separated by a NUL char. + +.SH OST MIGRATE OPTIONS +.P The .B lfs migrate command can be used for moving files from one (or more) OSTs to other @@ -253,7 +272,7 @@ component to copy the default layout from the specified .BR topdir : .RS .EX -.B # lfs find dir -type f -L mdt -0 | xargs -0 lfs migrate --copy topdir +.B # lfs find dir -type f -L mdt -0 | lfs migrate -0 --copy topdir .EE .RE and finally migrate the directory diff --git a/lustre/doc/lfs-mirror-extend.1 b/lustre/doc/lfs-mirror-extend.1 index 0992cda..371d679 100644 --- a/lustre/doc/lfs-mirror-extend.1 +++ b/lustre/doc/lfs-mirror-extend.1 @@ -3,27 +3,36 @@ lfs-mirror-extend \- add mirror(s) to an existing file .SH SYNOPSIS .SY "lfs mirror extend" -.RB [ --no-verify ] -.RB [ --mirror-count | -N\c -.RI [ MIRROR_COUNT ]] -.RB [ --bandwidth-limit | -W -.IR BANDWIDTH ] -.RB [ --stats | --stats-interval\c -.RI = STATS_INTERVAL ] -.RI [ SETSTRIPE_OPTIONS |\c -.B -f -.IR VICTIM_FILE ] -.IR FILENAME " ..." + [\fB--bandwidth-limit\fR|\fB-W\fR \fIMB_PER_SEC\fR] + [\fB--mirror-count\fR|\fB-N\fR[\fIMIRROR_COUNT\fR]] + [\fB--no-verify\fR] + [\fB--stats\fR|\fB--stats-interval\fR=\fISECONDS\fR] + [\fISETSTRIPE_OPTIONS\fR|\fB-f\fR \fIVICTIM_FILE\fR] + \fIFILES_FROM\fR .YS .SH DESCRIPTION -This command adds mirror(s) to an existing file specified by the path name -.IR FILENAME . +This command adds mirror(s) to existing file(s) specified by \fIFILES_FROM\fR. +.P +The file to be extended can already be a mirrored file, or just a regular +non-mirrored file. If it's a non-mirrored file, the command will convert it +to a mirrored file. .P -The file -.I FILENAME -can already be a mirrored file, or just a regular non-mirrored file. -If it's a non-mirrored file, -then the command will convert it to a mirrored file. +.I FILES_FROM +may be: +.TP +.IR FILENAME " [...]" +File names are listed on command line. +Multiple file names are separated by space char. +.TP +.BR -0 ", " --null +Read file names from stdin by default. Each file name is followed by a NUL char. +Usually is used after a pipeline from \fBlfs find --print0\fR command. +.TP +.BR --files-from = \fILIST_FILE +Read file names from file \fILIST_FILE\fR. One line for each file name. +If \fILIST_FILE\fR is \fB-\fR, read from stdin. +If \fB--null\fR is specified at the same time, +file names are separated by a NUL char. .P The .BR --mirror-count | -N diff --git a/lustre/tests/sanity.sh b/lustre/tests/sanity.sh index 77ed27b..78daede 100755 --- a/lustre/tests/sanity.sh +++ b/lustre/tests/sanity.sh @@ -8333,6 +8333,36 @@ test_56x() { } run_test 56x "lfs migration support" +test_56xB() { + local td=$DIR/$tdir + local tf1=$td/${tfile}_1 + local tf2=$td/${tfile}_2 + local tf3=$td/${tfile}_3 + local flist=/tmp/flist.$$ + + (( $OSTCOUNT >= 2 )) || skip "needs >= 2 OSTs" + + test_mkdir $td + dd if=/dev/urandom of=$tf1 bs=1K count=1 || error "failed to create $tf1" + dd if=/dev/urandom of=$tf2 bs=1K count=1 || error "failed to create $tf2" + dd if=/dev/urandom of=$tf3 bs=1K count=1 || error "failed to create $tf3" + + stack_trap "rm -f $flist" + + $LFS find $td -type f > $flist || error "failed to generate list file" + $LFS migrate -o 0 --files-from=$flist || + error "failed to run lfs migrate with --files-from" + + $LFS find $td -type f -print0 | $LFS migrate -o 1 --null || + error "failed to run lfs migrate from pipe" + + $LFS find $td -type f -print0 > $flist || + error "failed to generate 0-ending list file" + $LFS migrate -o 0 -0 --files-from=$flist || + error "failed to run lfs migrate with -0 --files-from" +} +run_test 56xB "lfs migrate with -0, --null, --files-from arguments" + test_56xa() { [[ $OSTCOUNT -lt 2 ]] && skip_env "needs >= 2 OSTs" check_swap_layouts_support diff --git a/lustre/utils/lfs.c b/lustre/utils/lfs.c index af97818..7ecd998 100755 --- a/lustre/utils/lfs.c +++ b/lustre/utils/lfs.c @@ -243,6 +243,8 @@ static inline int lfs_mirror_delete(int argc, char **argv) SSM_CMD_COMMON("migrate ") \ "\t\t[--block|-b] [--non-block|-n]\n" \ "\t\t[--non-direct|-D] [--verbose|-v] FILENAME\n" \ + "\t\t[--non-direct|-D] [--verbose|-v]\n" \ + "\t\t-0|--null|--files-from=LIST_FILE|FILENAME ...\n" #define SETDIRSTRIPE_USAGE \ " [--mdt-count|-c stripe_count>\n" \ @@ -302,8 +304,8 @@ command_t mirror_cmdlist[] = { "\t\t[--no-verify] [--stats|--stats-interval=STATS_INTERVAL]\n" "\t\t[--bandwidth-limit|--W BANDWIDTH]\n" "\t\t[-f VICTIM_FILE]\n" - "\t\t" SSM_SETSTRIPE_OPT "]" - " FILENAME ...\n" }, + "\t\t" SSM_SETSTRIPE_OPT "]\n" + "\t\t-0|--null|--files-from=LIST_FILE|FILENAME ...\n" }, { .pc_name = "split", .pc_func = lfs_mirror_split, .pc_help = "Split a mirrored file.\n" "usage: lfs mirror split {--mirror-id MIRROR_ID |\n" @@ -3725,6 +3727,7 @@ enum { LFS_QUOTA_ISOFTLIMIT_OPT, LFS_QUOTA_IHARDLIMIT_OPT, LFS_QUOTA_IGRACE_OPT, + LFS_FILES_FROM, }; #ifndef LCME_USER_MIRROR_FLAGS @@ -3784,13 +3787,14 @@ static int lfs_setstripe_internal(int argc, char **argv, unsigned long long bandwidth_bytes_sec = 0; unsigned long long bandwidth_unit = ONE_MB; long stats_interval_sec = 0; + bool null_mode = false; + const char *files_from = NULL; + FILE *files_from_fp = NULL; + int delim = '\n'; + char *buf = NULL; + size_t bufsize = 0; struct option long_opts[] = { -/* find { .val = '0', .name = "null", .has_arg = no_argument }, */ -/* find { .val = 'A', .name = "atime", .has_arg = required_argument }*/ - /* --block is only valid in migrate mode */ - { .val = 'b', .name = "block", .has_arg = no_argument }, -/* find { .val = 'B', .name = "btime", .has_arg = required_argument }*/ { .val = LFS_COMP_ADD_OPT, .name = "comp-add", .has_arg = no_argument }, { .val = LFS_COMP_ADD_OPT, @@ -3826,6 +3830,13 @@ static int lfs_setstripe_internal(int argc, char **argv, { .val = LFS_STATS_INTERVAL_OPT, .name = "stats-interval", .has_arg = required_argument}, + { .val = LFS_FILES_FROM, + .name = "files-from", .has_arg = required_argument}, + { .val = '0', .name = "null", .has_arg = no_argument }, + /* find { .val = 'A', .name = "atime", .has_arg = required_argument }*/ + /* --block is only valid in migrate mode */ + { .val = 'b', .name = "block", .has_arg = no_argument }, + /* find { .val = 'B', .name = "btime", .has_arg = required_argument }*/ { .val = 'c', .name = "stripe-count", .has_arg = required_argument}, { .val = 'c', .name = "stripe_count", .has_arg = required_argument}, { .val = 'c', .name = "mdt-count", .has_arg = required_argument}, @@ -3899,7 +3910,7 @@ static int lfs_setstripe_internal(int argc, char **argv, snprintf(cmd, sizeof(cmd), "%s %s", progname, argv[0]); progname = cmd; while ((c = getopt_long(argc, argv, - "bc:C:dDE:f:hH:i:I:m:N::no:p:L:s:S:vx:W:y:z:", + "0bc:C:dDE:f:hH:i:I:m:N::no:p:L:s:S:vx:W:y:z:", long_opts, NULL)) >= 0) { size_units = 1; switch (c) { @@ -4058,6 +4069,12 @@ static int lfs_setstripe_internal(int argc, char **argv, } clear_hash_fixed = true; break; + case LFS_FILES_FROM: + files_from = optarg; + break; + case '0': + null_mode = true; + break; case 'b': if (!migrate_mode) { fprintf(stderr, @@ -4471,7 +4488,41 @@ create_mirror: fname = argv[optind]; - if (optind == argc) { + /* for 'lfs migrate' and 'lfs mirror extend' command, + * at least one of FILE/--null/--files-from=LIST_FILE must be specified. + * If both --null and --files-from=LIST_FILE are specified, read + * filenames from LIST_FILE and use '\0' as delimiter. + */ + if (opc == SO_MIGRATE || opc == SO_MIRROR_EXTEND) { + int num = 0; + + if (optind < argc) + num++; + if (null_mode) { + if (files_from_fp == NULL) + files_from_fp = stdin; + delim = 0; + num++; + } + if (files_from != NULL) { + if (strcmp("-", files_from) == 0) + files_from_fp = stdin; + else + files_from_fp = fopen(files_from, "r"); + if (files_from_fp == NULL) { + result = -errno; + fprintf(stderr, "%s %s: failed to open filelist file '%s'\n", + progname, argv[0], files_from); + goto error; + } + num++; + } + if (num < 1) { + fprintf(stderr, "%s %s: at least one of FILE/--null/--files-from=LIST_FILE must be specified\n", + progname, argv[0]); + goto usage_error; + } + } else if (optind == argc) { fprintf(stderr, "%s %s: FILE must be specified\n", progname, argv[0]); goto usage_error; @@ -4774,8 +4825,34 @@ create_mirror: } } - for (fname = argv[optind]; (optind < argc) && (fname != NULL); - fname = argv[++optind]) { + while (true) { + if (files_from_fp == NULL) { + /* file names from arguments */ + fname = argv[optind++]; + if (optind > argc || fname == NULL) + break; + } else { + /* file names from file/stdin */ + ssize_t len; + + errno = 0; + len = getdelim(&buf, &bufsize, delim, files_from_fp); + if (len == -1) { + if (errno != 0) { /* error */ + result = -errno; + fprintf(stderr, "%s %s: failed to read from list file\n", + progname, argv[0]); + goto error; + } else { /* EOF */ + break; + } + } + /* remove possible trailing '\n' */ + if (buf[len - 1] == '\n') + buf[len - 1] = '\0'; + fname = buf; + } + if (from_copy) { layout = layout_get_by_name_or_fid(template ?: fname, fname, 0, O_RDONLY); @@ -4920,6 +4997,9 @@ usage_error: error: llapi_layout_free(layout); lfs_mirror_list_free(mirror_list); + if (files_from_fp != NULL && files_from_fp != stdin) + fclose(files_from_fp); + free(buf); return result; }