From c6e7c0788d7cd766880d12eae6679782283dc479 Mon Sep 17 00:00:00 2001 From: Bobi Jam Date: Sun, 23 Sep 2018 00:42:37 +0800 Subject: [PATCH] LU-10258 lfs: lfs mirror copy command Add "lfs mirror copy" command to copy a mirror's content to other mirror(s) of a mirrored file. Usage: lfs mirror copy {--read-mirror|-i } {--write-mirror|-o [,,...]} Options: --read-mirror|-i This option indicates the content of which mirror specified by id0 needs to be read. The id0 is the numerical unique identifier for a mirror. --write-mirror|-o [,,...] This option indicates the content of which mirror(s) specified by mirror IDs needs to be written. The mirror IDs are separated with comma. If the mirror id -1 is used here, it means that all mirrors other than the read mirror are to be written. Note: Be ware that the written mirror(s) will be marked as non-stale mirror(s), be careful that after using this command, you could get a file with non-stale mirrors while containing different contents. Signed-off-by: Bobi Jam Change-Id: Id138368cdb29ec14b7c03a5db3b2dd1e0db5ea37 Reviewed-on: https://review.whamcloud.com/33220 Reviewed-by: Jian Yu Reviewed-by: Andreas Dilger Tested-by: Jenkins Tested-by: Maloo Reviewed-by: Oleg Drokin --- lustre/doc/lfs-mirror-copy.1 | 50 +++++ lustre/doc/lfs-mirror-read.1 | 1 + lustre/doc/lfs-mirror-write.1 | 4 +- lustre/include/lustre/lustreapi.h | 3 +- lustre/include/uapi/linux/lustre/lustre_user.h | 1 + lustre/lod/lod_object.c | 6 +- lustre/tests/mirror_io.c | 4 +- lustre/tests/sanity-flr.sh | 12 +- lustre/utils/lfs.c | 281 ++++++++++++++++++++++++- lustre/utils/liblustreapi_mirror.c | 3 +- 10 files changed, 346 insertions(+), 19 deletions(-) create mode 100644 lustre/doc/lfs-mirror-copy.1 diff --git a/lustre/doc/lfs-mirror-copy.1 b/lustre/doc/lfs-mirror-copy.1 new file mode 100644 index 0000000..68bef53 --- /dev/null +++ b/lustre/doc/lfs-mirror-copy.1 @@ -0,0 +1,50 @@ +.TH LFS-MIRROR-COPY 1 2018-09-23 "Lustre" "Lustre Utilities" +.SH NAME +lfs mirror copy \- copy a mirror's content to other mirror(s) of a mirrored file +.SH SYNOPSIS +.B lfs mirror copy +{\fB\-\-read-mirror|\-i\fR <\fIid0\fR>} +{\fB\-\-write-mirror|\-o\fR <\fIid1>[,,...]\fR} +<\fImirrored_file\fR> +.SH DESCRIPTION +This command copies a mirror's content to other mirror(s) of a mirrored file, +the file is specified by the path name \fImirrored_file\fR, the mirrors are +specified by their mirror ID. +.SH NOTE +Be ware that the written mirror(s) will be marked as non-stale mirror(s), +be careful that after using this command, you could get a file with non-stale +mirrors while containing different contents. +.br +.SH OPTIONS +.TP +.BR \-\-read-mirror|\-i\fR\ <\fIid0\fR> +This option indicates the content of which mirror specified by \fIid0\fR +needs to be read. The \fIid0\fR is the numerical unique identifier for +a mirror. +.TP +.BR \-\-write-mirror|\-o\fR\ <\fIid1\fR>[,<\fIid2\fR>,...] +This option indicates the content of which mirror(s) specified by mirror IDs +needs to be written. The mirror IDs are separated with comma. +If the mirror id \fB-1\fR is used here, it means that all mirrors other than +the read mirror are to be written. +.SH EXAMPLES +.TP +.B lfs mirror copy -i1 -o2,3 /mnt/lustre/file1 +Copy the content of mirror with mirror ID 1 to the mirrors with mirror ID 2 and +mirror ID 3 in /mnt/lustre/file1. +.TP +.B lfs mirror copy -i1 -o-1 /mnt/lustre/file1 +Copy the content of mirror with mirror ID 1 to all other mirrors in /mnt/lustre/file1. +.TP +.SH AUTHOR +The \fBlfs mirror copy\fR command is part of the Lustre filesystem. +.SH SEE ALSO +.BR lfs (1), +.BR lfs-setstripe (1), +.BR lfs-getstripe (1), +.BR lfs-mirror-create (1), +.BR lfs-mirror-extend (1), +.BR lfs-mirror-split (1), +.BR lfs-mirror-verify (1), +.BR lfs-mirror-read (1) +.BR lfs-mirror-write (1) diff --git a/lustre/doc/lfs-mirror-read.1 b/lustre/doc/lfs-mirror-read.1 index 44318a7..bde75cc 100644 --- a/lustre/doc/lfs-mirror-read.1 +++ b/lustre/doc/lfs-mirror-read.1 @@ -42,3 +42,4 @@ The \fBlfs mirror read\fR command is part of the Lustre filesystem. .BR lfs-mirror-split (1), .BR lfs-mirror-verify (1) .BR lfs-mirror-write (1) +.BR lfs-mirror-copy (1) diff --git a/lustre/doc/lfs-mirror-write.1 b/lustre/doc/lfs-mirror-write.1 index c0030ef..2faa402 100644 --- a/lustre/doc/lfs-mirror-write.1 +++ b/lustre/doc/lfs-mirror-write.1 @@ -20,7 +20,8 @@ a mirror. .BR \-\-inputfile|\-i\fR\ <\fIinput_file\fR> The path name of the input file, if not specified, the standard input stream will be used. The input stream or input_file cannot be the same mirrored file -as the \fImirrored_file\fR. +as the \fImirrored_file\fR. if you'd like to copy a mirror's content to +another mirror of the same mirrored file, use \fBlfs mirror copy\fR command. .SH EXAMPLES .TP .B lfs mirror write --mirror-id 3 /mnt/lustre/file1 < /tmp/m2 @@ -41,3 +42,4 @@ The \fBlfs mirror write\fR command is part of the Lustre filesystem. .BR lfs-mirror-split (1), .BR lfs-mirror-verify (1), .BR lfs-mirror-read (1) +.BR lfs-mirror-copy (1) diff --git a/lustre/include/lustre/lustreapi.h b/lustre/include/lustre/lustreapi.h index d2a3608..87762b4d 100644 --- a/lustre/include/lustre/lustreapi.h +++ b/lustre/include/lustre/lustreapi.h @@ -1004,8 +1004,7 @@ int llapi_mirror_set(int fd, unsigned int id); int llapi_mirror_clear(int fd); ssize_t llapi_mirror_read(int fd, unsigned int id, void *buf, size_t count, off_t pos); -ssize_t llapi_mirror_copy_many(int fd, unsigned int src, - unsigned int *dst, size_t count); +ssize_t llapi_mirror_copy_many(int fd, __u16 src, __u16 *dst, size_t count); int llapi_mirror_copy(int fd, unsigned int src, unsigned int dst, off_t pos, size_t count); diff --git a/lustre/include/uapi/linux/lustre/lustre_user.h b/lustre/include/uapi/linux/lustre/lustre_user.h index b75f0c6..a99aaa2 100644 --- a/lustre/include/uapi/linux/lustre/lustre_user.h +++ b/lustre/include/uapi/linux/lustre/lustre_user.h @@ -698,6 +698,7 @@ struct lov_comp_md_entry_v1 { #define SEQ_ID_MASK SEQ_ID_MAX /* bit 30:16 of lcme_id is used to store mirror id */ #define MIRROR_ID_MASK 0x7FFF0000 +#define MIRROR_ID_NEG 0x8000 #define MIRROR_ID_SHIFT 16 static inline __u32 pflr_id(__u16 mirror_id, __u16 seqid) diff --git a/lustre/lod/lod_object.c b/lustre/lod/lod_object.c index 84a7e80..aed60a8 100644 --- a/lustre/lod/lod_object.c +++ b/lustre/lod/lod_object.c @@ -6143,10 +6143,14 @@ static int lod_prepare_resync_mirror(const struct lu_env *env, { struct lod_thread_info *info = lod_env_info(env); struct lod_layout_component *lod_comp; + bool neg = !!(MIRROR_ID_NEG & mirror_id); int i; + mirror_id &= ~MIRROR_ID_NEG; + for (i = 0; i < lo->ldo_mirror_count; i++) { - if (lo->ldo_mirrors[i].lme_id != mirror_id) + if ((!neg && lo->ldo_mirrors[i].lme_id != mirror_id) || + (neg && lo->ldo_mirrors[i].lme_id == mirror_id)) continue; lod_foreach_mirror_comp(lod_comp, lo, i) { diff --git a/lustre/tests/mirror_io.c b/lustre/tests/mirror_io.c index d3fb3a5..ab39f69 100644 --- a/lustre/tests/mirror_io.c +++ b/lustre/tests/mirror_io.c @@ -204,7 +204,7 @@ static void mirror_dump(int argc, char *argv[]) free(buf); } -static size_t add_tids(unsigned int *ids, size_t count, char *arg) +static size_t add_tids(__u16 *ids, size_t count, char *arg) { while (*arg) { char *end; @@ -240,7 +240,7 @@ static void mirror_copy(int argc, char *argv[]) int c; int i; - unsigned int ids[4096] = { 0 }; + __u16 ids[4096] = { 0 }; size_t count = 0; ssize_t result; diff --git a/lustre/tests/sanity-flr.sh b/lustre/tests/sanity-flr.sh index 2298ec1..fb94b57b 100644 --- a/lustre/tests/sanity-flr.sh +++ b/lustre/tests/sanity-flr.sh @@ -1327,21 +1327,15 @@ test_37() local osts=$(comma_list $(osts_nodes)) - # define OBD_FAIL_OST_SKIP_LV_CHECK 0x241 - do_nodes $osts lctl set_param fail_loc=0x241 - - mirror_io copy -i ${mirror_array[0]} \ - -t $(echo ${mirror_array[@]:1} | tr ' ' ',') $tf || - error "mirror copy error" - - do_nodes $osts lctl set_param fail_loc=0 + $LFS mirror copy -i ${mirror_array[0]} -o-1 $tf || + error "mirror copy error" # verify copying is successful by checking checksums remount_client $MOUNT for i in ${mirror_array[@]}; do sum=$($LFS mirror read -N $i $tf | md5sum) [ "$sum" = "${checksums[1]}" ] || - error "$i: mismatch checksum after copy" + error "$i: mismatch checksum after copy \'$sum\'" done rm -f $tf diff --git a/lustre/utils/lfs.c b/lustre/utils/lfs.c index 4eefe49..b16af99 100644 --- a/lustre/utils/lfs.c +++ b/lustre/utils/lfs.c @@ -126,6 +126,7 @@ static inline int lfs_mirror_resync(int argc, char **argv); static inline int lfs_mirror_verify(int argc, char **argv); static inline int lfs_mirror_read(int argc, char **argv); static inline int lfs_mirror_write(int argc, char **argv); +static inline int lfs_mirror_copy(int argc, char **argv); enum setstripe_origin { SO_SETSTRIPE, @@ -293,6 +294,10 @@ command_t mirror_cmdlist[] = { .pc_help = "Write to a specified mirror of a file.\n" "usage: lfs mirror write <--mirror-id|-N " "[--inputfile|-i ] \n" }, + { .pc_name = "copy", .pc_func = lfs_mirror_copy, + .pc_help = "Copy a specified mirror to other mirror(s) of a file.\n" + "usage: lfs mirror copy <--read-mirror|-i > " + "<--write-mirror|-o > \n" }, { .pc_name = "resync", .pc_func = lfs_mirror_resync, .pc_help = "Resynchronizes out-of-sync mirrored file(s).\n" "usage: lfs mirror resync [--only ] " @@ -593,6 +598,7 @@ command_t cmdlist[] = { "lfs mirror resync - resynchronize out-of-sync mirrored file(s)\n" "lfs mirror read - read a mirror content of a mirrored file\n" "lfs mirror write - write to a mirror of a mirrored file\n" + "lfs mirror copy - copy a mirror to other mirror(s) of a file\n" "lfs mirror verify - verify mirrored file(s)\n"}, {"getsom", lfs_getsom, 0, "To list the SOM info for a given file.\n" "usage: getsom [-s] [-b] [-f] \n" @@ -8763,6 +8769,279 @@ close_fd: return rc; } +struct collect_ids_data { + __u16 *cid_ids; + int cid_count; + __u16 cid_exclude; +}; + +static int collect_mirror_id(struct llapi_layout *layout, void *cbdata) +{ + struct collect_ids_data *cid = cbdata; + uint32_t id; + int rc; + + rc = llapi_layout_mirror_id_get(layout, &id); + if (rc < 0) + return rc; + + if ((__u16)id != cid->cid_exclude) { + int i; + + for (i = 0; i < cid->cid_count; i++) { + /* already collected the mirror id */ + if (id == cid->cid_ids[i]) + return LLAPI_LAYOUT_ITER_CONT; + } + cid->cid_ids[cid->cid_count] = id; + cid->cid_count++; + } + + return LLAPI_LAYOUT_ITER_CONT; +} + +static inline int get_other_mirror_ids(int fd, __u16 *ids, __u16 exclude_id) +{ + struct llapi_layout *layout; + struct collect_ids_data cid = { .cid_ids = ids, + .cid_count = 0, + .cid_exclude = exclude_id, }; + int rc; + + layout = llapi_layout_get_by_fd(fd, 0); + if (layout == NULL) { + fprintf(stderr, "could not get layout\n"); + return -EINVAL; + } + + rc = llapi_layout_comp_iterate(layout, collect_mirror_id, &cid); + if (rc < 0) { + fprintf(stderr, "failed to iterate layout\n"); + llapi_layout_free(layout); + + return rc; + } + llapi_layout_free(layout); + + return cid.cid_count; +} + +static inline int lfs_mirror_copy(int argc, char **argv) +{ + int rc = CMD_HELP; + __u16 read_mirror_id = 0; + __u16 ids[128] = { 0 }; + int count = 0; + struct llapi_layout *layout = NULL; + struct llapi_resync_comp comp_array[1024] = { { 0 } }; + int comp_size = 0; + char *fname; + int fd = 0; + int c; + int i; + ssize_t copied; + struct ll_ioc_lease *ioc = NULL; + struct ll_ioc_lease_id *resync_ioc; + + struct option long_opts[] = { + { .val = 'i', .name = "read-mirror", .has_arg = required_argument }, + { .val = 'o', .name = "write-mirror", .has_arg = required_argument }, + { .name = NULL } }; + + while ((c = getopt_long(argc, argv, "i:o:", long_opts, NULL)) >= 0) { + char *end; + + switch (c) { + case 'i': + read_mirror_id = strtoul(optarg, &end, 0); + if (*end != '\0' || read_mirror_id == 0) { + fprintf(stderr, + "%s %s: invalid read mirror ID '%s'\n", + progname, argv[0], optarg); + return rc; + } + break; + case 'o': + if (!strcmp(optarg, "-1")) { + /* specify all other mirrors */ + ids[0] = (__u16)-1; + count = 1; + } else { + count = parse_mirror_ids((__u16 *)ids, + ARRAY_SIZE(ids), + optarg); + if (count < 0) + return rc; + } + break; + default: + fprintf(stderr, "%s: option '%s' unrecognized\n", + progname, argv[optind - 1]); + return -EINVAL; + } + } + + if (argc == optind) { + fprintf(stderr, "%s %s: no mirrored file provided\n", + progname, argv[0]); + return rc; + } else if (argc > optind + 1) { + fprintf(stderr, "%s %s: too many files\n", progname, argv[0]); + return rc; + } + + if (read_mirror_id == 0) { + fprintf(stderr, + "%s %s: no valid read mirror ID %d is provided\n", + progname, argv[0], read_mirror_id); + return rc; + } + + if (count == 0) { + fprintf(stderr, + "%s %s: no write mirror ID is provided\n", + progname, argv[0]); + return rc; + } + + for (i = 0; i < count; i++) { + if (read_mirror_id == ids[i]) { + fprintf(stderr, + "%s %s: read and write mirror ID cannot be the same\n", + progname, argv[0]); + return rc; + } + } + + /* open mirror file */ + fname = argv[optind]; + + fd = open(fname, O_DIRECT | O_RDWR); + if (fd < 0) { + fprintf(stderr, "%s %s: cannot open '%s': %s\n", + progname, argv[0], fname, strerror(errno)); + return rc; + } + + /* write to all other mirrors */ + if (ids[0] == (__u16)-1) { + count = get_other_mirror_ids(fd, ids, read_mirror_id); + if (count <= 0) { + rc = count; + fprintf(stderr, + "%s %s: failed to get other mirror ids in '%s': %d\n", + progname, argv[0], fname, rc); + goto close_fd; + } + } + + /* verify mirror id */ + rc = verify_mirror_id_by_fd(fd, read_mirror_id); + if (rc) { + fprintf(stderr, + "%s %s: cannot find mirror with ID %u in '%s'\n", + progname, argv[0], read_mirror_id, fname); + goto close_fd; + } + + for (i = 0; i < count; i++) { + rc = verify_mirror_id_by_fd(fd, ids[i]); + if (rc) { + fprintf(stderr, + "%s %s: cannot find mirror with ID %u in '%s'\n", + progname, argv[0], ids[i], fname); + goto close_fd; + } + } + + ioc = calloc(sizeof(*ioc) + sizeof(__u32) * 4096, 1); + if (ioc == NULL) { + fprintf(stderr, + "%s %s: cannot alloc comp id array for ioc: %s\n", + progname, argv[0], strerror(errno)); + rc = -errno; + goto close_fd; + } + + /* get stale component info */ + layout = llapi_layout_get_by_fd(fd, 0); + if (layout == NULL) { + fprintf(stderr, "%s %s: failed to get layout of '%s': %s\n", + progname, argv[0], fname, strerror(errno)); + rc = -errno; + goto free_ioc; + } + comp_size = llapi_mirror_find_stale(layout, comp_array, + ARRAY_SIZE(comp_array), + ids, count); + llapi_layout_free(layout); + if (comp_size < 0) { + rc = comp_size; + goto free_ioc; + } + + /* prepare target mirror components instantiation */ + resync_ioc = (struct ll_ioc_lease_id *)ioc; + resync_ioc->lil_mode = LL_LEASE_WRLCK; + resync_ioc->lil_flags = LL_LEASE_RESYNC; + if (count == 1) + resync_ioc->lil_mirror_id = ids[0]; + else + resync_ioc->lil_mirror_id = read_mirror_id | MIRROR_ID_NEG; + rc = llapi_lease_set(fd, ioc); + if (rc < 0) { + fprintf(stderr, + "%s %s: '%s' llapi_lease_get_ext failed: %s\n", + progname, argv[0], fname, strerror(errno)); + goto free_ioc; + } + + copied = llapi_mirror_copy_many(fd, read_mirror_id, ids, count); + if (copied < 0) { + rc = copied; + fprintf(stderr, "%s %s: copy error: %d\n", + progname, argv[0], rc); + goto free_ioc; + } + + fprintf(stdout, "mirror copied successfully: "); + for (i = 0; i < copied; i++) + fprintf(stdout, "%d ", ids[i]); + fprintf(stdout, "\n"); + + ioc->lil_mode = LL_LEASE_UNLCK; + ioc->lil_flags = LL_LEASE_RESYNC_DONE; + ioc->lil_count = 0; + for (i = 0; i < comp_size; i++) { + int j; + + for (j = 0; j < copied; j++) { + if (comp_array[i].lrc_mirror_id != ids[j]) + continue; + + ioc->lil_ids[ioc->lil_count] = comp_array[i].lrc_id; + ioc->lil_count++; + } + } + rc = llapi_lease_set(fd, ioc); + if (rc <= 0) { + if (rc == 0) + rc = -EBUSY; + fprintf(stderr, + "%s %s: release lease lock of '%s' failed: %s\n", + progname, argv[0], fname, strerror(errno)); + goto free_ioc; + } + + rc = 0; + +free_ioc: + free(ioc); +close_fd: + close(fd); + + return rc; +} /** * struct verify_chunk - Mirror chunk to be verified. * @chunk: [start, end) of the chunk. @@ -9099,8 +9378,6 @@ int lfs_mirror_verify_chunk(int fd, size_t file_size, for (i = 1; i < chunk->mirror_count; i++) { if (crc_array[i] != crc_array[0]) { rc = -EINVAL; - if (!verbose) - goto error; fprintf(stderr, "%s: chunk "DEXT" has different checksum value on mirror %u and mirror %u.\n", diff --git a/lustre/utils/liblustreapi_mirror.c b/lustre/utils/liblustreapi_mirror.c index c7e5726..bae3b09 100644 --- a/lustre/utils/liblustreapi_mirror.c +++ b/lustre/utils/liblustreapi_mirror.c @@ -216,8 +216,7 @@ int llapi_mirror_truncate(int fd, unsigned int id, off_t length) * \result > 0 Number of mirrors successfully copied * \result < 0 The last seen error */ -ssize_t llapi_mirror_copy_many(int fd, unsigned int src, unsigned int *dst, - size_t count) +ssize_t llapi_mirror_copy_many(int fd, __u16 src, __u16 *dst, size_t count) { const size_t buflen = 4 * 1024 * 1024; /* 4M */ void *buf; -- 1.8.3.1