From f1daa8fc6575e5b9e4a2f1f2ae4ceaefb889a694 Mon Sep 17 00:00:00 2001 From: Jian Yu Date: Sat, 20 Jan 2018 10:24:25 -0800 Subject: [PATCH] LU-10287 flr: lfs mirror verify command This patch adds "lfs mirror verify" command to verify that each SYNC mirror of a mirrored file contains exactly the same data. It supports specifying multiple mirrored files in one command line. Usage: lfs mirror verify [--only ] [--verbose|-v] [ ...] Options: --only Only verify the mirrors specified by mirror_ids contain exactly the same data. This option cannot be used when multiple mirrored files are specified. --verbose|-v With this option, the command will print where the differences are if the data do not match. Otherwise, the command will just return an error in that case. Test-Parameters: testlist=sanity-flr Signed-off-by: Jian Yu Change-Id: Ib436dea73a8e7a0f8e30b246bb0062023a97cb81 Reviewed-on: https://review.whamcloud.com/30387 Tested-by: Jenkins Reviewed-by: Jinshan Xiong Reviewed-by: Andreas Dilger Tested-by: Maloo Reviewed-by: Oleg Drokin --- lustre/doc/lfs-mirror-verify.1 | 42 ++- lustre/tests/sanity-flr.sh | 97 +++++ lustre/utils/Makefile.am | 2 +- lustre/utils/lfs.c | 818 ++++++++++++++++++++++++++++++++++++++++- 4 files changed, 943 insertions(+), 16 deletions(-) diff --git a/lustre/doc/lfs-mirror-verify.1 b/lustre/doc/lfs-mirror-verify.1 index 98a7f47..a73edc4 100644 --- a/lustre/doc/lfs-mirror-verify.1 +++ b/lustre/doc/lfs-mirror-verify.1 @@ -1,37 +1,53 @@ .TH LFS-MIRROR-VERIFY 1 2017-07-25 "Lustre" "Lustre Utilities" .SH NAME -lfs mirror verify \- verify a mirrored file +lfs mirror verify \- verify mirrored file(s) .SH SYNOPSIS .B lfs mirror verify -[\fB\-\-only\fR <\fImirror_id\fR[,...]>] -<\fImirrored_file\fR> +[\fB\-\-only\fR <\fImirror_id\fR,\fImirror_id2\fR[,...]>] +[\fB\-\-verbose\fR|\fB\-v\fR] <\fImirrored_file\fR> [<\fImirrored_file2\fR> ...] .SH DESCRIPTION This command verifies that each SYNC mirror of a mirrored file specified by the -path name \fImirrored_file\fR contains exactly the same data. +path name \fImirrored_file\fR contains exactly the same data. It supports +specifying multiple mirrored files in one command line. .br This is a scrub tool that should be run in regular basis to make sure that mirrored files are not corrupted. The command won't repair the file if it turns out to be corrupted. Usually administrator should check the file content from -each mirror and decide which one is correct and then invoke \fBlfs\ mirror -\ resync\fR to repair it manually. +each mirror and decide which one is correct and then invoke \fBlfs\ mirror\ + resync\fR to repair it manually. .br -If \fB\-\-only\fR <\fImirror_id\fR[,...]> option is specified, then the -command will verify the mirror(s) specified by \fImirror_id\fR(s) contains -exactly the same data as the other mirrors for the mirrored file. +If \fB\-\-only\fR <\fImirror_id\fR,\fImirror_id2\fR[,...]> option is specified, +then the command will only verify the mirrors specified by \fImirror_id\fRs +contain exactly the same data. At least two \fImirror_id\fRs are required. +This option cannot be used when multiple mirrored files are specified. +.br +If \fB\-\-verbose\fR|\fB\-v\fR option is specified, then the command will print +where the differences are if the data do not match. Otherwise, the command will +just return an error in that case. This option can be repeated for multiple +times to print more information. .SH OPTIONS .TP -.BR \-\-only\fR\ <\fImirror_id\fR[,...]> -This option indicates which mirror(s) specified by \fImirror_id\fR(s) needs to +.B \fB\-\-only\fR <\fImirror_id\fR,\fImirror_id2\fR[,...]> +This option indicates which mirrors specified by \fImirror_id\fRs need to be verified. The \fImirror_id\fR is the numerical unique identifier for a mirror. Multiple \fImirror_id\fRs are separated by comma. +.TP +.B \fB\-\-verbose\fR|\fB\-v\fR +This option indicates the command will print where the differences are if the +data do not match. .SH EXAMPLES .TP .B lfs mirror verify /mnt/lustre/file1 Verify that each mirror of /mnt/lustre/file1 contains exactly the same data. .TP +.B lfs mirror verify -v /mnt/lustre/file1 /mnt/lustre/file2 +Separately verify that each mirror of /mnt/lustre/file1 and /mnt/lustre/file2 +contain exactly the same data. Print where the differences are if the data do +not match. +.TP .B lfs mirror verify --only 4,5 /mnt/lustre/file1 -Verify mirrors with mirror ID 4 and 5 contain exactly the same data as other -mirrors for /mnt/lustre/file1. +Verify mirrors with mirror ID 4 and 5 of /mnt/lustre/file1 contain exactly +the same data. .SH AUTHOR The \fBlfs mirror verify\fR command is part of the Lustre filesystem. .SH SEE ALSO diff --git a/lustre/tests/sanity-flr.sh b/lustre/tests/sanity-flr.sh index a64cb98..8164ee6 100644 --- a/lustre/tests/sanity-flr.sh +++ b/lustre/tests/sanity-flr.sh @@ -1420,6 +1420,103 @@ test_41() { } run_test 41 "lfs mirror resync check" +test_42() { + [[ $OSTCOUNT -lt 4 ]] && skip "need >= 4 OSTs" && return + + local td=$DIR/$tdir + local tf=$td/$tfile + local mirror_cmd="$LFS mirror verify" + local i + + # create parent directory + mkdir $td || error "mkdir $td failed" + + $mirror_cmd &> /dev/null && error "no file name given" + $mirror_cmd $tf &> /dev/null && error "cannot stat file $tf" + $mirror_cmd $td &> /dev/null && error "$td is not a regular file" + + # create mirrored files + $LFS mirror create -N -E 4M -E 10M -E EOF $tf || + error "create mirrored file $tf failed" + $LFS mirror create -N -E 2M -E EOF \ + -N -E 6M -E 8M -E EOF \ + -N -E 16M -E EOF $tf-1 || + error "create mirrored file $tf-1 failed" + $LFS mirror create -N -c 2 -o 1,3 -N -S 4M -c -1 $tf-2 || + error "create mirrored file $tf-2 failed" + + # write data in [0, 10M) + for i in $tf $tf-1 $tf-2; do + yes | dd of=$i bs=1M count=10 iflag=fullblock conv=notrunc || + error "write $i failed" + done + + # resync the mirrored files + $LFS mirror resync $tf-1 $tf-2 || + error "resync $tf-1 $tf-2 failed" + + # verify the mirrored files + $mirror_cmd $tf-1 $tf-2 || + error "verify $tf-1 $tf-2 failed" + + get_mirror_ids $tf-1 + $mirror_cmd --only ${mirror_array[0]} $tf-1 &> /dev/null && + error "at least 2 mirror ids needed with '--only' option" + $mirror_cmd --only ${mirror_array[0]},${mirror_array[1]} $tf-1 $tf-2 \ + &> /dev/null && + error "'--only' option cannot be used upon multiple files" + $mirror_cmd --only 65534,${mirror_array[0]},65535 $tf-1 && + error "invalid specified mirror ids" + + # change the content of $tf and merge it into $tf-1 + for i in 6 10; do + echo a | dd of=$tf bs=1M seek=$i conv=notrunc || + error "change $tf with seek=$i failed" + echo b | dd of=$tf-1 bs=1M seek=$i conv=notrunc || + error "change $tf-1 with seek=$i failed" + done + + $LFS mirror resync $tf-1 || error "resync $tf-1 failed" + $LFS mirror extend --no-verify -N -f $tf $tf-1 || + error "merge $tf into $tf-1 failed" + + # verify the mirrored files + echo "Verify $tf-1 without -v option:" + $mirror_cmd $tf-1 && + error "verify $tf-1 should fail" || echo "PASS" + + echo "Verify $tf-1 with -v option:" + $mirror_cmd -v $tf-1 && + error "verify $tf-1 should fail" + + get_mirror_ids $tf-1 + echo "Verify $tf-1 with --only option:" + $mirror_cmd -v --only ${mirror_array[1]},${mirror_array[-1]} $tf-1 && + error "verify $tf-1 with mirror ${mirror_array[1]} and" \ + "${mirror_array[-1]} should fail" + + $mirror_cmd --only ${mirror_array[0]},${mirror_array[1]} $tf-1 || + error "verify $tf-1 with mirror ${mirror_array[0]} and" \ + "${mirror_array[1]} should succeed" + + # set stale components in $tf-1 + for i in 0x40002 0x40003; do + $LFS setstripe --comp-set -I$i --comp-flags=stale $tf-1 || + error "set stale flag on component $i failed" + done + + # verify the mirrored file + echo "Verify $tf-1 with stale components:" + $mirror_cmd -vvv $tf-1 || + error "verify $tf-1 with stale components should succeed" + + echo "Verify $tf-1 with stale components and --only option:" + $mirror_cmd -vvv --only ${mirror_array[1]},${mirror_array[-1]} $tf-1 || + error "verify $tf-1 with mirror ${mirror_array[1]} and" \ + "${mirror_array[-1]} should succeed" +} +run_test 42 "lfs mirror verify" + ctrl_file=$(mktemp /tmp/CTRL.XXXXXX) lock_file=$(mktemp /var/lock/FLR.XXXXXX) diff --git a/lustre/utils/Makefile.am b/lustre/utils/Makefile.am index 795d49b..bdeafd9 100644 --- a/lustre/utils/Makefile.am +++ b/lustre/utils/Makefile.am @@ -60,7 +60,7 @@ lctl_LDADD := liblustreapi.la $(PTHREAD_LIBS) lctl_DEPENDENCIES := liblustreapi.la lfs_SOURCES = lfs.c lfs_project.c lfs_project.h -lfs_LDADD := liblustreapi.la +lfs_LDADD := liblustreapi.la -lz lfs_DEPENDENCIES := liblustreapi.la lustre_rsync_SOURCES = lustre_rsync.c lustre_rsync.h diff --git a/lustre/utils/lfs.c b/lustre/utils/lfs.c index 7c866b3..f1e22b8 100644 --- a/lustre/utils/lfs.c +++ b/lustre/utils/lfs.c @@ -57,10 +57,12 @@ #include #include #include +#include #include #include #include #include +#include #include "lfs_project.h" #include @@ -115,6 +117,7 @@ static int lfs_mirror(int argc, char **argv); static int lfs_mirror_list_commands(int argc, char **argv); static int lfs_list_commands(int argc, char **argv); static inline int lfs_mirror_resync(int argc, char **argv); +static inline int lfs_mirror_verify(int argc, char **argv); enum setstripe_origin { SO_SETSTRIPE, @@ -250,6 +253,11 @@ command_t mirror_cmdlist[] = { .pc_help = "Resynchronizes out-of-sync mirrored file(s).\n" "usage: lfs mirror resync [--only ] " " [...]\n"}, + { .pc_name = "verify", .pc_func = lfs_mirror_verify, + .pc_help = "Verify mirrored file(s).\n" + "usage: lfs mirror verify " + "[--only ] " + "[--verbose|-v] [ ...]\n"}, { .pc_name = "--list-commands", .pc_func = lfs_mirror_list_commands, .pc_help = "list commands supported by lfs mirror"}, { .pc_name = "help", .pc_func = Parser_help, .pc_help = "help" }, @@ -508,8 +516,8 @@ command_t cmdlist[] = { "lfs mirror create - create a mirrored file or directory\n" "lfs mirror extend - add mirror(s) to an existing file\n" "lfs mirror split - split a mirror from an existing mirrored file\n" - "lfs mirror resync - resynchronize an out-of-sync mirrored file\n" - "lfs mirror verify - verify a mirrored file\n"}, + "lfs mirror resync - resynchronize out-of-sync mirrored file(s)\n" + "lfs mirror verify - verify mirrored file(s)\n"}, {"help", Parser_help, 0, "help"}, {"exit", Parser_quit, 0, "quit"}, {"quit", Parser_quit, 0, "quit"}, @@ -6660,6 +6668,171 @@ static int parse_mirror_ids(__u16 *ids, int size, char *arg) return rc < 0 ? rc : nr; } +/** + * struct verify_mirror_id - Mirror id to be verified. + * @mirror_id: A specified mirror id. + * @is_valid_id: @mirror_id is valid or not in the mirrored file. + */ +struct verify_mirror_id { + __u16 mirror_id; + bool is_valid_id; +}; + +/** + * compare_mirror_ids() - Compare mirror ids. + * @layout: Mirror component list. + * @cbdata: Callback data in verify_mirror_id structure. + * + * This is a callback function called by llapi_layout_comp_iterate() + * to compare the specified mirror id with the one in the current + * component of @layout. If they are the same, then the specified + * mirror id is valid. + * + * Return: a negative error code on failure or + * LLAPI_LAYOUT_ITER_CONT: Proceed iteration + * LLAPI_LAYOUT_ITER_STOP: Stop iteration + */ +static inline +int compare_mirror_ids(struct llapi_layout *layout, void *cbdata) +{ + struct verify_mirror_id *mirror_id_cbdata = + (struct verify_mirror_id *)cbdata; + uint32_t mirror_id; + int rc = 0; + + rc = llapi_layout_mirror_id_get(layout, &mirror_id); + if (rc < 0) { + rc = -errno; + fprintf(stderr, + "%s: llapi_layout_mirror_id_get failed: %s.\n", + progname, strerror(errno)); + return rc; + } + + if (mirror_id_cbdata->mirror_id == mirror_id) { + mirror_id_cbdata->is_valid_id = true; + return LLAPI_LAYOUT_ITER_STOP; + } + + return LLAPI_LAYOUT_ITER_CONT; +} + +/** + * verify_mirror_ids() - Verify specified mirror ids. + * @fname: Mirrored file name. + * @mirror_ids: Specified mirror ids to be verified. + * @ids_nr: Number of specified mirror ids. + * + * This function verifies that specified @mirror_ids are valid + * in the mirrored file @fname. + * + * Return: 0 on success or a negative error code on failure. + */ +static inline +int verify_mirror_ids(const char *fname, __u16 *mirror_ids, int ids_nr) +{ + struct llapi_layout *layout = NULL; + struct verify_mirror_id mirror_id_cbdata = { 0 }; + struct stat stbuf; + uint32_t flr_state; + int i; + int fd; + int rc = 0; + int rc2 = 0; + + if (ids_nr <= 0) + return -EINVAL; + + if (stat(fname, &stbuf) < 0) { + fprintf(stderr, "%s: cannot stat file '%s': %s.\n", + progname, fname, strerror(errno)); + rc = -errno; + goto error; + } + + if (!S_ISREG(stbuf.st_mode)) { + fprintf(stderr, "%s: '%s' is not a regular file.\n", + progname, fname); + rc = -EINVAL; + goto error; + } + + fd = open(fname, O_DIRECT | O_RDONLY); + if (fd < 0) { + fprintf(stderr, "%s: cannot open '%s': %s.\n", + progname, fname, strerror(errno)); + rc = -errno; + goto error; + } + + rc = llapi_lease_get(fd, LL_LEASE_RDLCK); + if (rc < 0) { + fprintf(stderr, "%s: '%s' llapi_lease_get failed: %s.\n", + progname, fname, strerror(errno)); + goto close_fd; + } + + layout = llapi_layout_get_by_fd(fd, 0); + if (layout == NULL) { + fprintf(stderr, "%s: '%s' llapi_layout_get_by_fd failed: %s.\n", + progname, fname, strerror(errno)); + rc = -errno; + llapi_lease_put(fd); + goto close_fd; + } + + rc = llapi_layout_flags_get(layout, &flr_state); + if (rc < 0) { + fprintf(stderr, "%s: '%s' llapi_layout_flags_get failed: %s.\n", + progname, fname, strerror(errno)); + rc = -errno; + goto free_layout; + } + + flr_state &= LCM_FL_FLR_MASK; + switch (flr_state) { + case LCM_FL_NOT_FLR: + rc = -EINVAL; + fprintf(stderr, "%s: '%s' file state error: %s.\n", + progname, fname, lcm_flags_string(flr_state)); + goto free_layout; + default: + break; + } + + rc2 = 0; + for (i = 0; i < ids_nr; i++) { + mirror_id_cbdata.mirror_id = mirror_ids[i]; + mirror_id_cbdata.is_valid_id = false; + + rc = llapi_layout_comp_iterate(layout, compare_mirror_ids, + &mirror_id_cbdata); + if (rc < 0) { + rc = -errno; + fprintf(stderr, + "%s: '%s' failed to verify mirror id: %u.\n", + progname, fname, mirror_ids[i]); + goto free_layout; + } + + if (!mirror_id_cbdata.is_valid_id) { + rc2 = -EINVAL; + fprintf(stderr, + "%s: '%s' invalid specified mirror id: %u.\n", + progname, fname, mirror_ids[i]); + } + } + rc = rc2; + +free_layout: + llapi_layout_free(layout); + llapi_lease_put(fd); +close_fd: + close(fd); +error: + return rc; +} + static inline int lfs_mirror_resync_file(const char *fname, struct ll_ioc_lease *ioc, __u16 *mirror_ids, int ids_nr) @@ -6866,6 +7039,12 @@ static inline int lfs_mirror_resync(int argc, char **argv) } + if (ids_nr > 0) { + rc = verify_mirror_ids(argv[optind], mirror_ids, ids_nr); + if (rc < 0) + goto error; + } + /* set the lease on the file */ ioc = calloc(sizeof(*ioc) + sizeof(__u32) * 4096, 1); if (ioc == NULL) { @@ -6893,6 +7072,641 @@ error: } /** + * struct verify_chunk - Mirror chunk to be verified. + * @chunk: [start, end) of the chunk. + * @mirror_count: Number of mirror ids in @mirror_id array. + * @mirror_id: Array of valid mirror ids that cover the chunk. + */ +struct verify_chunk { + struct lu_extent chunk; + unsigned int mirror_count; + __u16 mirror_id[LUSTRE_MIRROR_COUNT_MAX]; +}; + +/** + * print_chunks() - Print chunk information. + * @fname: Mirrored file name. + * @chunks: Array of chunks. + * @chunk_count: Number of chunks in @chunks array. + * + * This function prints [start, end) of each chunk in @chunks + * for mirrored file @fname, and also prints the valid mirror ids + * that cover the chunk. + * + * Return: void. + */ +static inline +void print_chunks(const char *fname, struct verify_chunk *chunks, + int chunk_count) +{ + int i; + int j; + + fprintf(stdout, "Chunks to be verified in %s:\n", fname); + for (i = 0; i < chunk_count; i++) { + fprintf(stdout, DEXT, PEXT(&chunks[i].chunk)); + + if (chunks[i].mirror_count == 0) + fprintf(stdout, "\t["); + else { + fprintf(stdout, "\t[%u", chunks[i].mirror_id[0]); + for (j = 1; j < chunks[i].mirror_count; j++) + fprintf(stdout, ", %u", chunks[i].mirror_id[j]); + } + fprintf(stdout, "]\t%u\n", chunks[i].mirror_count); + } + fprintf(stdout, "\n"); +} + +/** + * print_checksums() - Print CRC-32 checksum values. + * @chunk: A chunk and its corresponding valid mirror ids. + * @crc: CRC-32 checksum values on the chunk for each valid mirror. + * + * This function prints CRC-32 checksum values on @chunk for + * each valid mirror that covers it. + * + * Return: void. + */ +static inline +void print_checksums(struct verify_chunk *chunk, unsigned long *crc) +{ + int i; + + fprintf(stdout, + "CRC-32 checksum value for chunk "DEXT":\n", + PEXT(&chunk->chunk)); + for (i = 0; i < chunk->mirror_count; i++) + fprintf(stdout, "Mirror %u:\t%#lx\n", + chunk->mirror_id[i], crc[i]); + fprintf(stdout, "\n"); +} + +/** + * filter_mirror_id() - Filter specified mirror ids. + * @chunks: Array of chunks. + * @chunk_count: Number of chunks in @chunks array. + * @mirror_ids: Specified mirror ids to be verified. + * @ids_nr: Number of specified mirror ids. + * + * This function scans valid mirror ids that cover each chunk in @chunks + * and filters specified mirror ids. + * + * Return: void. + */ +static inline +void filter_mirror_id(struct verify_chunk *chunks, int chunk_count, + __u16 *mirror_ids, int ids_nr) +{ + int i; + int j; + int k; + __u16 valid_id[LUSTRE_MIRROR_COUNT_MAX] = { 0 }; + unsigned int valid_count = 0; + + for (i = 0; i < chunk_count; i++) { + if (chunks[i].mirror_count == 0) + continue; + + valid_count = 0; + for (j = 0; j < ids_nr; j++) { + for (k = 0; k < chunks[i].mirror_count; k++) { + if (chunks[i].mirror_id[k] == mirror_ids[j]) { + valid_id[valid_count] = mirror_ids[j]; + valid_count++; + break; + } + } + } + + memcpy(chunks[i].mirror_id, valid_id, + sizeof(__u16) * valid_count); + chunks[i].mirror_count = valid_count; + } +} + +/** + * lfs_mirror_prepare_chunk() - Find mirror chunks to be verified. + * @layout: Mirror component list. + * @chunks: Array of chunks. + * @chunks_size: Array size of @chunks. + * + * This function scans the components in @layout from offset 0 to LUSTRE_EOF + * to find out chunk segments and store them in @chunks array. + * + * The @mirror_id array in each element of @chunks will store the valid + * mirror ids that cover the chunk. If a mirror component covering the + * chunk has LCME_FL_STALE or LCME_FL_OFFLINE flag, then the mirror id + * will not be stored into the @mirror_id array, and the chunk for that + * mirror will not be verified. + * + * The @mirror_count in each element of @chunks will store the number of + * mirror ids in @mirror_id array. If @mirror_count is 0, it indicates the + * chunk is invalid in all of the mirrors. And if @mirror_count is 1, it + * indicates the chunk is valid in only one mirror. In both cases, the + * chunk will not be verified. + * + * Here is an example: + * + * 0 1M 2M 3M 4M EOF + * +------+-------------+--------------------+ + * | | | S | mirror1 + * +------+------+------+------+-------------+ + * | | S | S | | mirror2 + * +-------------+------+------+-------------+ + * + * prepared @chunks array will contain 5 elements: + * (([0, 1M), [1, 2], 2), + * ([1M, 2M), [1, 2], 2), + * ([2M, 3M), [1], 1), + * ([3M, 4M], [], 0), + * ([4M, EOF), [2], 1)) + * + * Return: the actual array size of @chunks on success + * or a negative error code on failure. + */ +static inline +int lfs_mirror_prepare_chunk(struct llapi_layout *layout, + struct verify_chunk *chunks, + size_t chunks_size) +{ + uint64_t start; + uint64_t end; + uint32_t mirror_id; + uint32_t flags; + int idx = 0; + int i = 0; + int rc = 0; + + memset(chunks, 0, sizeof(*chunks) * chunks_size); + + while (1) { + rc = llapi_layout_comp_use(layout, LLAPI_LAYOUT_COMP_USE_FIRST); + if (rc < 0) { + fprintf(stderr, + "%s: move to the first layout component: %s.\n", + progname, strerror(errno)); + goto error; + } + + i = 0; + rc = 0; + chunks[idx].chunk.e_end = LUSTRE_EOF; + while (rc == 0) { + rc = llapi_layout_comp_extent_get(layout, &start, &end); + if (rc < 0) { + fprintf(stderr, + "%s: llapi_layout_comp_extent_get failed: %s.\n", + progname, strerror(errno)); + goto error; + } + + if (start > chunks[idx].chunk.e_start || + end <= chunks[idx].chunk.e_start) + goto next; + + if (end < chunks[idx].chunk.e_end) + chunks[idx].chunk.e_end = end; + + rc = llapi_layout_comp_flags_get(layout, &flags); + if (rc < 0) { + fprintf(stderr, + "%s: llapi_layout_comp_flags_get failed: %s.\n", + progname, strerror(errno)); + goto error; + } + + if (flags & LCME_FL_STALE || flags & LCME_FL_OFFLINE) + goto next; + + rc = llapi_layout_mirror_id_get(layout, &mirror_id); + if (rc < 0) { + fprintf(stderr, + "%s: llapi_layout_mirror_id_get failed: %s.\n", + progname, strerror(errno)); + goto error; + } + + chunks[idx].mirror_id[i] = mirror_id; + i++; + if (i >= ARRAY_SIZE(chunks[idx].mirror_id)) { + fprintf(stderr, + "%s: mirror_id array is too small.\n", + progname); + rc = -EINVAL; + goto error; + } + + next: + rc = llapi_layout_comp_use(layout, + LLAPI_LAYOUT_COMP_USE_NEXT); + if (rc < 0) { + fprintf(stderr, + "%s: move to the next layout component: %s.\n", + progname, strerror(errno)); + goto error; + } + } /* loop through all components */ + + chunks[idx].mirror_count = i; + + if (chunks[idx].chunk.e_end == LUSTRE_EOF) + break; + + idx++; + if (idx >= chunks_size) { + fprintf(stderr, "%s: chunks array is too small.\n", + progname); + rc = -EINVAL; + goto error; + } + + chunks[idx].chunk.e_start = chunks[idx - 1].chunk.e_end; + } + +error: + return rc < 0 ? rc : idx + 1; +} + +/** + * lfs_mirror_verify_chunk() - Verify a chunk. + * @fd: File descriptor of the mirrored file. + * @file_size: Size of the mirrored file. + * @chunk: A chunk and its corresponding valid mirror ids. + * @verbose: Verbose mode. + * + * This function verifies a @chunk contains exactly the same data + * ammong the mirrors that cover it. + * + * If @verbose is specified, then the function will print where the + * differences are if the data do not match. Otherwise, it will + * just return an error in that case. + * + * Return: 0 on success or a negative error code on failure. + */ +static inline +int lfs_mirror_verify_chunk(int fd, size_t file_size, + struct verify_chunk *chunk, int verbose) +{ + const size_t buflen = 4 * 1024 * 1024; /* 4M */ + void *buf; + size_t page_size = sysconf(_SC_PAGESIZE); + ssize_t bytes_read; + ssize_t bytes_done; + size_t count; + off_t pos; + unsigned long crc; + unsigned long crc_array[LUSTRE_MIRROR_COUNT_MAX] = { 0 }; + int i; + int rc = 0; + + if (file_size == 0) + return 0; + + rc = posix_memalign(&buf, page_size, buflen); + if (rc) /* error code is returned directly */ + return -rc; + + if (verbose > 1) { + fprintf(stdout, "Verifying chunk "DEXT" on mirror:", + PEXT(&chunk->chunk)); + for (i = 0; i < chunk->mirror_count; i++) + fprintf(stdout, " %u", chunk->mirror_id[i]); + fprintf(stdout, "\n"); + } + + bytes_done = 0; + count = MIN(chunk->chunk.e_end, file_size) - chunk->chunk.e_start; + pos = chunk->chunk.e_start; + while (bytes_done < count) { + /* compute initial CRC-32 checksum */ + crc = crc32(0L, Z_NULL, 0); + memset(crc_array, 0, sizeof(crc_array)); + + bytes_read = 0; + for (i = 0; i < chunk->mirror_count; i++) { + bytes_read = llapi_mirror_read(fd, chunk->mirror_id[i], + buf, buflen, pos); + if (bytes_read < 0) { + rc = bytes_read; + fprintf(stderr, + "%s: failed to read data from mirror %u: %s.\n", + progname, chunk->mirror_id[i], + strerror(-rc)); + goto error; + } + + /* compute new CRC-32 checksum */ + crc_array[i] = crc32(crc, buf, bytes_read); + } + + if (verbose) + print_checksums(chunk, crc_array); + + /* compare CRC-32 checksum values */ + for (i = 1; i < chunk->mirror_count; i++) { + if (crc_array[i] != crc_array[0]) { + rc = -EINVAL; + if (!verbose) + goto error; + + fprintf(stderr, + "%s: chunk "DEXT" has different checksum value on mirror %u and mirror %u.\n", + progname, PEXT(&chunk->chunk), + chunk->mirror_id[0], + chunk->mirror_id[i]); + } + } + + pos += bytes_read; + bytes_done += bytes_read; + } + + if (verbose > 1 && rc == 0) { + fprintf(stdout, "Verifying chunk "DEXT" on mirror:", + PEXT(&chunk->chunk)); + for (i = 0; i < chunk->mirror_count; i++) + fprintf(stdout, " %u", chunk->mirror_id[i]); + fprintf(stdout, " PASS\n\n"); + } + +error: + free(buf); + return rc; +} + +/** + * lfs_mirror_verify_file() - Verify a mirrored file. + * @fname: Mirrored file name. + * @mirror_ids: Specified mirror ids to be verified. + * @ids_nr: Number of specified mirror ids. + * @verbose: Verbose mode. + * + * This function verifies that each SYNC mirror of a mirrored file + * specified by @fname contains exactly the same data. + * + * If @mirror_ids is specified, then the function will verify the + * mirrors specified by @mirror_ids contain exactly the same data. + * + * If @verbose is specified, then the function will print where the + * differences are if the data do not match. Otherwise, it will + * just return an error in that case. + * + * Return: 0 on success or a negative error code on failure. + */ +static inline +int lfs_mirror_verify_file(const char *fname, __u16 *mirror_ids, int ids_nr, + int verbose) +{ + struct verify_chunk chunks_array[1024] = { }; + struct llapi_layout *layout = NULL; + struct stat stbuf; + uint32_t flr_state; + int fd; + int chunk_count = 0; + int idx = 0; + int rc = 0; + int rc1 = 0; + int rc2 = 0; + + if (stat(fname, &stbuf) < 0) { + fprintf(stderr, "%s: cannot stat file '%s': %s.\n", + progname, fname, strerror(errno)); + rc = -errno; + goto error; + } + + if (!S_ISREG(stbuf.st_mode)) { + fprintf(stderr, "%s: '%s' is not a regular file.\n", + progname, fname); + rc = -EINVAL; + goto error; + } + + if (stbuf.st_size == 0) { + if (verbose) + fprintf(stdout, "%s: '%s' file size is 0.\n", + progname, fname); + rc = 0; + goto error; + } + + fd = open(fname, O_DIRECT | O_RDONLY); + if (fd < 0) { + fprintf(stderr, "%s: cannot open '%s': %s.\n", + progname, fname, strerror(errno)); + rc = -errno; + goto error; + } + + rc = llapi_lease_get(fd, LL_LEASE_RDLCK); + if (rc < 0) { + fprintf(stderr, "%s: '%s' llapi_lease_get failed: %s.\n", + progname, fname, strerror(errno)); + goto close_fd; + } + + layout = llapi_layout_get_by_fd(fd, 0); + if (layout == NULL) { + fprintf(stderr, "%s: '%s' llapi_layout_get_by_fd failed: %s.\n", + progname, fname, strerror(errno)); + rc = -errno; + llapi_lease_put(fd); + goto close_fd; + } + + rc = llapi_layout_flags_get(layout, &flr_state); + if (rc < 0) { + fprintf(stderr, "%s: '%s' llapi_layout_flags_get failed: %s.\n", + progname, fname, strerror(errno)); + rc = -errno; + goto free_layout; + } + + flr_state &= LCM_FL_FLR_MASK; + switch (flr_state) { + case LCM_FL_NOT_FLR: + rc = -EINVAL; + fprintf(stderr, "%s: '%s' file state error: %s.\n", + progname, fname, lcm_flags_string(flr_state)); + goto free_layout; + default: + break; + } + + /* find out mirror chunks to be verified */ + chunk_count = lfs_mirror_prepare_chunk(layout, chunks_array, + ARRAY_SIZE(chunks_array)); + if (chunk_count < 0) { + rc = chunk_count; + goto free_layout; + } + + if (ids_nr > 0) + /* filter specified mirror ids */ + filter_mirror_id(chunks_array, chunk_count, mirror_ids, ids_nr); + + if (verbose > 2) + print_chunks(fname, chunks_array, chunk_count); + + for (idx = 0; idx < chunk_count; idx++) { + if (chunks_array[idx].chunk.e_start >= stbuf.st_size) { + if (verbose) + fprintf(stdout, + "%s: '%s' chunk "DEXT" exceeds file size %#llx: skipped\n", + progname, fname, + PEXT(&chunks_array[idx].chunk), + (unsigned long long)stbuf.st_size); + break; + } + + if (chunks_array[idx].mirror_count == 0) { + fprintf(stderr, + "%s: '%s' chunk "DEXT" is invalid in all of the mirrors: ", + progname, fname, + PEXT(&chunks_array[idx].chunk)); + if (verbose) { + fprintf(stderr, "skipped\n"); + continue; + } + rc = -EINVAL; + fprintf(stderr, "failed\n"); + goto free_layout; + } + + if (chunks_array[idx].mirror_count == 1) { + if (verbose) + fprintf(stdout, + "%s: '%s' chunk "DEXT" is only valid in mirror %u: skipped\n", + progname, fname, + PEXT(&chunks_array[idx].chunk), + chunks_array[idx].mirror_id[0]); + continue; + } + + rc = llapi_lease_check(fd); + if (rc != LL_LEASE_RDLCK) { + fprintf(stderr, "%s: '%s' lost lease lock.\n", + progname, fname); + goto free_layout; + } + + /* verify one chunk */ + rc1 = lfs_mirror_verify_chunk(fd, stbuf.st_size, + &chunks_array[idx], verbose); + if (rc1 < 0) { + rc2 = rc1; + if (!verbose) { + rc = rc1; + goto free_layout; + } + } + } + + if (rc2 < 0) + rc = rc2; + +free_layout: + llapi_layout_free(layout); + llapi_lease_put(fd); +close_fd: + close(fd); +error: + return rc; +} + +/** + * lfs_mirror_verify() - Parse and execute lfs mirror verify command. + * @argc: The count of lfs mirror verify command line arguments. + * @argv: Array of strings for lfs mirror verify command line arguments. + * + * This function parses lfs mirror verify command and verifies the + * specified mirrored file(s). + * + * Return: 0 on success or a negative error code on failure. + */ +static inline int lfs_mirror_verify(int argc, char **argv) +{ + __u16 mirror_ids[LUSTRE_MIRROR_COUNT_MAX] = { 0 }; + int ids_nr = 0; + int c; + int verbose = 0; + int rc = 0; + int rc1 = 0; + char cmd[PATH_MAX]; + + struct option long_opts[] = { + { .val = 'o', .name = "only", .has_arg = required_argument }, + { .val = 'v', .name = "verbose", .has_arg = no_argument }, + { .name = NULL } }; + + snprintf(cmd, sizeof(cmd), "%s %s", progname, argv[0]); + progname = cmd; + while ((c = getopt_long(argc, argv, "o:v", long_opts, NULL)) >= 0) { + switch (c) { + case 'o': + rc = parse_mirror_ids(mirror_ids, + ARRAY_SIZE(mirror_ids), + optarg); + if (rc < 0) { + fprintf(stderr, + "%s: bad mirror ids '%s'.\n", + progname, optarg); + goto error; + } + ids_nr = rc; + if (ids_nr < 2) { + fprintf(stderr, + "%s: at least 2 mirror ids needed with '--only' option.\n", + progname); + rc = CMD_HELP; + goto error; + } + break; + case 'v': + verbose++; + break; + default: + fprintf(stderr, "%s: options '%s' unrecognized.\n", + progname, argv[optind - 1]); + rc = -EINVAL; + goto error; + } + } + + if (argc == optind) { + fprintf(stderr, "%s: no file name given.\n", progname); + rc = CMD_HELP; + goto error; + } + + if (ids_nr > 0 && argc > optind + 1) { + fprintf(stderr, + "%s: '--only' cannot be used upon multiple files.\n", + progname); + rc = CMD_HELP; + goto error; + + } + + if (ids_nr > 0) { + rc = verify_mirror_ids(argv[optind], mirror_ids, ids_nr); + if (rc < 0) + goto error; + } + + rc = 0; + for (; optind < argc; optind++) { + rc1 = lfs_mirror_verify_file(argv[optind], mirror_ids, ids_nr, + verbose); + if (rc1 < 0) + rc = rc1; + } +error: + return rc; +} + +/** * lfs_mirror() - Parse and execute lfs mirror commands. * @argc: The count of lfs mirror command line arguments. * @argv: Array of strings for lfs mirror command line arguments. -- 1.8.3.1