From 90ba8b4ac360b1987178445bd2ccd64f7958d912 Mon Sep 17 00:00:00 2001 From: Bobi Jam Date: Wed, 28 Apr 2021 13:26:18 +0800 Subject: [PATCH] LU-14642 test: add fsx mirror file test mode - add fsx mirror file test mode with "-M" option so that fsx can exert its IO to FLR file as well as extend/split/resync the FLR file. - add sanity-flr test_70b() to test fsx with flrmode. - fix a bug in "lfs mirror verify" to accomodate max mirror count instead of (max - 1) mirrors. - improve "lfs mirror verify -v" print proper data range of its crc-32 checksum values. Test-Parameters: trivial testlist=sanity-flr env=ONLY=100,FSXNUM=10000 Signed-off-by: Bobi Jam Change-Id: Ib55c7b25dcd82fa0b197ad21268b16c82aab5da9 Reviewed-on: https://review.whamcloud.com/43473 Tested-by: jenkins Tested-by: Maloo Reviewed-by: Andreas Dilger Reviewed-by: John L. Hammond Reviewed-by: Oleg Drokin --- lustre/tests/Makefile.am | 1 + lustre/tests/fsx.c | 222 ++++++++++++++++++++++++++++++++++++++++++++- lustre/tests/sanity-flr.sh | 14 +++ lustre/utils/lfs.c | 12 +-- 4 files changed, 240 insertions(+), 9 deletions(-) diff --git a/lustre/tests/Makefile.am b/lustre/tests/Makefile.am index 0e6f42e..2f9418d 100644 --- a/lustre/tests/Makefile.am +++ b/lustre/tests/Makefile.am @@ -133,6 +133,7 @@ lov_getstripe_old_LDADD = $(LIBLUSTREAPI) flocks_test_LDADD = $(LIBLUSTREAPI) $(PTHREAD_LIBS) create_foreign_dir_LDADD = $(LIBLUSTREAPI) check_fallocate_LDADD = $(LIBLUSTREAPI) +fsx_LDADD = $(LIBLUSTREAPI) if LIBAIO aiocp_LDADD= -laio endif diff --git a/lustre/tests/fsx.c b/lustre/tests/fsx.c index 5217a906..5234ffc 100644 --- a/lustre/tests/fsx.c +++ b/lustre/tests/fsx.c @@ -75,6 +75,9 @@ #include #include +#include +#include + /* * Each test run will work with one or more separate file descriptors for the * same file. This allows testing cache coherency across multiple mountpoints @@ -116,6 +119,7 @@ int logcount; /* total ops */ int jmpbuf_good; jmp_buf jmpbuf; +unsigned int mirror_ids[LUSTRE_MIRROR_COUNT_MAX]; /* * Define operations */ @@ -133,7 +137,19 @@ jmp_buf jmpbuf; #define OP_PUNCH_HOLE 6 #define OP_ZERO_RANGE 7 #define OP_CLOSEOPEN 8 -#define OP_MAX_FULL 9 +#define OP_MIRROR_OPS 9 +#define OP_MAX_FULL 10 + +#define MIRROR_EXTEND 0 +#define MIRROR_SPLIT 1 +#define MIRROR_RESYNC 2 +#define MIRROR_OPS 3 + +char *mirror_op_str[] = { + [MIRROR_EXTEND] = "MIRROR_EXTEND", + [MIRROR_SPLIT] = "MIRROR_SPLIT", + [MIRROR_RESYNC] = "MIRROR_RESYNC", +}; #define OP_SKIPPED 101 #define OP_DIRECT O_DIRECT @@ -181,6 +197,7 @@ int truncbdy = 1; /* -t flag */ int writebdy = 1; /* -w flag */ long monitorstart = -1; /* -m flag */ long monitorend = -1; /* -m flag */ +long flrmode; /* -M flag */ int lite; /* -L flag */ long numops = -1; /* -N flag */ int randomoplen = 1; /* -O flag disables it */ @@ -394,6 +411,17 @@ logdump(void) prt("CLOSE/OPEN%s", lp->operation & OP_DIRECT ? "_OD" : " "); break; + case OP_MIRROR_OPS: { + prt("%s ", mirror_op_str[lp->args[0]]); + if (lp->args[0] == MIRROR_EXTEND) + prt("to %d mirrors", lp->args[1] + 1); + else if (lp->args[0] == MIRROR_SPLIT) + prt("mirror %d to %d mirrors", lp->args[2], + lp->args[1] - 1); + else if (lp->args[0] == MIRROR_RESYNC) + prt("%d mirrors", lp->args[1]); + break; + } case OP_SKIPPED: prt("SKIPPED (no operation)"); break; @@ -453,6 +481,7 @@ void report_failure(int status) { logdump(); + prt("Using seed %d\n", seed); if (fsxgoodfd) { if (good_buf) { @@ -694,6 +723,30 @@ output_line(struct test_file *tf, int op, unsigned int offset, (int)size < 0 ? -(int)size : size); } +void +mirror_output_line(struct test_file *tf, int op, int mirrors, int id) +{ + if (!(!quiet && + ((progressinterval && testcalls % progressinterval == 0) || + (debug && (monitorstart == -1))))) + return; + + prt("%06lu %lu.%06u %-10s ", + testcalls, tv.tv_sec, (int)tv.tv_usec, mirror_op_str[op]); + + switch (op) { + case MIRROR_EXTEND: + prt("to %d mirrors\n", mirrors + 1); + break; + case MIRROR_SPLIT: + prt("mirror %d to %d mirrors\n", id, mirrors - 1); + break; + case MIRROR_RESYNC: + prt("%d mirrors\n", mirrors); + break; + } +} + void output_debug(unsigned int offset, unsigned int size, const char *what) { struct timeval t; @@ -1275,6 +1328,162 @@ docloseopen(void) tf->o_direct ? "open(O_DIRECT) done" : "open done"); } +static int +get_mirror_ids(int fd, unsigned int *ids) +{ + struct llapi_layout *layout; + uint16_t count; + int rc; + + layout = llapi_layout_get_by_fd(fd, 0); + if (layout == NULL) + return 0; + + /* only get mirror count */ + rc = llapi_layout_mirror_count_get(layout, &count); + if (rc < 0) + prt("llapi_layout_mirror_count_get: %d\n", rc); + if (count == 0) + return 0; + + count = 0; + + rc = llapi_layout_comp_use(layout, LLAPI_LAYOUT_COMP_USE_FIRST); + if (rc < 0) { + prt("llapi_layout_comp_use(USE_FIRST): %d\n", rc); + goto free; + } + + do { + unsigned int id; + + rc = llapi_layout_mirror_id_get(layout, &id); + if (rc < 0) { + prt("llapi_layout_mirror_id_get: %d\n", rc); + goto free; + } + + if (!count || ids[count - 1] != id) + ids[count++] = id; + + rc = llapi_layout_comp_use(layout, LLAPI_LAYOUT_COMP_USE_NEXT); + if (rc < 0) { + prt("llapi_layout_comp_use(USE_NEXT): %d\n", rc); + goto free; + } + } while (rc == 0); + +free: + llapi_layout_free(layout); + + return rc < 0 ? rc : count; +} + +void +do_mirror_ops(int op) +{ + int mirror_count; + char cmd[PATH_MAX * 2]; + int i = 0; + int rc; + + if (testcalls <= simulatedopcount) + return; + + tf = get_tf(); + + mirror_count = get_mirror_ids(tf->fd, mirror_ids); + if (mirror_count < 0) { + prterr("get_mirror_ids"); + report_failure(182); + } + + switch (op) { + case MIRROR_EXTEND: + if (mirror_count == LUSTRE_MIRROR_COUNT_MAX) + return; + snprintf(cmd, sizeof(cmd), "lfs mirror extend -N -c-1 %s", + tf->path); + break; + case MIRROR_SPLIT: + if (mirror_count == 0 || mirror_count == 1) + return; + + i = random() % mirror_count; + if (i == 0) + i++; + + snprintf(cmd, sizeof(cmd), + "lfs mirror split -d --mirror-id=%d %s", + mirror_ids[i], tf->path); + break; + case MIRROR_RESYNC: + if (mirror_count < 2) + return; + + snprintf(cmd, sizeof(cmd), + "lfs mirror resync %s", tf->path); + break; + } + + if (close(tf->fd)) + report_failure(183); + output_debug(monitorstart, 0, "close done"); + + log4(OP_MIRROR_OPS, op, mirror_count, i); + + mirror_output_line(tf, op, mirror_count, i); + + rc = system(cmd); + if (rc < 0) { + prt("%s: %d\n", cmd, errno); + report_failure(184); + } else if (WIFEXITED(rc)) { + rc = WEXITSTATUS(rc); + if (rc > 0) { + prt("%s: %d\n", cmd, rc); + report_failure(184); + } + } + output_debug(monitorstart, 0, cmd); + + switch (op) { + case MIRROR_SPLIT: + if (mirror_count == 2) + break; + case MIRROR_EXTEND: + case MIRROR_RESYNC: + /* verify mirror */ + snprintf(cmd, sizeof(cmd), + "lfs mirror verify %s", tf->path); + + rc = system(cmd); + if (rc < 0) { + prt("%s: %d\n", cmd, errno); + report_failure(184); + } else if (WIFEXITED(rc)) { + rc = WEXITSTATUS(rc); + if (rc > 0) { + prt("%s: %d\n", cmd, rc); + snprintf(cmd, sizeof(cmd), + "lfs mirror verify -v %s", tf->path); + rc = system(cmd); + report_failure(184); + } + } + } + + output_debug(monitorstart, 0, cmd); + + tf->fd = open(tf->path, O_RDWR | tf->o_direct, 0); + if (tf->fd < 0) { + prterr(tf->o_direct ? "open(O_DIRECT)" : "open"); + report_failure(185); + } + output_debug(monitorstart, 0, + tf->o_direct ? "open(O_DIRECT) done" : "open done"); +} + #define TRIM_OFF_LEN(off, len, size) \ do { \ if (size) \ @@ -1385,6 +1594,10 @@ test(void) if (closeopen) docloseopen(); break; + case OP_MIRROR_OPS: + if (flrmode) + do_mirror_ops(random() % MIRROR_OPS); + break; default: prterr("unknown operation %d: Operation not supported"); report_failure(42); @@ -1462,7 +1675,7 @@ usage(void) " order with 'rotate' or chose them at 'random'. (default random)\n" " -L: fsxLite - no file creations & no file size changes\n" /* OSX: -I: start interactive mode since operation opnum\n\ */ -/* OSX: -M: slow motion mode, wait 1 second before each op\n\ */ +" -M: mirror file test mode\n" " -N numops: total # operations to do (default infinity)\n" " -O: use oplen (see -o flag) for every op (default random)\n" " -P: save .fsxlog and .fsxgood files in dirpath (default ./)\n" @@ -1550,7 +1763,7 @@ main(int argc, char **argv) setvbuf(stdout, (char *)0, _IOLBF, 0); /* line buffered stdout */ while ((ch = getopt(argc, argv, - "b:c:dfl:m:no:p:qr:s:t:w:xyzD:FHI:LN:OP:RS:WZ::")) + "b:c:dfl:m:no:p:qr:s:t:w:xyzD:FHI:LMN:OP:RS:WZ::")) != EOF) switch (ch) { case 'b': @@ -1653,6 +1866,9 @@ main(int argc, char **argv) case 'L': lite = 1; break; + case 'M': + flrmode = 1; + break; case 'N': numops = getnum(optarg, &endp); if (numops < 0) diff --git a/lustre/tests/sanity-flr.sh b/lustre/tests/sanity-flr.sh index d4e2e99..4d14a8f 100644 --- a/lustre/tests/sanity-flr.sh +++ b/lustre/tests/sanity-flr.sh @@ -3021,6 +3021,20 @@ test_70() { } run_test 70 "mirror create and split race" +test_100() { + local tf=$DIR/$tdir/$tfile + + + test_mkdir $DIR/$tdir + + $LFS setstripe -N -E1M -c-1 -Eeof -c-1 $tf || + error "setstripe $tf failed" + + FSXNUM=${FSXNUM:-1000} + $FSX -p 5 -N $FSXNUM -S 0 -M $tf || error "fsx FLR file $tf failed" +} +run_test 100 "flr mode fsx test" + ctrl_file=$(mktemp /tmp/CTRL.XXXXXX) lock_file=$(mktemp /var/lock/FLR.XXXXXX) diff --git a/lustre/utils/lfs.c b/lustre/utils/lfs.c index 94d2696..e05f918 100644 --- a/lustre/utils/lfs.c +++ b/lustre/utils/lfs.c @@ -11623,13 +11623,13 @@ void print_chunks(const char *fname, struct verify_chunk *chunks, * Return: void. */ static inline -void print_checksums(struct verify_chunk *chunk, unsigned long *crc) +void print_checksums(struct verify_chunk *chunk, unsigned long *crc, + unsigned long long pos, unsigned long long len) { int i; fprintf(stdout, - "CRC-32 checksum value for chunk "DEXT":\n", - PEXT(&chunk->chunk)); + "CRC-32 checksum value for chunk "DEXT":\n", pos, pos + len); for (i = 0; i < chunk->mirror_count; i++) fprintf(stdout, "Mirror %u:\t%#lx\n", chunk->mirror_id[i], crc[i]); @@ -11781,8 +11781,6 @@ int lfs_mirror_prepare_chunk(struct llapi_layout *layout, goto error; } - chunks[idx].mirror_id[i] = mirror_id; - i++; if (i >= ARRAY_SIZE(chunks[idx].mirror_id)) { fprintf(stderr, "%s: mirror_id array is too small.\n", @@ -11790,6 +11788,8 @@ int lfs_mirror_prepare_chunk(struct llapi_layout *layout, rc = -EINVAL; goto error; } + chunks[idx].mirror_id[i] = mirror_id; + i++; next: rc = llapi_layout_comp_use(layout, @@ -11895,7 +11895,7 @@ int lfs_mirror_verify_chunk(int fd, size_t file_size, } if (verbose) - print_checksums(chunk, crc_array); + print_checksums(chunk, crc_array, pos, buflen); /* compare CRC-32 checksum values */ for (i = 1; i < chunk->mirror_count; i++) { -- 1.8.3.1