From: Bobi Jam Date: Sat, 14 Oct 2017 07:49:27 +0000 (-0700) Subject: LU-9771 flr: lfs mirror resync command X-Git-Tag: 2.10.56~9^2^2~6 X-Git-Url: https://git.whamcloud.com/?a=commitdiff_plain;h=79da3738df220b2ea055098aaa3aaba59934616e;p=fs%2Flustre-release.git LU-9771 flr: lfs mirror resync command This patch adds "lfs mirror resync" command to resynchronize a mirrored file. Usage: lfs mirror resync [--only ] Options: --only Indicates which mirror(s) specified by mirror_id(s) needs to be resynchronized. The mirror_id is the numerical unique identifier for a mirror. Multiple mirror_ids are separated by comma. Test-Parameters: testlist=sanity-flr Signed-off-by: Bobi Jam Change-Id: Ic2aa474d1023a4364aa9d90c59120b1d5c89a269 Signed-off-by: Jinshan Xiong Reviewed-on: https://review.whamcloud.com/29243 Tested-by: Jenkins Reviewed-by: Bobi Jam Reviewed-by: Dmitry Eremin Tested-by: Maloo Reviewed-by: Andreas Dilger --- diff --git a/lustre/include/lustre/lustreapi.h b/lustre/include/lustre/lustreapi.h index 62089af..f711ac0 100644 --- a/lustre/include/lustre/lustreapi.h +++ b/lustre/include/lustre/lustreapi.h @@ -450,9 +450,28 @@ int llapi_ladvise(int fd, unsigned long long flags, int num_advise, /* llapi_layout user interface */ +extern char *lcm_flags_string(__u16 flags); + +/** + * An array element storing component info to be resynced during mirror + * resynchronization. + */ +struct llapi_resync_comp { + uint64_t lrc_start; + uint64_t lrc_end; + uint32_t lrc_mirror_id; + uint32_t lrc_id; /* component id */ + bool lrc_synced; +}; + /** Opaque data type abstracting the layout of a Lustre file. */ struct llapi_layout; +int llapi_mirror_find_stale(struct llapi_layout *layout, + struct llapi_resync_comp *comp, size_t comp_size, + __u16 *mirror_ids, int ids_nr); +ssize_t llapi_mirror_resync_one(int fd, struct llapi_layout *layout, + uint32_t dst, uint64_t start, uint64_t end); /* * Flags to control how layouts are retrieved. */ diff --git a/lustre/lod/lod_object.c b/lustre/lod/lod_object.c index e39d7d6..8d1622b 100644 --- a/lustre/lod/lod_object.c +++ b/lustre/lod/lod_object.c @@ -5347,10 +5347,15 @@ static void lod_stale_components(struct lod_object *lo, int primary, /* The writing extent decides which components in the primary * are affected... */ + CDEBUG(D_LAYOUT, "primary mirror %d, "DEXT"\n", primary, PEXT(extent)); lod_foreach_mirror_comp(pri_comp, lo, primary) { if (!lu_extent_is_overlapped(extent, &pri_comp->llc_extent)) continue; + CDEBUG(D_LAYOUT, "primary comp %u "DEXT"\n", + lod_comp_index(lo, pri_comp), + PEXT(&pri_comp->llc_extent)); + for (i = 0; i < lo->ldo_mirror_count; i++) { if (i == primary) continue; diff --git a/lustre/tests/mirror_io.c b/lustre/tests/mirror_io.c index 8297771..bf48699 100644 --- a/lustre/tests/mirror_io.c +++ b/lustre/tests/mirror_io.c @@ -337,11 +337,11 @@ enum resync_errors { OPEN_TEST_FILE = 1 << 4, }; -static enum resync_errors resync_parse_error(const char *arg) +static enum resync_errors resync_parse_error(const char *err) { struct { const char *loc; - enum resync_errors error; + enum resync_errors error; } cmds[] = { { "resync_start", AFTER_RESYNC_START }, { "invalid_ids", INVALID_IDS }, @@ -352,165 +352,11 @@ static enum resync_errors resync_parse_error(const char *arg) int i; for (i = 0; i < ARRAY_SIZE(cmds); i++) - if (strcmp(arg, cmds[i].loc) == 0) + if (strcmp(err, cmds[i].loc) == 0) return cmds[i].error; - syserr(1, "unknown error string: %s", arg); - return 0; -} - -struct resync_comp { - uint64_t start; - uint64_t end; - uint32_t mirror_id; - uint32_t id; /* component id */ - bool synced; -}; - -/* find all stale components */ -static size_t mirror_find_stale(struct llapi_layout *layout, - struct resync_comp *comp, size_t max_count) -{ - int idx = 0; - int rc; - - rc = llapi_layout_comp_use(layout, LLAPI_LAYOUT_COMP_USE_FIRST); - syserr(rc < 0, "llapi_layout_comp_move"); - - while (rc == 0) { - uint32_t id; - uint32_t mirror_id; - uint32_t flags; - uint64_t start, end; - - rc = llapi_layout_mirror_id_get(layout, &mirror_id); - syserr(rc < 0, "llapi_layout_comp_id_get"); - - rc = llapi_layout_comp_id_get(layout, &id); - syserr(rc < 0, "llapi_layout_comp_id_get"); - - rc = llapi_layout_comp_flags_get(layout, &flags); - syserr(rc < 0, "llapi_layout_comp_flags_get"); - - rc = llapi_layout_comp_extent_get(layout, &start, &end); - syserr(rc < 0, "llapi_layout_comp_flags_get"); - - if (flags & LCME_FL_STALE) { - comp[idx].id = id; - comp[idx].mirror_id = mirror_id; - comp[idx].start = start; - comp[idx].end = end; - idx++; - - syserr(idx >= max_count, "array too small"); - } - - rc = llapi_layout_comp_use(layout, LLAPI_LAYOUT_COMP_USE_NEXT); - syserr(rc < 0, "llapi_layout_comp_move"); - } - - return idx; -} - -/* locate @layout to a valid component covering file [file_start, file_end) */ -static uint32_t mirror_find(struct llapi_layout *layout, - uint64_t file_start, uint64_t file_end, uint64_t *endp) -{ - uint32_t mirror_id = 0; - int rc; - - rc = llapi_layout_comp_use(layout, LLAPI_LAYOUT_COMP_USE_FIRST); - syserr(rc < 0, "llapi_layout_comp_move"); - - *endp = 0; - while (rc == 0) { - uint64_t start, end; - uint32_t flags, id, rid; - - llapi_layout_mirror_id_get(layout, &rid); - syserr(rc < 0, "llapi_layout_mirror_id_get"); - - rc = llapi_layout_comp_id_get(layout, &id); - syserr(rc < 0, "llapi_layout_comp_id_get"); - - rc = llapi_layout_comp_flags_get(layout, &flags); - syserr(rc < 0, "llapi_layout_comp_flags_get"); - - rc = llapi_layout_comp_extent_get(layout, &start, &end); - syserr(rc < 0, "llapi_layout_comp_extent_get"); - - if (!(flags & LCME_FL_STALE)) { - if (file_start >= start && file_start < end) { - if (mirror_id == 0) - mirror_id = rid; - else if (mirror_id != rid || *endp != start) - break; - - file_start = *endp = end; - if (end >= file_end) - break; - } - } - - rc = llapi_layout_comp_use(layout, LLAPI_LAYOUT_COMP_USE_NEXT); - syserr(rc < 0, "llapi_layout_comp_move"); - } - - return mirror_id; -} - -static char *endstr(uint64_t end) -{ - static char buf[32]; - - if (end == (uint64_t)-1) - return "eof"; - - snprintf(buf, sizeof(buf), "%lx", end); - return buf; -} - -static ssize_t mirror_resync_one(int fd, struct llapi_layout *layout, - uint32_t dst, uint64_t start, uint64_t end) -{ - uint64_t mirror_end; - ssize_t result = 0; - size_t count; - - if (end == OBD_OBJECT_EOF) - count = OBD_OBJECT_EOF; - else - count = end - start; - - while (count > 0) { - uint32_t src; - size_t to_copy; - ssize_t copied; - - src = mirror_find(layout, start, end, &mirror_end); - syserr(!src, "could find component covering %lu\n", start); - - if (mirror_end == OBD_OBJECT_EOF) - to_copy = count; - else - to_copy = MIN(count, mirror_end - start); - - copied = llapi_mirror_copy(fd, src, dst, start, to_copy); - syserr(copied < 0, "llapi_mirror_copy returned %zd\n", copied); - - printf("src (%u) [%lx -> %s) -> dst (%u), copied %zd bytes\n", - src, start, endstr(mirror_end), dst, copied); - - result += copied; - if (copied < to_copy) /* end of file */ - break; - - if (count != OBD_OBJECT_EOF) - count -= copied; - start += copied; - } - - return result; + fprintf(stderr, "unknown error string: %s\n", err); + return -1; } static void mirror_resync(int argc, char *argv[]) @@ -525,7 +371,7 @@ static void mirror_resync(int argc, char *argv[]) struct llapi_layout *layout; struct ll_ioc_lease *ioc; - struct resync_comp comp_array[1024] = { { 0 } }; + struct llapi_resync_comp comp_array[1024] = { { 0 } }; size_t comp_size = 0; uint32_t flr_state; @@ -558,27 +404,37 @@ static void mirror_resync(int argc, char *argv[]) ioc->lil_mode = LL_LEASE_WRLCK; ioc->lil_flags = LL_LEASE_RESYNC; rc = llapi_lease_get_ext(fd, ioc); + if (rc < 0) + free(ioc); syserr(rc < 0, "llapi_lease_get_ext resync"); - if (error_inject & AFTER_RESYNC_START) + if (error_inject & AFTER_RESYNC_START) { + free(ioc); syserrx(1, "hit by error injection"); + } layout = llapi_layout_get_by_fd(fd, 0); + if (layout == NULL) + free(ioc); syserr(layout == NULL, "llapi_layout_get_by_fd"); rc = llapi_layout_flags_get(layout, &flr_state); + if (rc) + free(ioc); syserr(rc, "llapi_layout_flags_get"); flr_state &= LCM_FL_FLR_MASK; - syserrx(flr_state != LCM_FL_WRITE_PENDING && - flr_state != LCM_FL_SYNC_PENDING, - "file state error: %d", flr_state); + if (flr_state != LCM_FL_WRITE_PENDING && + flr_state != LCM_FL_SYNC_PENDING) { + free(ioc); + syserrx(true, "file state error: %d", flr_state); + } if (error_inject & DELAY_BEFORE_COPY) sleep(delay); - comp_size = mirror_find_stale(layout, comp_array, - ARRAY_SIZE(comp_array)); + comp_size = llapi_mirror_find_stale(layout, comp_array, + ARRAY_SIZE(comp_array), NULL, 0); printf("%s: found %zd stale components\n", fname, comp_size); @@ -592,35 +448,37 @@ static void mirror_resync(int argc, char *argv[]) rc = llapi_lease_check(fd); syserr(rc != LL_LEASE_WRLCK, "lost lease lock"); - mirror_id = comp_array[idx].mirror_id; - end = comp_array[idx].end; + mirror_id = comp_array[idx].lrc_mirror_id; + end = comp_array[idx].lrc_end; printf("%s: resyncing mirror: %u, components: %u ", - fname, mirror_id, comp_array[idx].id); + fname, mirror_id, comp_array[idx].lrc_id); for (i = idx + 1; i < comp_size; i++) { - if (mirror_id != comp_array[i].mirror_id || - end != comp_array[i].start) + if (mirror_id != comp_array[i].lrc_mirror_id || + end != comp_array[i].lrc_start) break; - printf("%u ", comp_array[i].id); - end = comp_array[i].end; + printf("%u ", comp_array[i].lrc_id); + end = comp_array[i].lrc_end; } printf("\b\n"); - res = mirror_resync_one(fd, layout, mirror_id, - comp_array[idx].start, end); + res = llapi_mirror_resync_one(fd, layout, mirror_id, + comp_array[idx].lrc_start, end); if (res > 0) { int j; printf("components synced: "); for (j = idx; j < i; j++) { - comp_array[j].synced = true; - printf("%u ", comp_array[j].id); + comp_array[j].lrc_synced = true; + printf("%u ", comp_array[j].lrc_id); } printf("\n"); } + if (res < 0) + free(ioc); syserrx(res < 0, "llapi_mirror_copy_many"); idx = i; @@ -631,8 +489,8 @@ static void mirror_resync(int argc, char *argv[]) ioc->lil_flags = LL_LEASE_RESYNC_DONE; ioc->lil_count = 0; for (idx = 0; idx < comp_size; idx++) { - if (comp_array[idx].synced) { - ioc->lil_ids[ioc->lil_count] = comp_array[idx].id; + if (comp_array[idx].lrc_synced) { + ioc->lil_ids[ioc->lil_count] = comp_array[idx].lrc_id; ioc->lil_count++; } } @@ -649,10 +507,9 @@ static void mirror_resync(int argc, char *argv[]) close(open(argv[optind], O_RDONLY)); rc = llapi_lease_get_ext(fd, ioc); - syserr(rc < 0, "llapi_lease_get_ext resync done"); - - syserr(rc == 0, "file busy"); + syserr(rc <= 0, "llapi_lease_get_ext resync failed"); + free(ioc); close(fd); } diff --git a/lustre/tests/sanity-flr.sh b/lustre/tests/sanity-flr.sh index 2e7fa22..b2e6ae8 100644 --- a/lustre/tests/sanity-flr.sh +++ b/lustre/tests/sanity-flr.sh @@ -1232,6 +1232,38 @@ test_40() { } run_test 40 "PFLR rdonly state instantiation check" +test_41() { + local tf=$DIR/$tfile + + rm -f $tf + $LFS mirror create -N -E2m -E4m -E-1 -N -E1m -E2m -E3m -E-1 $tf || + error "create PFLR file $tf failed" + + # file should be in ro status + verify_flr_state $tf "ro" + + # write data in [0, 2M) + dd if=/dev/zero of=$tf bs=1M count=2 conv=notrunc || + error "writing $tf failed" + + verify_flr_state $tf "wp" + + # file should have stale component + $LFS getstripe $tf | grep lcme_flags | grep stale > /dev/null || + error "after writing $tf, it does not contain stale component" + + $LFS mirror resync $tf || error "mirror resync $tf failed" + + verify_flr_state $tf "ro" + + # file should not have stale component + $LFS getstripe $tf | grep lcme_flags | grep stale && + error "after resyncing $tf, it contains stale component" + + return 0 +} +run_test 41 "lfs mirror resync check" + ctrl_file=$(mktemp /tmp/CTRL.XXXXXX) lock_file=$(mktemp /var/lock/FLR.XXXXXX) @@ -1274,8 +1306,11 @@ resync_file_200() { exec 200<>$lock_file while [ -f $ctrl_file ]; do - local index=$((RANDOM % ${#options[@]})) local lock_taken=false + local index=$((RANDOM % ${#options[@]})) + local cmd="mirror_io resync ${options[$index]}" + + [ "${options[$index]}" = "" ] && cmd="$LFS mirror resync" [ $((RANDOM % 4)) -eq 0 ] && { index=0 @@ -1283,12 +1318,10 @@ resync_file_200() { echo -n "lock to " } - echo -n "resync file $tf with '${options[$index]}' .." + echo -n "resync file $tf with '$cmd' .." $lock_taken && flock -x 200 - mirror_io resync ${options[$index]} $tf &> /dev/null && - echo "done" || echo "failed" - + $cmd $tf &> /dev/null && echo "done" || echo "failed" $lock_taken && flock -u 200 sleep 0.$((RANDOM % 8 + 1)) diff --git a/lustre/utils/lfs.c b/lustre/utils/lfs.c index 5aebbae..c6f0f94 100644 --- a/lustre/utils/lfs.c +++ b/lustre/utils/lfs.c @@ -111,6 +111,7 @@ static int lfs_ladvise(int argc, char **argv); static int lfs_mirror(int argc, char **argv); static int lfs_mirror_list_commands(int argc, char **argv); static int lfs_list_commands(int argc, char **argv); +static inline int lfs_mirror_resync(int argc, char **argv); enum setstripe_origin { SO_SETSTRIPE, @@ -232,16 +233,20 @@ static const char *progname; command_t mirror_cmdlist[] = { { .pc_name = "create", .pc_func = lfs_mirror_create, .pc_help = "Create a mirrored file.\n" - "usage: lfs mirror create " - "<--mirror-count|-N[mirror_count]> " - "[setstripe options|--parent] ... \n" + "usage: lfs mirror create " + "<--mirror-count|-N[mirror_count]> " + "[setstripe options|--parent] ... \n" MIRROR_CREATE_HELP }, { .pc_name = "extend", .pc_func = lfs_mirror_extend, .pc_help = "Extend a mirrored file.\n" - "usage: lfs mirror extend " - "<--mirror-count|-N[mirror_count]> [--no-verify] " - "[setstripe options|--parent|-f ] ... \n" + "usage: lfs mirror extend " + "<--mirror-count|-N[mirror_count]> [--no-verify] " + "[setstripe options|--parent|-f ] ... \n" MIRROR_EXTEND_HELP }, + { .pc_name = "resync", .pc_func = lfs_mirror_resync, + .pc_help = "Resynchronizes an out-of-sync mirrored file.\n" + "usage: lfs mirror resync [--only ] " + "\n"}, { .pc_name = "--list-commands", .pc_func = lfs_mirror_list_commands, .pc_help = "list commands supported by lfs mirror"}, { .pc_name = "help", .pc_func = Parser_help, .pc_help = "help" }, @@ -6138,6 +6143,277 @@ next: return rc; } +/** The input string contains a comma delimited list of component ids and + * ranges, for example "1,2-4,7". + */ +static int parse_mirror_ids(__u16 *ids, int size, char *arg) +{ + bool end_of_loop = false; + char *ptr = NULL; + int nr = 0; + int rc; + + if (arg == NULL) + return -EINVAL; + + while (!end_of_loop) { + int start_index; + int end_index; + int i; + char *endptr = NULL; + + rc = -EINVAL; + ptr = strchrnul(arg, ','); + end_of_loop = *ptr == '\0'; + *ptr = '\0'; + + start_index = strtol(arg, &endptr, 0); + if (endptr == arg) /* no data at all */ + break; + if (*endptr != '-' && *endptr != '\0') /* has invalid data */ + break; + if (start_index < 0) + break; + + end_index = start_index; + if (*endptr == '-') { + end_index = strtol(endptr + 1, &endptr, 0); + if (*endptr != '\0') + break; + if (end_index < start_index) + break; + } + + for (i = start_index; i <= end_index && size > 0; i++) { + int j; + + /* remove duplicate */ + for (j = 0; j < nr; j++) { + if (ids[j] == i) + break; + } + if (j == nr) { /* no duplicate */ + ids[nr++] = i; + --size; + } + } + + if (size == 0 && i < end_index) + break; + + *ptr = ','; + arg = ++ptr; + rc = 0; + } + if (!end_of_loop && ptr != NULL) + *ptr = ','; + + return rc < 0 ? rc : nr; +} + +static inline int lfs_mirror_resync(int argc, char **argv) +{ + const char *fname; + struct stat stbuf; + int fd; + int c; + int rc; + int idx; + + struct llapi_layout *layout; + struct ll_ioc_lease *ioc = NULL; + struct llapi_resync_comp comp_array[1024] = { { 0 } }; + __u16 mirror_ids[128] = { 0 }; + int ids_nr = 0; + int comp_size = 0; + uint32_t flr_state; + + struct option long_opts[] = { + { .val = 'o', .name = "only", .has_arg = required_argument }, + { .name = NULL } }; + + while ((c = getopt_long(argc, argv, "o:", long_opts, NULL)) >= 0) { + switch (c) { + case 'o': + rc = parse_mirror_ids(mirror_ids, + sizeof(mirror_ids) / sizeof(__u16), + optarg); + if (rc < 0) { + fprintf(stderr, + "%s: bad mirror ids '%s'.\n", + argv[0], optarg); + goto error; + } + ids_nr = rc; + break; + default: + fprintf(stderr, "%s: options '%s' unrecognized.\n", + argv[0], argv[optind - 1]); + rc = -EINVAL; + goto error; + } + } + + if (argc > optind + 1) { + fprintf(stderr, "%s: too many files.\n", argv[0]); + rc = CMD_HELP; + goto error; + } + if (argc == optind) { + fprintf(stderr, "%s: no file name given.\n", argv[0]); + rc = CMD_HELP; + goto error; + } + + fname = argv[optind]; + if (stat(fname, &stbuf) < 0) { + fprintf(stderr, "%s: cannot stat file '%s': %s.\n", + argv[0], fname, strerror(errno)); + rc = -errno; + goto error; + } + if (!S_ISREG(stbuf.st_mode)) { + fprintf(stderr, "%s: '%s' is not a regular file.\n", + argv[0], fname); + rc = -EINVAL; + goto error; + } + + fd = open(fname, O_DIRECT | O_RDWR); + if (fd < 0) { + fprintf(stderr, "%s: cannot open '%s': %s.\n", + argv[0], fname, strerror(errno)); + rc = -errno; + goto error; + } + + /* set the lease on the file */ + ioc = calloc(sizeof(*ioc) + sizeof(__u32) * 4096, 1); + if (ioc == NULL) { + fprintf(stderr, "%s: cannot alloc id array for ioc: %s.\n", + argv[0], strerror(errno)); + rc = -errno; + goto close_fd; + } + + ioc->lil_mode = LL_LEASE_WRLCK; + ioc->lil_flags = LL_LEASE_RESYNC; + rc = llapi_lease_get_ext(fd, ioc); + if (rc < 0) { + fprintf(stderr, "%s: llapi_lease_get_ext resync failed: %s.\n", + argv[0], strerror(errno)); + goto free_ioc; + } + + layout = llapi_layout_get_by_fd(fd, 0); + if (layout == NULL) { + fprintf(stderr, "%s: llapi_layout_get_by_fd failed: %s.\n", + argv[0], strerror(errno)); + rc = -errno; + goto free_ioc; + } + + rc = llapi_layout_flags_get(layout, &flr_state); + if (rc) { + fprintf(stderr, "%s: llapi_layout_flags_get failed: %s.\n", + argv[0], strerror(errno)); + rc = -errno; + goto free_ioc; + } + + flr_state &= LCM_FL_FLR_MASK; + if (flr_state != LCM_FL_WRITE_PENDING && + flr_state != LCM_FL_SYNC_PENDING) { + fprintf(stderr, "%s: file state error: %s.\n", + argv[0], lcm_flags_string(flr_state)); + rc = 1; + goto free_ioc; + } + + /* get stale component info */ + comp_size = llapi_mirror_find_stale(layout, comp_array, + ARRAY_SIZE(comp_array), + mirror_ids, ids_nr); + if (comp_size < 0) { + rc = comp_size; + goto free_ioc; + } + + idx = 0; + while (idx < comp_size) { + ssize_t result; + uint64_t end; + __u16 mirror_id; + int i; + + rc = llapi_lease_check(fd); + if (rc != LL_LEASE_WRLCK) { + fprintf(stderr, "lost lease lock.\n"); + goto free_ioc; + } + + mirror_id = comp_array[idx].lrc_mirror_id; + end = comp_array[idx].lrc_end; + + /* try to combine adjacent component */ + for (i = idx + 1; i < comp_size; i++) { + if (mirror_id != comp_array[i].lrc_mirror_id || + end != comp_array[i].lrc_start) + break; + end = comp_array[i].lrc_end; + } + + result = llapi_mirror_resync_one(fd, layout, mirror_id, + comp_array[idx].lrc_start, + end); + if (result < 0) { + fprintf(stderr, "llapi_mirror_resync_one: %ld.\n", + result); + rc = result; + goto free_ioc; + } else if (result > 0) { + int j; + + /* mark synced components */ + for (j = idx; j < i; j++) + comp_array[j].lrc_synced = true; + } + + idx = i; + } + + /* prepare ioc for lease put */ + ioc->lil_mode = LL_LEASE_UNLCK; + ioc->lil_flags = LL_LEASE_RESYNC_DONE; + ioc->lil_count = 0; + for (idx = 0; idx < comp_size; idx++) { + if (comp_array[idx].lrc_synced) { + ioc->lil_ids[ioc->lil_count] = comp_array[idx].lrc_id; + ioc->lil_count++; + } + } + + llapi_layout_free(layout); + + rc = llapi_lease_get_ext(fd, ioc); + if (rc <= 0) { + if (rc == 0) /* lost lease lock */ + rc = -EBUSY; + fprintf(stderr, "%s: resync file '%s' failed: %s.\n", + argv[0], fname, strerror(errno)); + goto free_ioc; + } + rc = 0; + +free_ioc: + if (ioc) + free(ioc); +close_fd: + close(fd); +error: + return rc; +} + /** * lfs_mirror() - Parse and execute lfs mirror commands. * @argc: The count of lfs mirror command line arguments. diff --git a/lustre/utils/liblustreapi.c b/lustre/utils/liblustreapi.c index f238a17..ecac4c3 100644 --- a/lustre/utils/liblustreapi.c +++ b/lustre/utils/liblustreapi.c @@ -2593,7 +2593,7 @@ void lmv_dump_user_lmm(struct lmv_user_md *lum, char *pool_name, llapi_printf(LLAPI_MSG_NORMAL, "\n"); } -static char *lcm_flags_string(__u16 flags) +char *lcm_flags_string(__u16 flags) { switch (flags & LCM_FL_FLR_MASK) { case LCM_FL_NOT_FLR: diff --git a/lustre/utils/liblustreapi_layout.c b/lustre/utils/liblustreapi_layout.c index ae2826e..eacda04 100644 --- a/lustre/utils/liblustreapi_layout.c +++ b/lustre/utils/liblustreapi_layout.c @@ -35,6 +35,7 @@ #include #include #include +#include #include #include @@ -2214,3 +2215,213 @@ error: llapi_layout_free(new_layout); return -1; } + +/** + * Find all stale components. + * + * \param[in] layout component layout list. + * \param[out] comp array of stale component info. + * \param[in] comp_size array size of @comp. + * \param[in] mirror_ids array of mirror id that only components + * belonging to these mirror will be collected. + * \param[in] ids_nr number of mirror ids array. + * + * \retval number of component info collected on sucess or + * an error code on failure. + */ +int llapi_mirror_find_stale(struct llapi_layout *layout, + struct llapi_resync_comp *comp, size_t comp_size, + __u16 *mirror_ids, int ids_nr) +{ + int idx = 0; + int rc; + + rc = llapi_layout_comp_use(layout, LLAPI_LAYOUT_COMP_USE_FIRST); + if (rc < 0) { + fprintf(stderr, "%s: move to the first layout component: %s.\n", + __func__, strerror(errno)); + goto error; + } + + while (rc == 0) { + uint32_t id; + uint32_t mirror_id; + uint32_t flags; + uint64_t start, end; + + rc = llapi_layout_comp_flags_get(layout, &flags); + if (rc < 0) { + fprintf(stderr, "llapi_layout_comp_flags_get: %s.\n", + strerror(errno)); + goto error; + } + + if (!(flags & LCME_FL_STALE)) + goto next; + + rc = llapi_layout_mirror_id_get(layout, &mirror_id); + if (rc < 0) { + fprintf(stderr, "llapi_layout_mirror_id_get: %s.\n", + strerror(errno)); + goto error; + } + + /* the caller only wants stale components from specific + * mirrors */ + if (ids_nr > 0) { + int j; + + for (j = 0; j < ids_nr; j++) { + if (mirror_ids[j] == mirror_id) + break; + } + + /* not in the specified mirror */ + if (j == ids_nr) + goto next; + } + + rc = llapi_layout_comp_id_get(layout, &id); + if (rc < 0) { + fprintf(stderr, "llapi_layout_comp_id_get: %s.\n", + strerror(errno)); + goto error; + } + + rc = llapi_layout_comp_extent_get(layout, &start, &end); + if (rc < 0) { + fprintf(stderr, "llapi_layout_comp_extent_get: %s.\n", + strerror(errno)); + goto error; + } + + /* pack this component into @comp array */ + comp[idx].lrc_id = id; + comp[idx].lrc_mirror_id = mirror_id; + comp[idx].lrc_start = start; + comp[idx].lrc_end = end; + idx++; + + if (idx >= comp_size) { + fprintf(stderr, "%s: resync_comp array too small.\n", + __func__); + rc = -EINVAL; + goto error; + } + + next: + rc = llapi_layout_comp_use(layout, LLAPI_LAYOUT_COMP_USE_NEXT); + if (rc < 0) { + fprintf(stderr, "%s: move to the next layout " + "component: %s.\n", __func__, strerror(errno)); + rc = -EINVAL; + goto error; + } + } +error: + return rc < 0 ? rc : idx; +} + +/* locate @layout to a valid component covering file [file_start, file_end) */ +static uint32_t llapi_mirror_find(struct llapi_layout *layout, + uint64_t file_start, uint64_t file_end, + uint64_t *endp) +{ + uint32_t mirror_id = 0; + int rc; + + rc = llapi_layout_comp_use(layout, LLAPI_LAYOUT_COMP_USE_FIRST); + if (rc < 0) + return rc; + + *endp = 0; + while (rc == 0) { + uint64_t start, end; + uint32_t flags, id, rid; + + rc = llapi_layout_comp_flags_get(layout, &flags); + if (rc < 0) + return rc; + + if (flags & LCME_FL_STALE) + goto next; + + rc = llapi_layout_mirror_id_get(layout, &rid); + if (rc < 0) + return rc; + + rc = llapi_layout_comp_id_get(layout, &id); + if (rc < 0) + return rc; + + rc = llapi_layout_comp_extent_get(layout, &start, &end); + if (rc < 0) + return rc; + + if (file_start >= start && file_start < end) { + if (!mirror_id) + mirror_id = rid; + else if (mirror_id != rid || *endp != start) + break; + + file_start = *endp = end; + if (end >= file_end) + break; + } + + next: + rc = llapi_layout_comp_use(layout, LLAPI_LAYOUT_COMP_USE_NEXT); + if (rc < 0) + return rc; + } + + return mirror_id; +} + +ssize_t llapi_mirror_resync_one(int fd, struct llapi_layout *layout, + uint32_t dst, uint64_t start, uint64_t end) +{ + uint64_t mirror_end = 0; + ssize_t result = 0; + size_t count; + + if (end == OBD_OBJECT_EOF) + count = OBD_OBJECT_EOF; + else + count = end - start; + + while (count > 0) { + uint32_t src; + size_t to_copy; + ssize_t copied; + + src = llapi_mirror_find(layout, start, end, &mirror_end); + if (src == 0) { + fprintf(stderr, "llapi_mirror_find cannot find " + "component covering %lu.\n", start); + return -ENOENT; + } + + if (mirror_end == OBD_OBJECT_EOF) + to_copy = count; + else + to_copy = MIN(count, mirror_end - start); + + copied = llapi_mirror_copy(fd, src, dst, start, to_copy); + if (copied < 0) { + fprintf(stderr, "llapi_mirror_copy returned %zd.\n", + copied); + return copied; + } + + result += copied; + if (copied < to_copy) /* end of file */ + break; + + if (count != OBD_OBJECT_EOF) + count -= copied; + start += copied; + } + + return result; +} diff --git a/lustre/utils/liblustreapi_lease.c b/lustre/utils/liblustreapi_lease.c index 339b2f5..cb254b1 100644 --- a/lustre/utils/liblustreapi_lease.c +++ b/lustre/utils/liblustreapi_lease.c @@ -50,7 +50,10 @@ static inline const char *lease_mode2str(int mode) * \param fd File to get lease on. * \param data ll_ioc_lease data. * - * \retval 0 on success. + * For getting lease lock, it will return zero for success. For unlock, it will + * return the lock type it owned for succuess. + * + * \retval >= 0 on success. * \retval -errno on error. */ int llapi_lease_get_ext(int fd, struct ll_ioc_lease *data) @@ -78,7 +81,9 @@ int llapi_lease_get_ext(int fd, struct ll_ioc_lease *data) * \param fd File to get the lease on. * \param mode Lease mode, either LL_LEASE_RDLCK or LL_LEASE_WRLCK. * - * \retval 0 on success. + * \see llapi_lease_get_ext(). + * + * \retval >= 0 on success. * \retval -errno on error. */ int llapi_lease_get(int fd, int mode)