From: Qian Yingjin Date: Fri, 11 Nov 2022 09:01:02 +0000 (-0500) Subject: EX-6373 pcc: asynchronous PCCRO attach command support X-Git-Url: https://git.whamcloud.com/?a=commitdiff_plain;h=ab6bdf0fc76adc94403686826a5d657391ffcd20;p=fs%2Flustre-release.git EX-6373 pcc: asynchronous PCCRO attach command support Currently PCCRO attach via the command "lfs pcc attach" will block during the data copying. There is a requirement that this command can also do data copy asynchronously. Thus we add an option "--async|-A" to the command which will not block while the file data is being fetched. Add sanity-pcc/test_{103, 104} to verify that it works correctly. Change-Id: I6f31190c8b9e9b9876b34f8e484c6c8b7f16b6db Signed-off-by: Qian Yingjin Reviewed-on: https://review.whamcloud.com/c/ex/lustre-release/+/49133 Tested-by: jenkins Tested-by: Maloo Reviewed-by: Feng Lei Reviewed-by: Andreas Dilger --- diff --git a/lustre/doc/llapi_pcc_attach.3 b/lustre/doc/llapi_pcc_attach.3 index a283559..f6741db 100644 --- a/lustre/doc/llapi_pcc_attach.3 +++ b/lustre/doc/llapi_pcc_attach.3 @@ -40,9 +40,11 @@ argument, which is a data structure, which contains the following values: .nf .LP - LU_PCC_NONE - LU_PCC_READWRITE - LU_PCC_READONLY + LU_PCC_NONE = 0x0, + LU_PCC_READWRITE = 0x01, + LU_PCC_READONLY = 0x02, + LU_PCC_TYPE_MASK = LU_PCC_READWRITE | LU_PCC_READONLY, + LU_PCC_FL_ASYNC = 0x10, .fi .TP LU_PCC_NONE @@ -53,6 +55,12 @@ means RW-PCC mode. .TP LU_PCC_READONLY means RO-PCC mode. +.TP +LU_PCC_TYPE_MASK +is the bits mask for PCC modes. +.TP +LU_PCC_FL_ASYNC +indicates to do data copying in asynchronous way during attach. .SH RETURN VALUES .PP .B llapi_pcc_attach() diff --git a/lustre/include/uapi/linux/lustre/lustre_user.h b/lustre/include/uapi/linux/lustre/lustre_user.h index 63460d7..cab8535 100644 --- a/lustre/include/uapi/linux/lustre/lustre_user.h +++ b/lustre/include/uapi/linux/lustre/lustre_user.h @@ -2724,6 +2724,7 @@ enum lu_pcc_type { LU_PCC_READWRITE = 0x01, LU_PCC_READONLY = 0x02, LU_PCC_TYPE_MASK = LU_PCC_READWRITE | LU_PCC_READONLY, + LU_PCC_FL_ASYNC = 0x10, LU_PCC_MAX }; @@ -2807,8 +2808,8 @@ static inline enum hsmtool_type hsmtool_string2type(const char *str) } struct lu_pcc_attach { - __u32 pcca_type; /* PCC type */ - __u32 pcca_id; /* Attach ID */ + __u32 pcca_type;/* PCC type */ + __u32 pcca_id;/* Attach ID */ }; enum lu_pcc_detach_flags { diff --git a/lustre/llite/pcc.c b/lustre/llite/pcc.c index 79e250e..a9fa394 100644 --- a/lustre/llite/pcc.c +++ b/lustre/llite/pcc.c @@ -4346,17 +4346,32 @@ static int pcc_readonly_attach_sync(struct file *file, struct inode *inode, int pcc_ioctl_attach(struct file *file, struct inode *inode, struct lu_pcc_attach *attach) { + struct pcc_super *super = ll_i2pccs(inode); int rc = 0; ENTRY; - switch (attach->pcca_type) { + switch (attach->pcca_type & LU_PCC_TYPE_MASK) { case LU_PCC_READWRITE: rc = -ENOTSUPP; break; case LU_PCC_READONLY: - rc = pcc_readonly_attach_sync(file, inode, - attach->pcca_id, false); + if (attach->pcca_type & LU_PCC_FL_ASYNC && + atomic_read(&super->pccs_attaches_queued) < + super->pccs_maximum_queued_attaches) { + rc = pcc_readonly_attach_async(file, inode, + attach->pcca_id); + /* + * Ignore EEXIST error if the file has already attached. + * Ignore EINPROGRESS error if the file is being + * attached, i.e. copy data from OSTs into PCC. + */ + if (rc == -EEXIST || rc == -EINPROGRESS) + rc = 0; + } else { + rc = pcc_readonly_attach_sync(file, inode, + attach->pcca_id, false); + } break; default: rc = -EINVAL; diff --git a/lustre/tests/sanity-pcc.sh b/lustre/tests/sanity-pcc.sh index f2c3a59..3a53c8a 100644 --- a/lustre/tests/sanity-pcc.sh +++ b/lustre/tests/sanity-pcc.sh @@ -4740,6 +4740,74 @@ test_102() { } run_test 102 "PCC-RO should not hange for io_uring I/O engine" +test_103() { + local loopfile="$TMP/$tfile" + local mntpt="/mnt/pcc.$tdir" + local hsm_root="$mntpt/$tdir" + local file=$DIR/$tfile + + $LCTL get_param -n mdc.*.connect_flags | grep -q pcc_ro || + skip "Server does not support PCC-RO" + + setup_loopdev client $loopfile $mntpt 300 + mkdir $hsm_root || error "mkdir $hsm_root failed" + setup_pcc_mapping client \ + "projid={100}\ roid=$HSM_ARCHIVE_NUMBER\ pccro=1" + + $LFS setstripe -c -1 $file || error "failed to setstripe $file" + fallocate -l 200M $file || error "fallocate $file failed" + stack_trap "rm -f $file" + cancel_lru_locks osc + $LFS pcc attach -i $HSM_ARCHIVE_NUMBER -A $file || + error "failed to PCCRO attach -A $file" + $LFS pcc state $file | grep -q 'flags: attaching' || + error "Not do asynchronous PCCRO attach for $file" + wait_readonly_attach_fini $file client + + $LFS pcc detach $file || error "failed to PCC detach $file" + $LFS pcc attach -A $file || error "failed to PCCRO attach -A $file" + $LFS pcc state $file | grep -q 'flags: attaching' || + error "Not do asynchronous PCCRO attach for $file" + wait_readonly_attach_fini $file client +} +run_test 103 "Test asynchronous manual attach for PCCRO" + +test_104() { + local loopfile="$TMP/$tfile" + local mntpt="/mnt/pcc.$tdir" + local hsm_root="$mntpt/$tdir" + local file=$DIR/$tfile + + $LCTL get_param -n mdc.*.connect_flags | grep -q pcc_ro || + skip "Server does not support PCC-RO" + + setup_loopdev client $loopfile $mntpt 600 + mkdir $hsm_root || error "mkdir $hsm_root failed" + setup_pcc_mapping client \ + "projid={0}\ roid=$HSM_ARCHIVE_NUMBER\ pccro=1" + + $LFS setstripe -c -1 $file || error "failed to setstripe $file" + fallocate -l 500M $file || error "fallocate $file failed" + stack_trap "rm -f $file" + cancel_lru_locks osc + # $LFS pcc attach will call realpath(): + # In rhel8, it uses stat() to tell about the file at the end of any + # chain of symlinks. And it will obtain the size on the client. + # In Ubuntu2204, it uses readlink() to obtain the path name. + # As fallocate() does not update the size on MDT (LU-16334), thus we + # use stat to obtain the size on the client explictly. Otherwise, the + # known size on the client is zero, and lower than pcc_async_threshold, + # the client would do attach at open() in the synchronous way. + stat $file || error "failed to stat $file" + $LFS pcc attach -i $HSM_ARCHIVE_NUMBER -A $file || + error "failed to PCCRO attach -A $file" + $LFS pcc state $file + $LFS pcc state $file | grep -q 'flags: attaching' || + error "Not do asynchronous PCCRO attach for $file" + wait_readonly_attach_fini $file client +} +run_test 104 "Test asynchronous manual attach and auto open attach for PCCRO" + test_200() { local loopfile="$TMP/$tfile" local mntpt="/mnt/pcc.$tdir" diff --git a/lustre/utils/lfs.c b/lustre/utils/lfs.c index c7df3f0..53bb376 100644 --- a/lustre/utils/lfs.c +++ b/lustre/utils/lfs.c @@ -376,19 +376,21 @@ command_t mirror_cmdlist[] = { command_t pcc_cmdlist[] = { { .pc_name = "attach", .pc_func = lfs_pcc_attach, .pc_help = "Copy given files to PCC, the local cache storage.\n" - "usage: lfs pcc attach [--id|-i ID] [--readonly|-r] [--write|-w] [-p|--pin] ...\n" + "usage: lfs pcc attach [--id|-i ID] [--readonly|-r] [--write|-w] [--pin|-p] [--async|-A] ...\n" "\t-i: archive ID for PCC\n" "\t-r: readonly attach\n" "\t-w: writeable attach\n" - "\t-p: pin and attach the file\n" }, + "\t-p: pin and attach the file\n" + "\t-A: copy data asynchronously during readonly attach\n" }, { .pc_name = "attach_fid", .pc_func = lfs_pcc_attach_fid, .pc_help = "Copy given files to PCC by FID(s), the local cache storage.\n" - "usage: lfs pcc attach_fid [--id|-i ID] {--mnt|-m MOUNTPATH} [--readonly|-r] [--write|-w] [-p|--pin] ...\n" + "usage: lfs pcc attach_fid [--id|-i ID] {--mnt|-m MOUNTPATH} [--readonly|-r] [--write|-w] [--pin|-p] [--async|-A] ...\n" "\t-i: archive ID for PCC\n" "\t-m: Lustre mount point\n" "\t-r: readonly attach\n" "\t-w: writeable attach\n" - "\t-p: pin and attach the file\n" }, + "\t-p: pin and attach the file\n" + "\t-A: copy data asynchronously during readonly attach\n" }, { .pc_name = "state", .pc_func = lfs_pcc_state, .pc_help = "Display the PCC state for given files.\n" "usage: lfs pcc state ...\n" }, @@ -12156,6 +12158,7 @@ static int lfs_pcc_attach(int argc, char **argv) { .val = 'r', .name = "readonly", .has_arg = no_argument }, { .val = 'w', .name = "write", .has_arg = no_argument }, { .val = 'p', .name = "pin", .has_arg = no_argument }, + { .val = 'A', .name = "async", .has_arg = no_argument }, { .name = NULL } }; int c; int rc = 0; @@ -12167,7 +12170,7 @@ static int lfs_pcc_attach(int argc, char **argv) bool pin = false; optind = 0; - while ((c = getopt_long(argc, argv, "i:prw", + while ((c = getopt_long(argc, argv, "i:prwA", long_opts, NULL)) != -1) { switch (c) { case 'i': @@ -12182,14 +12185,19 @@ static int lfs_pcc_attach(int argc, char **argv) } break; case 'r': - type = LU_PCC_READONLY; + type |= LU_PCC_READONLY; + type &= ~LU_PCC_READWRITE; break; case 'w': - type = LU_PCC_READWRITE; + type |= LU_PCC_READWRITE; + type &= ~LU_PCC_READONLY; break; case 'p': pin = true; break; + case 'A': + type |= LU_PCC_FL_ASYNC; + break; case '?': return CMD_HELP; default: @@ -12205,6 +12213,13 @@ static int lfs_pcc_attach(int argc, char **argv) return CMD_HELP; } + if ((type & LU_PCC_READWRITE) && (type & LU_PCC_FL_ASYNC)) { + fprintf(stderr, + "%s: PCCRW does not support asynchronous attach\n", + argv[0]); + return CMD_HELP; + } + while (optind < argc) { int rc2; @@ -12259,8 +12274,9 @@ static int lfs_pcc_attach_fid(int argc, char **argv) { .val = 'w', .name = "write", .has_arg = no_argument }, { .val = 'm', .name = "mnt", .has_arg = required_argument }, { .val = 'p', .name = "pin", .has_arg = no_argument }, + { .val = 'A', .name = "async", .has_arg = no_argument }, { .name = NULL } }; - char short_opts[] = "i:m:prw"; + char short_opts[] = "i:m:prwA"; int c; int rc = 0; __u32 attach_id = 0; @@ -12287,10 +12303,12 @@ static int lfs_pcc_attach_fid(int argc, char **argv) } break; case 'r': - type = LU_PCC_READONLY; + type |= LU_PCC_READONLY; + type &= ~LU_PCC_READWRITE; break; case 'w': - type = LU_PCC_READWRITE; + type |= LU_PCC_READWRITE; + type &= ~LU_PCC_READONLY; break; case 'm': mntpath = optarg; @@ -12298,6 +12316,9 @@ static int lfs_pcc_attach_fid(int argc, char **argv) case 'p': pin = true; break; + case 'A': + type |= LU_PCC_FL_ASYNC; + break; case '?': return CMD_HELP; default: @@ -12316,6 +12337,13 @@ static int lfs_pcc_attach_fid(int argc, char **argv) mntpath = argv[optind++]; } + if((type & LU_PCC_READWRITE) && (type & LU_PCC_FL_ASYNC)) { + fprintf(stderr, + "%s: PCCRW does not support asynchronous attach\n", + argv[0]); + return CMD_HELP; + } + if (argc <= optind) { fprintf(stderr, "%s: must specify one or more fids\n", argv[0]); return CMD_HELP; diff --git a/lustre/utils/liblustreapi_pcc.c b/lustre/utils/liblustreapi_pcc.c index 2a90ee3..c95218f 100644 --- a/lustre/utils/liblustreapi_pcc.c +++ b/lustre/utils/liblustreapi_pcc.c @@ -50,7 +50,7 @@ * Fetch and attach a file to readwrite PCC. * */ -static int llapi_readwrite_pcc_attach_fd(int fd, __u32 archive_id) +static int llapi_pcc_attach_rw_fd(int fd, __u32 archive_id) { int rc; struct ll_ioc_lease *data; @@ -87,7 +87,7 @@ static int llapi_readwrite_pcc_attach_fd(int fd, __u32 archive_id) return rc; } -static int llapi_readwrite_pcc_attach(const char *path, __u32 archive_id) +static int llapi_pcc_attach_rw(const char *path, __u32 archive_id) { int fd; int rc; @@ -100,19 +100,19 @@ static int llapi_readwrite_pcc_attach(const char *path, __u32 archive_id) return rc; } - rc = llapi_readwrite_pcc_attach_fd(fd, archive_id); + rc = llapi_pcc_attach_rw_fd(fd, archive_id); close(fd); return rc; } -static int llapi_readonly_pcc_attach_fd(int fd, __u32 roid) +static int llapi_pcc_attach_ro_fd(int fd, __u32 roid, enum lu_pcc_type type) { struct lu_pcc_attach attach; int rc; attach.pcca_id = roid; - attach.pcca_type = LU_PCC_READONLY; + attach.pcca_type = type; rc = ioctl(fd, LL_IOC_PCC_ATTACH, &attach); if (rc) { rc = -errno; @@ -124,7 +124,8 @@ static int llapi_readonly_pcc_attach_fd(int fd, __u32 roid) return rc; } -static int llapi_readonly_pcc_attach(const char *path, __u32 roid) +static int llapi_pcc_attach_ro(const char *path, __u32 roid, + enum lu_pcc_type type) { int fd; int rc; @@ -144,7 +145,7 @@ static int llapi_readonly_pcc_attach(const char *path, __u32 roid) return rc; } - rc = llapi_readonly_pcc_attach_fd(fd, roid); + rc = llapi_pcc_attach_ro_fd(fd, roid, type); close(fd); return rc; @@ -156,10 +157,10 @@ int llapi_pcc_attach(const char *path, __u32 id, enum lu_pcc_type type) switch (type & LU_PCC_TYPE_MASK) { case LU_PCC_READWRITE: - rc = llapi_readwrite_pcc_attach(path, id); + rc = llapi_pcc_attach_rw(path, id); break; case LU_PCC_READONLY: - rc = llapi_readonly_pcc_attach(path, id); + rc = llapi_pcc_attach_ro(path, id, type); break; default: rc = -EINVAL; @@ -168,9 +169,8 @@ int llapi_pcc_attach(const char *path, __u32 id, enum lu_pcc_type type) return rc; } -static int llapi_readwrite_pcc_attach_fid(const char *mntpath, - const struct lu_fid *fid, - __u32 rwid) +static int llapi_pcc_attach_rw_fid(const char *mntpath, + const struct lu_fid *fid, __u32 rwid) { int rc; int fd; @@ -184,15 +184,15 @@ static int llapi_readwrite_pcc_attach_fid(const char *mntpath, return rc; } - rc = llapi_readwrite_pcc_attach_fd(fd, rwid); + rc = llapi_pcc_attach_rw_fd(fd, rwid); close(fd); return rc; } -static int llapi_readonly_pcc_attach_fid(const char *mntpath, - const struct lu_fid *fid, - __u32 roid) +static int llapi_pcc_attach_ro_fid(const char *mntpath, + const struct lu_fid *fid, __u32 roid, + enum lu_pcc_type type) { int rc; int fd; @@ -206,7 +206,7 @@ static int llapi_readonly_pcc_attach_fid(const char *mntpath, return rc; } - rc = llapi_readonly_pcc_attach_fd(fd, roid); + rc = llapi_pcc_attach_ro_fd(fd, roid, type); close(fd); return rc; @@ -220,10 +220,10 @@ int llapi_pcc_attach_fid(const char *mntpath, const struct lu_fid *fid, switch (type & LU_PCC_TYPE_MASK) { case LU_PCC_READWRITE: - rc = llapi_readwrite_pcc_attach_fid(mntpath, fid, id); + rc = llapi_pcc_attach_rw_fid(mntpath, fid, id); break; default: - rc = llapi_readonly_pcc_attach_fid(mntpath, fid, id); + rc = llapi_pcc_attach_ro_fid(mntpath, fid, id, type); break; } return rc;