EX-6373 pcc: asynchronous PCCRO attach command support

Currently PCCRO attach via the command "lfs pcc attach" will block
during the data copying.
There is a requirement that this command can also do data copy
asynchronously. Thus we add an option "--async|-A" to the command
which will not block while the file data is being fetched.

Add sanity-pcc/test_{103, 104} to verify that it works correctly.

Change-Id: I6f31190c8b9e9b9876b34f8e484c6c8b7f16b6db
Signed-off-by: Qian Yingjin <qian@ddn.com>
Reviewed-on: https://review.whamcloud.com/c/ex/lustre-release/+/49133
Tested-by: jenkins <devops@whamcloud.com>
Tested-by: Maloo <maloo@whamcloud.com>
Reviewed-by: Feng Lei <flei@whamcloud.com>
Reviewed-by: Andreas Dilger <adilger@whamcloud.com>

diff --git a/lustre/doc/llapi_pcc_attach.3 b/lustre/doc/llapi_pcc_attach.3
index a283559..f6741db 100644 (file)
--- a/lustre/doc/llapi_pcc_attach.3
+++ b/lustre/doc/llapi_pcc_attach.3
@@ -40,9 +40,11 @@ argument, which is a
 data structure, which contains the following values:
 .nf
 .LP
-       LU_PCC_NONE
-       LU_PCC_READWRITE
-       LU_PCC_READONLY
+       LU_PCC_NONE      = 0x0,
+       LU_PCC_READWRITE = 0x01,
+       LU_PCC_READONLY  = 0x02,
+       LU_PCC_TYPE_MASK = LU_PCC_READWRITE | LU_PCC_READONLY,
+       LU_PCC_FL_ASYNC  = 0x10,
 .fi
 .TP
 LU_PCC_NONE
@@ -53,6 +55,12 @@ means RW-PCC mode.
 .TP
 LU_PCC_READONLY
 means RO-PCC mode.
+.TP
+LU_PCC_TYPE_MASK
+is the bits mask for PCC modes.
+.TP
+LU_PCC_FL_ASYNC
+indicates to do data copying in asynchronous way during attach.
 .SH RETURN VALUES
 .PP
 .B llapi_pcc_attach()

diff --git a/lustre/include/uapi/linux/lustre/lustre_user.h b/lustre/include/uapi/linux/lustre/lustre_user.h
index 63460d7..cab8535 100644 (file)
--- a/lustre/include/uapi/linux/lustre/lustre_user.h
+++ b/lustre/include/uapi/linux/lustre/lustre_user.h
@@ -2724,6 +2724,7 @@ enum lu_pcc_type {
        LU_PCC_READWRITE        = 0x01,
        LU_PCC_READONLY         = 0x02,
        LU_PCC_TYPE_MASK        = LU_PCC_READWRITE | LU_PCC_READONLY,
+       LU_PCC_FL_ASYNC         = 0x10,
        LU_PCC_MAX
 };
 
@@ -2807,8 +2808,8 @@ static inline enum hsmtool_type hsmtool_string2type(const char *str)
 }
 
 struct lu_pcc_attach {
-       __u32 pcca_type; /* PCC type */
-       __u32 pcca_id; /* Attach ID */
+       __u32 pcca_type;/* PCC type */
+       __u32 pcca_id;/* Attach ID */
 };
 
 enum lu_pcc_detach_flags {

diff --git a/lustre/llite/pcc.c b/lustre/llite/pcc.c
index 79e250e..a9fa394 100644 (file)
--- a/lustre/llite/pcc.c
+++ b/lustre/llite/pcc.c
@@ -4346,17 +4346,32 @@ static int pcc_readonly_attach_sync(struct file *file, struct inode *inode,
 int pcc_ioctl_attach(struct file *file, struct inode *inode,
                     struct lu_pcc_attach *attach)
 {
+       struct pcc_super *super = ll_i2pccs(inode);
        int rc = 0;
 
        ENTRY;
 
-       switch (attach->pcca_type) {
+       switch (attach->pcca_type & LU_PCC_TYPE_MASK) {
        case LU_PCC_READWRITE:
                rc = -ENOTSUPP;
                break;
        case LU_PCC_READONLY:
-               rc = pcc_readonly_attach_sync(file, inode,
-                                             attach->pcca_id, false);
+               if (attach->pcca_type & LU_PCC_FL_ASYNC &&
+                   atomic_read(&super->pccs_attaches_queued) <
+                   super->pccs_maximum_queued_attaches) {
+                       rc = pcc_readonly_attach_async(file, inode,
+                                                      attach->pcca_id);
+                       /*
+                        * Ignore EEXIST error if the file has already attached.
+                        * Ignore EINPROGRESS error if the file is being
+                        * attached, i.e. copy data from OSTs into PCC.
+                        */
+                       if (rc == -EEXIST || rc == -EINPROGRESS)
+                               rc = 0;
+               } else {
+                       rc = pcc_readonly_attach_sync(file, inode,
+                                                     attach->pcca_id, false);
+               }
                break;
        default:
                rc = -EINVAL;

diff --git a/lustre/tests/sanity-pcc.sh b/lustre/tests/sanity-pcc.sh
index f2c3a59..3a53c8a 100644 (file)
--- a/lustre/tests/sanity-pcc.sh
+++ b/lustre/tests/sanity-pcc.sh
@@ -4740,6 +4740,74 @@ test_102() {
 }
 run_test 102 "PCC-RO should not hange for io_uring I/O engine"
 
+test_103() {
+       local loopfile="$TMP/$tfile"
+       local mntpt="/mnt/pcc.$tdir"
+       local hsm_root="$mntpt/$tdir"
+       local file=$DIR/$tfile
+
+       $LCTL get_param -n mdc.*.connect_flags | grep -q pcc_ro ||
+               skip "Server does not support PCC-RO"
+
+       setup_loopdev client $loopfile $mntpt 300
+       mkdir $hsm_root || error "mkdir $hsm_root failed"
+       setup_pcc_mapping client \
+               "projid={100}\ roid=$HSM_ARCHIVE_NUMBER\ pccro=1"
+
+       $LFS setstripe -c -1 $file || error "failed to setstripe $file"
+       fallocate -l 200M $file || error "fallocate $file failed"
+       stack_trap "rm -f $file"
+       cancel_lru_locks osc
+       $LFS pcc attach -i $HSM_ARCHIVE_NUMBER -A $file ||
+               error "failed to PCCRO attach -A $file"
+       $LFS pcc state $file | grep -q 'flags: attaching' ||
+               error "Not do asynchronous PCCRO attach for $file"
+       wait_readonly_attach_fini $file client
+
+       $LFS pcc detach $file || error "failed to PCC detach $file"
+       $LFS pcc attach -A $file || error "failed to PCCRO attach -A $file"
+       $LFS pcc state $file | grep -q 'flags: attaching' ||
+               error "Not do asynchronous PCCRO attach for $file"
+       wait_readonly_attach_fini $file client
+}
+run_test 103 "Test asynchronous manual attach for PCCRO"
+
+test_104() {
+       local loopfile="$TMP/$tfile"
+       local mntpt="/mnt/pcc.$tdir"
+       local hsm_root="$mntpt/$tdir"
+       local file=$DIR/$tfile
+
+       $LCTL get_param -n mdc.*.connect_flags | grep -q pcc_ro ||
+               skip "Server does not support PCC-RO"
+
+       setup_loopdev client $loopfile $mntpt 600
+       mkdir $hsm_root || error "mkdir $hsm_root failed"
+       setup_pcc_mapping client \
+               "projid={0}\ roid=$HSM_ARCHIVE_NUMBER\ pccro=1"
+
+       $LFS setstripe -c -1 $file || error "failed to setstripe $file"
+       fallocate -l 500M $file || error "fallocate $file failed"
+       stack_trap "rm -f $file"
+       cancel_lru_locks osc
+       # $LFS pcc attach will call realpath():
+       # In rhel8, it uses stat() to tell about the file at the end of any
+       # chain of symlinks. And it will obtain the size on the client.
+       # In Ubuntu2204, it uses readlink() to obtain the path name.
+       # As fallocate() does not update the size on MDT (LU-16334), thus we
+       # use stat to obtain the size on the client explictly. Otherwise, the
+       # known size on the client is zero, and lower than pcc_async_threshold,
+       # the client would do attach at open() in the synchronous way.
+       stat $file || error "failed to stat $file"
+       $LFS pcc attach -i $HSM_ARCHIVE_NUMBER -A $file ||
+               error "failed to PCCRO attach -A $file"
+       $LFS pcc state $file
+       $LFS pcc state $file | grep -q 'flags: attaching' ||
+               error "Not do asynchronous PCCRO attach for $file"
+       wait_readonly_attach_fini $file client
+}
+run_test 104 "Test asynchronous manual attach and auto open attach for PCCRO"
+
 test_200() {
        local loopfile="$TMP/$tfile"
        local mntpt="/mnt/pcc.$tdir"

diff --git a/lustre/utils/lfs.c b/lustre/utils/lfs.c
index c7df3f0..53bb376 100644 (file)
--- a/lustre/utils/lfs.c
+++ b/lustre/utils/lfs.c
@@ -376,19 +376,21 @@ command_t mirror_cmdlist[] = {
 command_t pcc_cmdlist[] = {
        { .pc_name = "attach", .pc_func = lfs_pcc_attach,
          .pc_help = "Copy given files to PCC, the local cache storage.\n"
-               "usage: lfs pcc attach [--id|-i ID] [--readonly|-r] [--write|-w] [-p|--pin] <file> ...\n"
+               "usage: lfs pcc attach [--id|-i ID] [--readonly|-r] [--write|-w] [--pin|-p] [--async|-A] <file> ...\n"
                "\t-i: archive ID for PCC\n"
                "\t-r: readonly attach\n"
                "\t-w: writeable attach\n"
-               "\t-p: pin and attach the file\n" },
+               "\t-p: pin and attach the file\n"
+               "\t-A: copy data asynchronously during readonly attach\n" },
        { .pc_name = "attach_fid", .pc_func = lfs_pcc_attach_fid,
          .pc_help = "Copy given files to PCC by FID(s), the local cache storage.\n"
-               "usage: lfs pcc attach_fid [--id|-i ID] {--mnt|-m MOUNTPATH} [--readonly|-r] [--write|-w] [-p|--pin] <FID> ...\n"
+               "usage: lfs pcc attach_fid [--id|-i ID] {--mnt|-m MOUNTPATH} [--readonly|-r] [--write|-w] [--pin|-p] [--async|-A] <FID> ...\n"
                "\t-i: archive ID for PCC\n"
                "\t-m: Lustre mount point\n"
                "\t-r: readonly attach\n"
                "\t-w: writeable attach\n"
-               "\t-p: pin and attach the file\n" },
+               "\t-p: pin and attach the file\n"
+               "\t-A: copy data asynchronously during readonly attach\n" },
        { .pc_name = "state", .pc_func = lfs_pcc_state,
          .pc_help = "Display the PCC state for given files.\n"
                "usage: lfs pcc state <FILE> ...\n" },
@@ -12156,6 +12158,7 @@ static int lfs_pcc_attach(int argc, char **argv)
        { .val = 'r',   .name = "readonly",     .has_arg = no_argument },
        { .val = 'w',   .name = "write",        .has_arg = no_argument },
        { .val = 'p',   .name = "pin",          .has_arg = no_argument },
+       { .val = 'A',   .name = "async",        .has_arg = no_argument },
        { .name = NULL } };
        int c;
        int rc = 0;
@@ -12167,7 +12170,7 @@ static int lfs_pcc_attach(int argc, char **argv)
        bool pin = false;
 
        optind = 0;
-       while ((c = getopt_long(argc, argv, "i:prw",
+       while ((c = getopt_long(argc, argv, "i:prwA",
                                long_opts, NULL)) != -1) {
                switch (c) {
                case 'i':
@@ -12182,14 +12185,19 @@ static int lfs_pcc_attach(int argc, char **argv)
                        }
                        break;
                case 'r':
-                       type = LU_PCC_READONLY;
+                       type |= LU_PCC_READONLY;
+                       type &= ~LU_PCC_READWRITE;
                        break;
                case 'w':
-                       type = LU_PCC_READWRITE;
+                       type |= LU_PCC_READWRITE;
+                       type &= ~LU_PCC_READONLY;
                        break;
                case 'p':
                        pin = true;
                        break;
+               case 'A':
+                       type |= LU_PCC_FL_ASYNC;
+                       break;
                case '?':
                        return CMD_HELP;
                default:
@@ -12205,6 +12213,13 @@ static int lfs_pcc_attach(int argc, char **argv)
                return CMD_HELP;
        }
 
+       if ((type & LU_PCC_READWRITE) && (type & LU_PCC_FL_ASYNC)) {
+               fprintf(stderr,
+                       "%s: PCCRW does not support asynchronous attach\n",
+                       argv[0]);
+               return CMD_HELP;
+       }
+
        while (optind < argc) {
                int rc2;
 
@@ -12259,8 +12274,9 @@ static int lfs_pcc_attach_fid(int argc, char **argv)
        { .val = 'w',   .name = "write",        .has_arg = no_argument },
        { .val = 'm',   .name = "mnt",          .has_arg = required_argument },
        { .val = 'p',   .name = "pin",          .has_arg = no_argument },
+       { .val = 'A',   .name = "async",        .has_arg = no_argument },
        { .name = NULL } };
-       char short_opts[] = "i:m:prw";
+       char short_opts[] = "i:m:prwA";
        int c;
        int rc = 0;
        __u32 attach_id = 0;
@@ -12287,10 +12303,12 @@ static int lfs_pcc_attach_fid(int argc, char **argv)
                        }
                        break;
                case 'r':
-                       type = LU_PCC_READONLY;
+                       type |= LU_PCC_READONLY;
+                       type &= ~LU_PCC_READWRITE;
                        break;
                case 'w':
-                       type = LU_PCC_READWRITE;
+                       type |= LU_PCC_READWRITE;
+                       type &= ~LU_PCC_READONLY;
                        break;
                case 'm':
                        mntpath = optarg;
@@ -12298,6 +12316,9 @@ static int lfs_pcc_attach_fid(int argc, char **argv)
                case 'p':
                        pin = true;
                        break;
+               case 'A':
+                       type |= LU_PCC_FL_ASYNC;
+                       break;
                case '?':
                        return CMD_HELP;
                default:
@@ -12316,6 +12337,13 @@ static int lfs_pcc_attach_fid(int argc, char **argv)
                mntpath = argv[optind++];
        }
 
+       if((type & LU_PCC_READWRITE) && (type & LU_PCC_FL_ASYNC)) {
+               fprintf(stderr,
+                       "%s: PCCRW does not support asynchronous attach\n",
+                       argv[0]);
+               return CMD_HELP;
+       }
+
        if (argc <= optind) {
                fprintf(stderr, "%s: must specify one or more fids\n", argv[0]);
                return CMD_HELP;

diff --git a/lustre/utils/liblustreapi_pcc.c b/lustre/utils/liblustreapi_pcc.c
index 2a90ee3..c95218f 100644 (file)
--- a/lustre/utils/liblustreapi_pcc.c
+++ b/lustre/utils/liblustreapi_pcc.c
@@ -50,7 +50,7 @@
  * Fetch and attach a file to readwrite PCC.
  *
  */
-static int llapi_readwrite_pcc_attach_fd(int fd, __u32 archive_id)
+static int llapi_pcc_attach_rw_fd(int fd, __u32 archive_id)
 {
        int rc;
        struct ll_ioc_lease *data;
@@ -87,7 +87,7 @@ static int llapi_readwrite_pcc_attach_fd(int fd, __u32 archive_id)
        return rc;
 }
 
-static int llapi_readwrite_pcc_attach(const char *path, __u32 archive_id)
+static int llapi_pcc_attach_rw(const char *path, __u32 archive_id)
 {
        int fd;
        int rc;
@@ -100,19 +100,19 @@ static int llapi_readwrite_pcc_attach(const char *path, __u32 archive_id)
                return rc;
        }
 
-       rc = llapi_readwrite_pcc_attach_fd(fd, archive_id);
+       rc = llapi_pcc_attach_rw_fd(fd, archive_id);
 
        close(fd);
        return rc;
 }
 
-static int llapi_readonly_pcc_attach_fd(int fd, __u32 roid)
+static int llapi_pcc_attach_ro_fd(int fd, __u32 roid, enum lu_pcc_type type)
 {
        struct lu_pcc_attach attach;
        int rc;
 
        attach.pcca_id = roid;
-       attach.pcca_type = LU_PCC_READONLY;
+       attach.pcca_type = type;
        rc = ioctl(fd, LL_IOC_PCC_ATTACH, &attach);
        if (rc) {
                rc = -errno;
@@ -124,7 +124,8 @@ static int llapi_readonly_pcc_attach_fd(int fd, __u32 roid)
        return rc;
 }
 
-static int llapi_readonly_pcc_attach(const char *path, __u32 roid)
+static int llapi_pcc_attach_ro(const char *path, __u32 roid,
+                              enum lu_pcc_type type)
 {
        int fd;
        int rc;
@@ -144,7 +145,7 @@ static int llapi_readonly_pcc_attach(const char *path, __u32 roid)
                return rc;
        }
 
-       rc = llapi_readonly_pcc_attach_fd(fd, roid);
+       rc = llapi_pcc_attach_ro_fd(fd, roid, type);
 
        close(fd);
        return rc;
@@ -156,10 +157,10 @@ int llapi_pcc_attach(const char *path, __u32 id, enum lu_pcc_type type)
 
        switch (type & LU_PCC_TYPE_MASK) {
        case LU_PCC_READWRITE:
-               rc = llapi_readwrite_pcc_attach(path, id);
+               rc = llapi_pcc_attach_rw(path, id);
                break;
        case LU_PCC_READONLY:
-               rc = llapi_readonly_pcc_attach(path, id);
+               rc = llapi_pcc_attach_ro(path, id, type);
                break;
        default:
                rc = -EINVAL;
@@ -168,9 +169,8 @@ int llapi_pcc_attach(const char *path, __u32 id, enum lu_pcc_type type)
        return rc;
 }
 
-static int llapi_readwrite_pcc_attach_fid(const char *mntpath,
-                                         const struct lu_fid *fid,
-                                         __u32 rwid)
+static int llapi_pcc_attach_rw_fid(const char *mntpath,
+                                  const struct lu_fid *fid, __u32 rwid)
 {
        int rc;
        int fd;
@@ -184,15 +184,15 @@ static int llapi_readwrite_pcc_attach_fid(const char *mntpath,
                return rc;
        }
 
-       rc = llapi_readwrite_pcc_attach_fd(fd, rwid);
+       rc = llapi_pcc_attach_rw_fd(fd, rwid);
 
        close(fd);
        return rc;
 }
 
-static int llapi_readonly_pcc_attach_fid(const char *mntpath,
-                                        const struct lu_fid *fid,
-                                        __u32 roid)
+static int llapi_pcc_attach_ro_fid(const char *mntpath,
+                                  const struct lu_fid *fid, __u32 roid,
+                                  enum lu_pcc_type type)
 {
        int rc;
        int fd;
@@ -206,7 +206,7 @@ static int llapi_readonly_pcc_attach_fid(const char *mntpath,
                return rc;
        }
 
-       rc = llapi_readonly_pcc_attach_fd(fd, roid);
+       rc = llapi_pcc_attach_ro_fd(fd, roid, type);
 
        close(fd);
        return rc;
@@ -220,10 +220,10 @@ int llapi_pcc_attach_fid(const char *mntpath, const struct lu_fid *fid,
 
        switch (type & LU_PCC_TYPE_MASK) {
        case LU_PCC_READWRITE:
-               rc = llapi_readwrite_pcc_attach_fid(mntpath, fid, id);
+               rc = llapi_pcc_attach_rw_fid(mntpath, fid, id);
                break;
        default:
-               rc = llapi_readonly_pcc_attach_fid(mntpath, fid, id);
+               rc = llapi_pcc_attach_ro_fid(mntpath, fid, id, type);
                break;
        }
        return rc;