From 2dadefb4148f753dd93ee1dbebb3aac49bda2f8d Mon Sep 17 00:00:00 2001 From: Qian Yingjin Date: Thu, 13 Dec 2018 10:41:14 +0800 Subject: [PATCH] LU-10092 pcc: change detach behavior and add keep option After introduce the feature of auto-attach at open, when the PCC cached file is detach by "pcc detach" command, it will be attached automatically at the next open. This may be not what the user wants. To solve this problem, we change the defualt detach behavior and add an option "--keep|-k" for the detach of RW-PCC. The manual "lfs pcc detach" command will detach the file from PCC permanently. And it will also remove the PCC copy by default. When the file is detached with "keep" option, it only unmaps the relationship between the file inode and PCC copy, but keep the PCC copy. The file is allowed to be attached automatically at the next open when the file is still valid in cache. Note here that currently auto detach caused by inode reclaim or revocation of the layout lock would not delete the PCC copy too. Test-Parameters: clientcount=3 testlist=sanity-pcc,sanity-pcc,sanity-pcc Signed-off-by: Qian Yingjin Change-Id: I010df54177ae4cfeddcc0a9982c1aee58ee683de Reviewed-on: https://review.whamcloud.com/33844 Tested-by: Jenkins Reviewed-by: Patrick Farrell Reviewed-by: Li Xi Tested-by: Maloo Reviewed-by: Oleg Drokin --- lustre/doc/lfs-pcc-detach.1 | 44 +++++++++ lustre/doc/lfs-pcc.1 | 30 +++---- lustre/include/lustre/lustreapi.h | 10 ++- lustre/include/uapi/linux/lustre/lustre_user.h | 16 +++- lustre/llite/dir.c | 6 +- lustre/llite/file.c | 23 ++++- lustre/llite/pcc.c | 74 +++++++++++++-- lustre/llite/pcc.h | 2 +- lustre/tests/sanity-pcc.sh | 119 +++++++++++++++++++------ lustre/utils/lfs.c | 56 +++++++++--- lustre/utils/liblustreapi_pcc.c | 29 +++--- 11 files changed, 321 insertions(+), 88 deletions(-) create mode 100644 lustre/doc/lfs-pcc-detach.1 diff --git a/lustre/doc/lfs-pcc-detach.1 b/lustre/doc/lfs-pcc-detach.1 new file mode 100644 index 0000000..7ac48c9 --- /dev/null +++ b/lustre/doc/lfs-pcc-detach.1 @@ -0,0 +1,44 @@ +.TH LFS-PCC-DETACH 1 2019-04-15 "Lustre" "Lustre Utilities" +.SH NAME +lfs pcc detach|detach_fid \- Detach given files from PCC +.SH SYNOPSIS +.B lfs pcc detach [\fB--keep\fR|\fB-k\fR] <\fIfile \fR...> +.br +.B lfs pcc detach_fid [\fB--keep\fR|\fB-k\fR] <\fImntpath\fR> <\fIfid \fR...> +.SH DESCRIPTION +.TP +.B lfs pcc detach [\fB--keep\fR|\fB-k\fR] <\fIfile \fR...> +Detach given files from the persistent client cache. +.TP +.B lfs pcc detach_fid [\fB--keep\fR|\fB-k\fR] <\fImntpath\fR> <\fIfid \fR...> +Detach files from the persistent client cache by FID(s). +.SH OPTIONS +.TP +.B --keep | -k +By default, the detach command will detach the file from PCC permanently and +remove the PCC copy after detach. This option will only detach the file, but +keep the PCC copy in cache. It allows the detaching file to be attached +automatically at next open if the cached copy of the file is still valid. +.SH EXAMPLES +.TP +.B $ lfs pcc detach /mnt/lustre/test +Detach the file permanently from PCC. The cached file on PCC will be removed +after detach. IO to the file will come to Lustre OSTs after this command. +.TP +.B $ lfs pcc detach_fid /mnt/lustre 0x200000401:0x1:0x0 +Detach the file referenced by FID "0x200000401:0x1:0x0" from PCC permanently, and +the cached file on PCC will be removed after detach. +.TP +.B $ lfs pcc detach -k /mnt/lustre/test +Detach the file "/mnt/lustre/test" from PCC. The client will try to attach +this file again at the next open if the cached copy is still valid. +.TP +.B $ lfs pcc detach_fid -k /mnt/lustre 0x200000401:0x1:0x0 +Detach the file referenced by FID "0x200000401:0x1:0x0" from PCC. The client +will try to attach this file again at the next open if the cached copy is still +valid. +.SH SEE ALSO +.BR lfs (1), +.BR lfs-hsm (1), +.BR lfs-pcc (1), +.BR lctl-pcc (8) diff --git a/lustre/doc/lfs-pcc.1 b/lustre/doc/lfs-pcc.1 index 56a760f..886a96d 100644 --- a/lustre/doc/lfs-pcc.1 +++ b/lustre/doc/lfs-pcc.1 @@ -1,29 +1,22 @@ .TH LFS-PCC 1 2019-04-15 "Lustre" "Lustre Utilities" .SH NAME -lfs pcc commands used to interact with PCC features. +lfs pcc commands used to interact with the Persistent Client Cache (PCC). .SH SYNOPSIS .B lfs pcc attach <\fB--id\fR|\fB-i\fR \fINUM\fR> <\fIfile \fR...> .br .B lfs pcc attach <\fB--id\fR|\fB-i\fR \fINUM\fR> <\fB--mnt\fR|\fB-m\fR \fImntpath\fR> <\fIfid \fR...> .br -.B lfs pcc detach <\fIfile \fR...> -.br -.B lfs pcc detach_fid <\fImntpath\fR> <\fIfid \fR...> -.br .B lfs pcc state <\fIfile \fR...> .SH DESCRIPTION .TP .B lfs pcc attach <\fB--id\fR|\fB-i\fR \fINUM\fR> <\fIfile \fR...> -Attach given files on the persistent client cache. +Attach given files on the persistent client cache. Use +.B lfs pcc detach +to remove the cached files from PCC either manually, or through automatic +mechanisms for the purpose of the cache space management. .TP .B lfs pcc attach <\fB--id\fR|\fB-i\fR \fINUM\fR> <\fB--mnt\fR|\fB-m\fR \fImntpath\fR> <\fIfid \fR...> -Attach given fils into the persistent client cache by FID(s). -.TP -.B lfs pcc detach <\fIfile \fR...> -Detach given files from the persistent client cache. -.TP -.B lfs pcc detach_fid <\fImntpath\fR> <\fIfid \fR...> -Detach files from the persistent client cache by FID(s). +Attach given files into the persistent client cache by FID(s). .TP .B lfs pcc state <\fIfile \fR...> Display the PCC state for given files. @@ -78,11 +71,11 @@ any I/O to the Lustre file will direct to the RW-PCC copy. .B $ lfs pcc attach_fid -i 1 -m /mnt/lustre 0x200000401:0x1:0x0 Attach an existing file referenced by FID "0x200000401:0x1:0x0" into PCC. .TP -.B $ lfs pcc detach /mnt/lustre/file -Detach the file from RW-PCC, IO to the file will come to Lustre after this -command. -.B $ lfs pcc detach_fid /mnt/lustre 0x200000401:0x1:0x0 -Detach the file referenced by FID "0x200000401:0x1:0x0" from PCC. +.B $ lfs pcc state /mnt/lustre/file +.br +file: /mnt/lustre/file, type: readwrite, PCC file: /mnt/pcc/0004/0000/0bd1/0000/0002/0000/0x200000bd1:0x4:0x0, user number: 1, flags: 6 +.br +Display the PCC state of the file "/mnt/lustre/file". .TP .B $ lfs pcc state /mnt/lustre/file .br @@ -93,4 +86,5 @@ Display the PCC state of the file "/mnt/lustre/file". .SH SEE ALSO .BR lfs (1), .BR lfs-hsm (1), +.BR lfs-pcc-detach (1), .BR lctl-pcc (8) diff --git a/lustre/include/lustre/lustreapi.h b/lustre/include/lustre/lustreapi.h index 08cfba0..420b606 100644 --- a/lustre/include/lustre/lustreapi.h +++ b/lustre/include/lustre/lustreapi.h @@ -525,10 +525,12 @@ int llapi_pcc_attach_fid(const char *mntpath, const struct lu_fid *fid, __u32 id, enum lu_pcc_type type); int llapi_pcc_attach_fid_str(const char *mntpath, const char *fidstr, __u32 id, enum lu_pcc_type type); -int llapi_pcc_detach_fd(int fd); -int llapi_pcc_detach_fid(const char *mntpath, const struct lu_fid *fid); -int llapi_pcc_detach_fid_str(const char *mntpath, const char *fidstr); -int llapi_pcc_detach_file(const char *path); +int llapi_pcc_detach_fd(int fd, __u32 option); +int llapi_pcc_detach_fid(const char *mntpath, const struct lu_fid *fid, + __u32 option); +int llapi_pcc_detach_fid_str(const char *mntpath, const char *fidstr, + __u32 option); +int llapi_pcc_detach_file(const char *path, __u32 option); int llapi_pcc_state_get_fd(int fd, struct lu_pcc_state *state); int llapi_pcc_state_get(const char *path, struct lu_pcc_state *state); int llapi_pccdev_set(const char *mntpath, const char *cmd); diff --git a/lustre/include/uapi/linux/lustre/lustre_user.h b/lustre/include/uapi/linux/lustre/lustre_user.h index 710aca8..7d321f8 100644 --- a/lustre/include/uapi/linux/lustre/lustre_user.h +++ b/lustre/include/uapi/linux/lustre/lustre_user.h @@ -482,8 +482,8 @@ struct ll_ioc_lease_id { #define LL_IOC_LADVISE _IOR('f', 250, struct llapi_lu_ladvise) #define LL_IOC_HEAT_GET _IOWR('f', 251, struct lu_heat) #define LL_IOC_HEAT_SET _IOW('f', 251, __u64) -#define LL_IOC_PCC_DETACH _IO('f', 252) -#define LL_IOC_PCC_DETACH_BY_FID _IOW('f', 252, struct lu_pcc_detach) +#define LL_IOC_PCC_DETACH _IOW('f', 252, struct lu_pcc_detach) +#define LL_IOC_PCC_DETACH_BY_FID _IOW('f', 252, struct lu_pcc_detach_fid) #define LL_IOC_PCC_STATE _IOR('f', 252, struct lu_pcc_state) #ifndef FS_IOC_FSGETXATTR @@ -2324,9 +2324,19 @@ struct lu_pcc_attach { __u32 pcca_id; /* archive ID for readwrite, group ID for readonly */ }; -struct lu_pcc_detach { +enum lu_pcc_detach_opts { + PCC_DETACH_OPT_NONE = 0, /* Detach only, keep the PCC copy */ + PCC_DETACH_OPT_UNCACHE, /* Remove the cached file after detach */ +}; + +struct lu_pcc_detach_fid { /* fid of the file to detach */ struct lu_fid pccd_fid; + __u32 pccd_opt; +}; + +struct lu_pcc_detach { + __u32 pccd_opt; }; enum lu_pcc_state_flags { diff --git a/lustre/llite/dir.c b/lustre/llite/dir.c index b5cbb7e..ab56ccb 100644 --- a/lustre/llite/dir.c +++ b/lustre/llite/dir.c @@ -1984,7 +1984,7 @@ migrate_free: case LL_IOC_FSSETXATTR: RETURN(ll_ioctl_fssetxattr(inode, cmd, arg)); case LL_IOC_PCC_DETACH_BY_FID: { - struct lu_pcc_detach *detach; + struct lu_pcc_detach_fid *detach; struct lu_fid *fid; struct inode *inode2; unsigned long ino; @@ -1994,7 +1994,7 @@ migrate_free: RETURN(-ENOMEM); if (copy_from_user(detach, - (const struct lu_pcc_detach __user *)arg, + (const struct lu_pcc_detach_fid __user *)arg, sizeof(*detach))) GOTO(out_detach, rc = -EFAULT); @@ -2013,7 +2013,7 @@ migrate_free: if (!inode_owner_or_capable(inode2)) GOTO(out_iput, rc = -EPERM); - rc = pcc_ioctl_detach(inode2); + rc = pcc_ioctl_detach(inode2, detach->pccd_opt); out_iput: iput(inode2); out_detach: diff --git a/lustre/llite/file.c b/lustre/llite/file.c index 45248a3..61b539d 100644 --- a/lustre/llite/file.c +++ b/lustre/llite/file.c @@ -3846,14 +3846,29 @@ out_ladvise: rc = ll_heat_set(inode, flags); RETURN(rc); } - case LL_IOC_PCC_DETACH: + case LL_IOC_PCC_DETACH: { + struct lu_pcc_detach *detach; + + OBD_ALLOC_PTR(detach); + if (detach == NULL) + RETURN(-ENOMEM); + + if (copy_from_user(detach, + (const struct lu_pcc_detach __user *)arg, + sizeof(*detach))) + GOTO(out_detach_free, rc = -EFAULT); + if (!S_ISREG(inode->i_mode)) - RETURN(-EINVAL); + GOTO(out_detach_free, rc = -EINVAL); if (!inode_owner_or_capable(inode)) - RETURN(-EPERM); + GOTO(out_detach_free, rc = -EPERM); - RETURN(pcc_ioctl_detach(inode)); + rc = pcc_ioctl_detach(inode, detach->pccd_opt); +out_detach_free: + OBD_FREE_PTR(detach); + RETURN(rc); + } case LL_IOC_PCC_STATE: { struct lu_pcc_state __user *ustate = (struct lu_pcc_state __user *)arg; diff --git a/lustre/llite/pcc.c b/lustre/llite/pcc.c index 3a1f132..fcebed4 100644 --- a/lustre/llite/pcc.c +++ b/lustre/llite/pcc.c @@ -974,6 +974,7 @@ static void pcc_inode_init(struct pcc_inode *pcci, struct ll_inode_info *lli) { pcci->pcci_lli = lli; lli->lli_pcc_inode = pcci; + lli->lli_pcc_state = PCC_STATE_FL_NONE; atomic_set(&pcci->pcci_refcount, 0); pcci->pcci_type = LU_PCC_NONE; pcci->pcci_layout_gen = CL_LAYOUT_GEN_NONE; @@ -1817,8 +1818,9 @@ int pcc_page_mkwrite(struct vm_area_struct *vma, struct vm_fault *vmf, CDEBUG(D_MMAP, "%s: PCC backend fs not support ->page_mkwrite()\n", ll_i2sbi(inode)->ll_fsname); - pcc_ioctl_detach(inode); + pcc_ioctl_detach(inode, PCC_DETACH_OPT_NONE); up_read(&mm->mmap_sem); + *cached = true; RETURN(VM_FAULT_RETRY | VM_FAULT_NOPAGE); } /* Pause to allow for a race with concurrent detach */ @@ -1857,7 +1859,7 @@ int pcc_page_mkwrite(struct vm_area_struct *vma, struct vm_fault *vmf, */ if (OBD_FAIL_CHECK(OBD_FAIL_LLITE_PCC_DETACH_MKWRITE)) { pcc_io_fini(inode); - pcc_ioctl_detach(inode); + pcc_ioctl_detach(inode, PCC_DETACH_OPT_NONE); up_read(&mm->mmap_sem); RETURN(VM_FAULT_RETRY | VM_FAULT_NOPAGE); } @@ -1930,6 +1932,8 @@ void pcc_layout_invalidate(struct inode *inode) { struct pcc_inode *pcci; + ENTRY; + pcc_inode_lock(inode); pcci = ll_i2pcci(inode); if (pcci && pcc_inode_has_layout(pcci)) { @@ -1942,6 +1946,8 @@ void pcc_layout_invalidate(struct inode *inode) pcc_inode_put(pcci); } pcc_inode_unlock(inode); + + EXIT; } static int pcc_inode_remove(struct inode *inode, struct dentry *pcc_dentry) @@ -2368,10 +2374,53 @@ out_unlock: RETURN(rc); } -int pcc_ioctl_detach(struct inode *inode) +static int pcc_hsm_remove(struct inode *inode) +{ + struct hsm_user_request *hur; + __u32 gen; + int len; + int rc; + + ENTRY; + + rc = ll_layout_restore(inode, 0, OBD_OBJECT_EOF); + if (rc) { + CDEBUG(D_CACHE, DFID" RESTORE failure: %d\n", + PFID(&ll_i2info(inode)->lli_fid), rc); + RETURN(rc); + } + + ll_layout_refresh(inode, &gen); + + len = sizeof(struct hsm_user_request) + + sizeof(struct hsm_user_item); + OBD_ALLOC(hur, len); + if (hur == NULL) + RETURN(-ENOMEM); + + hur->hur_request.hr_action = HUA_REMOVE; + hur->hur_request.hr_archive_id = 0; + hur->hur_request.hr_flags = 0; + memcpy(&hur->hur_user_item[0].hui_fid, &ll_i2info(inode)->lli_fid, + sizeof(hur->hur_user_item[0].hui_fid)); + hur->hur_user_item[0].hui_extent.offset = 0; + hur->hur_user_item[0].hui_extent.length = OBD_OBJECT_EOF; + hur->hur_request.hr_itemcount = 1; + rc = obd_iocontrol(LL_IOC_HSM_REQUEST, ll_i2sbi(inode)->ll_md_exp, + len, hur, NULL); + if (rc) + CDEBUG(D_CACHE, DFID" HSM REMOVE failure: %d\n", + PFID(&ll_i2info(inode)->lli_fid), rc); + + OBD_FREE(hur, len); + RETURN(rc); +} + +int pcc_ioctl_detach(struct inode *inode, __u32 opt) { struct ll_inode_info *lli = ll_i2info(inode); struct pcc_inode *pcci; + bool hsm_remove = false; int rc = 0; ENTRY; @@ -2382,11 +2431,26 @@ int pcc_ioctl_detach(struct inode *inode) !pcc_inode_has_layout(pcci)) GOTO(out_unlock, rc = 0); - __pcc_layout_invalidate(pcci); - pcc_inode_put(pcci); + LASSERT(atomic_read(&pcci->pcci_refcount) > 0); + + if (pcci->pcci_type == LU_PCC_READWRITE) { + if (opt == PCC_DETACH_OPT_UNCACHE) + hsm_remove = true; + + __pcc_layout_invalidate(pcci); + pcc_inode_put(pcci); + } out_unlock: pcc_inode_unlock(inode); + if (hsm_remove) { + const struct cred *old_cred; + + old_cred = override_creds(pcc_super_cred(inode->i_sb)); + rc = pcc_hsm_remove(inode); + revert_creds(old_cred); + } + RETURN(rc); } diff --git a/lustre/llite/pcc.h b/lustre/llite/pcc.h index da3a6cb..d2c8512 100644 --- a/lustre/llite/pcc.h +++ b/lustre/llite/pcc.h @@ -186,7 +186,7 @@ int pcc_readwrite_attach(struct file *file, struct inode *inode, int pcc_readwrite_attach_fini(struct file *file, struct inode *inode, __u32 gen, bool lease_broken, int rc, bool attached); -int pcc_ioctl_detach(struct inode *inode); +int pcc_ioctl_detach(struct inode *inode, __u32 opt); int pcc_ioctl_state(struct file *file, struct inode *inode, struct lu_pcc_state *state); void pcc_file_init(struct pcc_file *pccf); diff --git a/lustre/tests/sanity-pcc.sh b/lustre/tests/sanity-pcc.sh index d3dc578..e18ad79 100644 --- a/lustre/tests/sanity-pcc.sh +++ b/lustre/tests/sanity-pcc.sh @@ -268,9 +268,11 @@ lpcc_rw_test() { echo "Start to detach the $file" do_facet $SINGLEAGT $LFS pcc detach $file || error "PCC detach $file failed" + wait_request_state $(path2fid $file) REMOVE SUCCEED + check_lpcc_state $file "none" - # HSM released exists archived status - check_hsm_flags $file "0x0000000d" + # The file is removed from PCC + check_hsm_flags $file "0x00000000" check_file_data $SINGLEAGT $file "attach_detach" } @@ -320,6 +322,10 @@ test_1e() { error "failed to dd write to $file" check_lpcc_state $file "readwrite" + do_facet $SINGLEAGT $RUNAS $LFS pcc detach -k $file || + error "failed to detach file $file" + check_lpcc_state $file "none" + # non-root user is forbidden to access PCC file directly lpcc_path=$(lpcc_fid2path $hsm_root $file) do_facet $SINGLEAGT $RUNAS touch $lpcc_path && @@ -333,9 +339,14 @@ test_1e() { [[ $perm == "0" ]] || error "PCC file permission ($perm) is not zero" + do_facet $SINGLEAGT $RUNAS $LFS pcc attach -i $HSM_ARCHIVE_NUMBER \ + $file || error "failed to attach file $file" + check_lpcc_state $file "readwrite" + do_facet $SINGLEAGT $RUNAS $LFS pcc detach $file || error "failed to detach file $file" check_lpcc_state $file "none" + wait_request_state $(path2fid $file) REMOVE SUCCEED } run_test 1e "Test RW-PCC with non-root user" @@ -382,6 +393,7 @@ test_1f() { do_facet $SINGLEAGT $RUNAS $LFS pcc detach $file || error "failed to detach file $file" check_lpcc_state $file "none" + wait_request_state $(path2fid $file) REMOVE SUCCEED } run_test 1f "Test auto RW-PCC cache with non-root user" @@ -410,6 +422,7 @@ test_1g() { do_facet $SINGLEAGT $RUNAS $LFS pcc detach $file || error "failed to detach file $file" check_lpcc_state $file "none" + wait_request_state $(path2fid $file) REMOVE SUCCEED do_facet $SINGLEAGT $RUNAS dd if=$file of=/dev/null bs=1024 count=1 || error "non-root user cannot read to $file with permisson (777)" } @@ -557,7 +570,7 @@ test_3a() { do_facet $SINGLEAGT $LFS pcc attach -i $HSM_ARCHIVE_NUMBER $file || error "failed to attach file $file" check_lpcc_state $file "readwrite" - do_facet $SINGLEAGT $LFS pcc detach $file || + do_facet $SINGLEAGT $LFS pcc detach -k $file || error "failed to detach file $file" check_lpcc_state $file "none" @@ -565,7 +578,7 @@ test_3a() { do_facet $SINGLEAGT $LFS pcc attach -i $HSM_ARCHIVE_NUMBER $file || error "failed to attach file $file" check_lpcc_state $file "readwrite" - do_facet $SINGLEAGT $LFS pcc detach $file || + do_facet $SINGLEAGT $LFS pcc detach -k $file || error "failed to detach file $file" check_lpcc_state $file "none" } @@ -591,7 +604,7 @@ test_3b() { do_facet agt1 $LFS pcc attach -i 1 $file || error "failed to attach file $file" check_lpcc_state $file "readwrite" agt1 - do_facet agt1 $LFS pcc detach $file || + do_facet agt1 $LFS pcc detach -k $file || error "failed to detach file $file" check_lpcc_state $file "none" agt1 @@ -599,7 +612,7 @@ test_3b() { do_facet agt2 $LFS pcc attach -i 2 $file || error "failed to attach file $file" check_lpcc_state $file "readwrite" agt2 - do_facet agt2 $LFS pcc detach $file || + do_facet agt2 $LFS pcc detach -k $file || error "failed to detach file $file" check_lpcc_state $file "none" agt2 @@ -613,7 +626,7 @@ test_3b() { # The later attach PCC agent should succeed, # the former agent should be detached automatically. check_lpcc_state $file "none" agt1 - do_facet agt2 $LFS pcc detach $file || + do_facet agt2 $LFS pcc detach -k $file || error "failed to detach file $file" check_lpcc_state $file "none" agt2 } @@ -714,7 +727,7 @@ test_6() { error "mmap write data mismatch: $content" check_lpcc_state $file "readwrite" - do_facet $SINGLEAGT $LFS pcc detach $file || + do_facet $SINGLEAGT $LFS pcc detach -k $file || error "failed to detach file $file" content=$(do_facet $SINGLEAGT $MMAP_CAT $file) @@ -775,11 +788,11 @@ test_7b() { # HSM released exists archived status check_hsm_flags $file "0x0000000d" - # multiop mmap write increase the first character of each page with 1 + # multiop mmap write increases the first character of each page with 1 do_facet $SINGLEAGT $MULTIOP $file OSMWUc & pid=$! - do_facet $SINGLEAGT $LFS pcc detach $file || + do_facet $SINGLEAGT $LFS pcc detach -k $file || error "failed to detach file $file" wait $pid || error "multiop mmap write failed" @@ -881,7 +894,7 @@ test_usrgrp_quota() { error "dd write $file1 failed" # -EDQUOT error should be tolerated via fallback to normal Lustre path. check_lpcc_state $file1 "none" - do_facet $SINGLEAGT $LFS pcc detach $file1 || + do_facet $SINGLEAGT $LFS pcc detach -k $file1 || error "failed to detach file $file" rm $file1 $file2 } @@ -918,7 +931,7 @@ test_11() { error "failed to attach $file" check_lpcc_state $file "readwrite" check_file_data $SINGLEAGT $file "QQQQQ" - do_facet $SINGLEAGT $LFS pcc detach $file || + do_facet $SINGLEAGT $LFS pcc detach -k $file || error "failed to detach $file" rm $file || error "rm $file failed" @@ -950,15 +963,19 @@ run_test 11 "Test attach fault injection with simulated PCC file path" test_12() { local file=$DIR/$tfile + local hsm_root=$(hsm_root) + local -a lpcc_path local pid copytool setup -m "$MOUNT" -a "$HSM_ARCHIVE_NUMBER" setup_pcc_mapping echo -n race_rw_attach_hsmremove > $file + lpcc_path=$(lpcc_fid2path $hsm_root $file) do_facet $SINGLEAGT $LFS pcc attach -i $HSM_ARCHIVE_NUMBER $file || error "attach $file failed" - do_facet $SINGLEAGT $LFS pcc detach $file || error "detach $file failed" + do_facet $SINGLEAGT $LFS pcc detach -k $file || + error "detach $file failed" # HSM released exists archived status check_hsm_flags $file "0x0000000d" # define OBD_FAIL_LLITE_PCC_ATTACH_PAUSE 0x1414 @@ -970,6 +987,8 @@ test_12() { wait_request_state $(path2fid $file) RESTORE SUCCEED $LFS hsm_remove $file || error "hsm remove $file failed" wait $pid && error "RW-PCC attach $file should fail" + do_facet $SINGLEAGT "[ -f $lpcc_path ]" && + error "RW-PCC cached file '$lpcc_path' should be removed" return 0 } @@ -1001,7 +1020,7 @@ test_rule_id() { error "failed to dd write from $file" check_lpcc_state $file "readwrite" - do_facet $SINGLEAGT $myRUNAS $LFS pcc detach $file || + do_facet $SINGLEAGT $myRUNAS $LFS pcc detach -k $file || error "failed to detach file $file" check_lpcc_state $file "none" } @@ -1028,7 +1047,7 @@ test_13b() { do_facet $SINGLEAGT dd if=/dev/zero of=$file bs=1024 count=1 || error "failed to dd write to $file" check_lpcc_state $file "readwrite" - do_facet $SINGLEAGT $myRUNAS $LFS pcc detach $file || + do_facet $SINGLEAGT $myRUNAS $LFS pcc detach -k $file || error "failed to detach file $file" check_lpcc_state $file "none" rm $file || error "rm $file failed" @@ -1037,7 +1056,7 @@ test_13b() { do_facet $SINGLEAGT $RUNAS dd if=/dev/zero of=$file bs=1024 count=1 || error "failed to dd write to $file" check_lpcc_state $file "readwrite" - do_facet $SINGLEAGT $myRUNAS $LFS pcc detach $file || + do_facet $SINGLEAGT $myRUNAS $LFS pcc detach -k $file || error "failed to detach file $file" check_lpcc_state $file "none" rm $file || error "rm $file failed" @@ -1046,7 +1065,7 @@ test_13b() { do_facet $SINGLEAGT $RUNAS dd if=/dev/zero of=$file bs=1024 count=1 || error "failed to dd write to $file" check_lpcc_state $file "readwrite" - do_facet $SINGLEAGT $myRUNAS $LFS pcc detach $file || + do_facet $SINGLEAGT $myRUNAS $LFS pcc detach -k $file || error "failed to detach file $file" check_lpcc_state $file "none" rm $file || error "rm $file failed" @@ -1091,7 +1110,7 @@ test_13c() { do_facet $SINGLEAGT dd if=/dev/zero of=$file bs=1024 count=1 || error "failed to dd write to $file" check_lpcc_state $file "readwrite" - do_facet $SINGLEAGT $LFS pcc detach $file || + do_facet $SINGLEAGT $LFS pcc detach -k $file || error "failed to detach $file" rm $file || error "rm $file failed" @@ -1105,7 +1124,7 @@ test_13c() { do_facet $SINGLEAGT dd if=/dev/zero of=$file bs=1024 count=1 || error "failed to dd write to $file" check_lpcc_state $file "readwrite" - do_facet $SINGLEAGT $LFS pcc detach $file || + do_facet $SINGLEAGT $LFS pcc detach -k $file || error "failed to detach $file" rm $file || error "rm $file failed" @@ -1114,7 +1133,7 @@ test_13c() { do_facet $SINGLEAGT $myRUNAS dd if=/dev/zero of=$file bs=1024 count=1 || error "failed to dd write to $file" check_lpcc_state $file "readwrite" - do_facet $SINGLEAGT $LFS pcc detach $file || + do_facet $SINGLEAGT $LFS pcc detach -k $file || error "failed to detach $file" rm $file || error "rm $file failed" } @@ -1167,10 +1186,10 @@ test_15() { do_facet $SINGLEAGT $LCTL \ set_param ldlm.namespaces.*mdc*.lru_size=clear check_lpcc_state $file "readwrite" $SINGLEAGT "$RUNAS" - # Detach the file directly, as the file layout generation - # is not changed, so the file is still valid cached in PCC, - # and can be reused from PCC cache directly. - do_facet $SINGLEAGT $RUNAS $LFS pcc detach $file || + # Detach the file but keep the cache , as the file layout generation + # is not changed, so the file is still valid cached in PCC, and can + # be reused from PCC cache directly. + do_facet $SINGLEAGT $RUNAS $LFS pcc detach -k $file || error "PCC detach $file failed" check_lpcc_state $file "readwrite" $SINGLEAGT "$RUNAS" do_facet $SINGLEAGT $RUNAS $LFS pcc detach $file || @@ -1190,22 +1209,70 @@ test_15() { check_file_data $SINGLEAGT $file "autoattach_data" check_lpcc_state $file "readwrite" - # Detach the file directly, as the file layout generation + # Detach the file with -k option, as the file layout generation # is not changed, so the file is still valid cached in PCC, # and can be reused from PCC cache directly. - do_facet $SINGLEAGT $LFS pcc detach $file || + do_facet $SINGLEAGT $LFS pcc detach -k $file || error "PCC detach $file failed" check_lpcc_state $file "readwrite" # HSM released exists archived status check_hsm_flags $file "0x0000000d" check_file_data $SINGLEAGT $file "autoattach_data" + # HSM restore the PCC cached file, the layout generation + # was changed, so the file can not be auto attached. $LFS hsm_restore $file || error "failed to restore $file" wait_request_state $(path2fid $file) RESTORE SUCCEED check_lpcc_state $file "none" + # HSM exists archived status + check_hsm_flags $file "0x00000009" + } run_test 15 "Test auto attach at open when file is still valid cached" +test_16() { + local loopfile="$TMP/$tfile" + local mntpt="/mnt/pcc.$tdir" + local hsm_root="$mntpt/$tdir" + local file=$DIR/$tfile + local -a lpcc_path + + setup_loopdev $SINGLEAGT $loopfile $mntpt 50 + copytool setup -m "$MOUNT" -a "$HSM_ARCHIVE_NUMBER" + setup_pcc_mapping $SINGLEAGT \ + "projid={100}\ rwid=$HSM_ARCHIVE_NUMBER\ open_attach=1" + + do_facet $SINGLEAGT "echo -n detach_data > $file" + lpcc_path=$(lpcc_fid2path $hsm_root $file) + do_facet $SINGLEAGT $LFS pcc attach -i $HSM_ARCHIVE_NUMBER \ + $file || error "PCC attach $file failed" + check_lpcc_state $file "readwrite" + # HSM released exists archived status + check_hsm_flags $file "0x0000000d" + + echo "Test for reusing valid PCC cache" + # Valid PCC cache can be reused + do_facet $SINGLEAGT $LFS pcc detach -k $file || + error "PCC detach $file failed" + check_lpcc_state $file "readwrite" + # HSM released exists archived status + check_hsm_flags $file "0x0000000d" + + echo "Test for the default detach" + # Permanent detach by default, it will remove the PCC copy + do_facet $SINGLEAGT $LFS pcc detach $file || + error "PCC detach $file failed" + wait_request_state $(path2fid $file) REMOVE SUCCEED + check_lpcc_state $file "none" + # File is removed from PCC backend + check_hsm_flags $file "0x00000000" + do_facet $SINGLEAGT "[ -f $lpcc_path ]" && + error "RW-PCC cached file '$lpcc_path' should be removed" + + return 0 +} +run_test 16 "Test detach with different options" + complete $SECONDS check_and_cleanup_lustre exit_status diff --git a/lustre/utils/lfs.c b/lustre/utils/lfs.c index d11c61c..d09342b 100644 --- a/lustre/utils/lfs.c +++ b/lustre/utils/lfs.c @@ -10516,16 +10516,30 @@ static int lfs_pcc_attach_fid(int argc, char **argv) static int lfs_pcc_detach(int argc, char **argv) { + struct option long_opts[] = { + { .val = 'k', .name = "keep", .has_arg = no_argument }, + { .name = NULL } }; + char short_opts[] = "k"; + int c; int rc = 0; const char *path; char fullpath[PATH_MAX]; + __u32 detach_opt = PCC_DETACH_OPT_UNCACHE; - optind = 1; - - if (argc <= 1) { - fprintf(stderr, "%s: must specify one or more file names\n", - argv[0]); - return CMD_HELP; + optind = 0; + while ((c = getopt_long(argc, argv, short_opts, + long_opts, NULL)) != -1) { + switch (c) { + case 'k': + detach_opt = PCC_DETACH_OPT_NONE; + break; + case '?': + return CMD_HELP; + default: + fprintf(stderr, "%s: option '%s' unrecognized\n", + argv[0], argv[optind - 1]); + return CMD_HELP; + } } while (optind < argc) { @@ -10540,7 +10554,7 @@ static int lfs_pcc_detach(int argc, char **argv) continue; } - rc2 = llapi_pcc_detach_file(fullpath); + rc2 = llapi_pcc_detach_file(fullpath, detach_opt); if (rc2 < 0) { rc2 = -errno; fprintf(stderr, "%s: cannot detach '%s' from PCC: " @@ -10554,16 +10568,30 @@ static int lfs_pcc_detach(int argc, char **argv) static int lfs_pcc_detach_fid(int argc, char **argv) { + struct option long_opts[] = { + { .val = 'k', .name = "keep", .has_arg = no_argument }, + { .name = NULL } }; + char short_opts[] = "k"; + int c; int rc = 0; const char *fid; const char *mntpath; + __u32 detach_opt = PCC_DETACH_OPT_UNCACHE; - optind = 1; - - if (argc <= 2) { - fprintf(stderr, "%s: not enough argument\n", - argv[0]); - return CMD_HELP; + optind = 0; + while ((c = getopt_long(argc, argv, short_opts, + long_opts, NULL)) != -1) { + switch (c) { + case 'k': + detach_opt = PCC_DETACH_OPT_NONE; + break; + case '?': + return CMD_HELP; + default: + fprintf(stderr, "%s: option '%s' unrecognized\n", + argv[0], argv[optind - 1]); + return CMD_HELP; + } } mntpath = argv[optind++]; @@ -10573,7 +10601,7 @@ static int lfs_pcc_detach_fid(int argc, char **argv) fid = argv[optind++]; - rc2 = llapi_pcc_detach_fid_str(mntpath, fid); + rc2 = llapi_pcc_detach_fid_str(mntpath, fid, detach_opt); if (rc2 < 0) { fprintf(stderr, "%s: cannot detach '%s' on '%s' " "from PCC: %s\n", argv[0], fid, mntpath, diff --git a/lustre/utils/liblustreapi_pcc.c b/lustre/utils/liblustreapi_pcc.c index 8e52b8f..398239d 100644 --- a/lustre/utils/liblustreapi_pcc.c +++ b/lustre/utils/liblustreapi_pcc.c @@ -182,14 +182,17 @@ int llapi_pcc_attach_fid_str(const char *mntpath, const char *fidstr, * detach PCC cache of a file by using fd. * * \param fd File handle. + * \param option Detach option * * \return 0 on success, an error code otherwise. */ -int llapi_pcc_detach_fd(int fd) +int llapi_pcc_detach_fd(int fd, __u32 option) { + struct lu_pcc_detach detach; int rc; - rc = ioctl(fd, LL_IOC_PCC_DETACH); + detach.pccd_opt = option; + rc = ioctl(fd, LL_IOC_PCC_DETACH, &detach); return rc; } @@ -198,14 +201,16 @@ int llapi_pcc_detach_fd(int fd) * * \param mntpath Fullpath to the client mount point. * \param fid FID of the file. + * \param option Detach option. * * \return 0 on success, an error code otherwise. */ -int llapi_pcc_detach_fid(const char *mntpath, const struct lu_fid *fid) +int llapi_pcc_detach_fid(const char *mntpath, const struct lu_fid *fid, + __u32 option) { int rc; int fd; - struct lu_pcc_detach detach; + struct lu_pcc_detach_fid detach; rc = get_root_path(WANT_FD, NULL, &fd, (char *)mntpath, -1); if (rc) { @@ -222,6 +227,7 @@ int llapi_pcc_detach_fid(const char *mntpath, const struct lu_fid *fid) * files. */ detach.pccd_fid = *fid; + detach.pccd_opt = option; rc = ioctl(fd, LL_IOC_PCC_DETACH_BY_FID, &detach); close(fd); return rc; @@ -231,11 +237,13 @@ int llapi_pcc_detach_fid(const char *mntpath, const struct lu_fid *fid) * detach PCC cache of a file via FID. * * \param mntpath Fullpath to the client mount point. - * \param fid FID string of the file. + * \param fidstr FID string of the file. + * \param option Detach option. * * \return 0 on success, an error code otherwise. */ -int llapi_pcc_detach_fid_str(const char *mntpath, const char *fidstr) +int llapi_pcc_detach_fid_str(const char *mntpath, const char *fidstr, + __u32 option) { int rc; struct lu_fid fid; @@ -252,7 +260,7 @@ int llapi_pcc_detach_fid_str(const char *mntpath, const char *fidstr) return -EINVAL; } - rc = llapi_pcc_detach_fid(mntpath, &fid); + rc = llapi_pcc_detach_fid(mntpath, &fid, option); return rc; } @@ -260,11 +268,12 @@ int llapi_pcc_detach_fid_str(const char *mntpath, const char *fidstr) /** * detach PCC cache of a file. * - * \param path Fullpath to the file to operate on. + * \param path Fullpath to the file to operate on. + * \param option Detach option. * * \return 0 on success, an error code otherwise. */ -int llapi_pcc_detach_file(const char *path) +int llapi_pcc_detach_file(const char *path, __u32 option) { int rc; int fd; @@ -277,7 +286,7 @@ int llapi_pcc_detach_file(const char *path) return rc; } - rc = llapi_pcc_detach_fd(fd); + rc = llapi_pcc_detach_fd(fd, option); close(fd); return rc; } -- 1.8.3.1