LU-12373 pcc: delete stale PCC copy when remove PCC backend

By defualt, when removing a PCC backend from a client, the action
is to scan the PCC backend FS, uncache (detach and remove) all
scanned PCC copies from PCC by FIDs.

However, during the tests, we found that some old stale PCC copies
are not removed when an adminstrator runs "lctl pcc del|clear".
The reason is that these PCC copies are already detached from PCC
when running the commands.

This patch fixes this bug: when removing a PCC backend from a
client, it will also delete all non-cached PCC copies from PCC
backend to free up the space.

Test-Parameters: clientcount=3 testlist=sanity-pcc,sanity-pcc,sanity-pcc
Signed-off-by: Qian Yingjin <qian@ddn.com>
Change-Id: Id829abe7e6cb1294e6baea76452f4a9178711451
Reviewed-on: https://review.whamcloud.com/41925
Reviewed-by: Li Xi <lixi@ddn.com>
Tested-by: jenkins <devops@whamcloud.com>
Tested-by: Maloo <maloo@whamcloud.com>
Reviewed-by: Andreas Dilger <adilger@whamcloud.com>

diff --git a/lustre/include/uapi/linux/lustre/lustre_user.h b/lustre/include/uapi/linux/lustre/lustre_user.h
index 3eaf19a..ddab641 100644 (file)
--- a/lustre/include/uapi/linux/lustre/lustre_user.h
+++ b/lustre/include/uapi/linux/lustre/lustre_user.h
@@ -630,8 +630,9 @@ struct ll_ioc_lease_id {
 #define LL_IOC_HEAT_GET                        _IOWR('f', 251, struct lu_heat)
 #define LL_IOC_HEAT_SET                        _IOW('f', 251, __u64)
 #define LL_IOC_PCC_ATTACH              _IOW('f', 252, struct lu_pcc_attach)
-#define LL_IOC_PCC_DETACH              _IOW('f', 252, struct lu_pcc_detach)
-#define LL_IOC_PCC_DETACH_BY_FID       _IOW('f', 252, struct lu_pcc_detach_fid)
+#define LL_IOC_PCC_DETACH              _IOWR('f', 252, struct lu_pcc_detach)
+#define LL_IOC_PCC_DETACH_BY_FID       _IOWR('f', 252, \
+                                               struct lu_pcc_detach_fid)
 #define LL_IOC_PCC_STATE               _IOR('f', 252, struct lu_pcc_state)
 
 #ifndef        FS_IOC_FSGETXATTR
@@ -2709,6 +2710,8 @@ enum lu_pcc_detach_flags {
        PCC_DETACH_FL_KNOWN_READWRITE   = 0x02,
        /* Known the file was once used as PCC-RO */
        PCC_DETACH_FL_KNOWN_READONLY    = 0x04,
+       /* Indicate PCC cached copy is removed */
+       PCC_DETACH_FL_CACHE_REMOVED     = 0x08,
 };
 
 struct lu_pcc_detach_fid {

diff --git a/lustre/llite/dir.c b/lustre/llite/dir.c
index 7d556dc..838eccd 100644 (file)
--- a/lustre/llite/dir.c
+++ b/lustre/llite/dir.c
@@ -2157,7 +2157,12 @@ migrate_free:
                if (!inode_owner_or_capable(inode2))
                        GOTO(out_iput, rc = -EPERM);
 
-               rc = pcc_ioctl_detach(inode2, detach->pccd_flags);
+               rc = pcc_ioctl_detach(inode2, &detach->pccd_flags);
+               if (rc)
+                       GOTO(out_iput, rc);
+
+               if (copy_to_user((char __user *)arg, detach, sizeof(*detach)))
+                       GOTO(out_iput, rc = -EFAULT);
 out_iput:
                iput(inode2);
 out_detach:

diff --git a/lustre/llite/file.c b/lustre/llite/file.c
index ea18207..f0d28c4 100644 (file)
--- a/lustre/llite/file.c
+++ b/lustre/llite/file.c
@@ -4044,7 +4044,12 @@ out_pcc:
                if (!inode_owner_or_capable(inode))
                        GOTO(out_detach_free, rc = -EPERM);
 
-               rc = pcc_ioctl_detach(inode, detach->pccd_flags);
+               rc = pcc_ioctl_detach(inode, &detach->pccd_flags);
+               if (rc)
+                       GOTO(out_detach_free, rc);
+
+               if (copy_to_user((char __user *)arg, detach, sizeof(*detach)))
+                       GOTO(out_detach_free, rc = -EFAULT);
 out_detach_free:
                OBD_FREE_PTR(detach);
                RETURN(rc);

diff --git a/lustre/llite/pcc.c b/lustre/llite/pcc.c
index d373a10..24c8101 100644 (file)
--- a/lustre/llite/pcc.c
+++ b/lustre/llite/pcc.c
@@ -2635,10 +2635,12 @@ int pcc_page_mkwrite(struct vm_area_struct *vma, struct vm_fault *vmf,
 
        inode = file_inode(pccv->pccv_file);
        if (!pccv->pccv_vm_ops->page_mkwrite) {
+               __u32 flags = PCC_DETACH_FL_UNCACHE;
+
                CDEBUG(D_MMAP,
                       "%s: PCC backend fs not support ->page_mkwrite()\n",
                       ll_i2sbi(inode)->ll_fsname);
-               (void) pcc_ioctl_detach(inode, PCC_DETACH_FL_UNCACHE);
+               (void) pcc_ioctl_detach(inode, &flags);
                pcc_mmap_vma_reset(inode, vma);
                up_read(&mm->mmap_sem);
                *cached = true;
@@ -2696,7 +2698,9 @@ out:
         * Lustre I/O path.
         */
        if (rc & VM_FAULT_SIGBUS) {
-               (void) pcc_ioctl_detach(inode, PCC_DETACH_FL_UNCACHE);
+               __u32 flags = PCC_DETACH_FL_UNCACHE;
+
+               (void) pcc_ioctl_detach(inode, &flags);
                pcc_mmap_vma_reset(inode, vma);
                up_read(&mm->mmap_sem);
                RETURN(VM_FAULT_RETRY | VM_FAULT_NOPAGE);
@@ -2742,9 +2746,11 @@ out:
        pcc_io_fini(inode, PIT_FAULT, rc, cached);
 
        if ((rc & VM_FAULT_SIGBUS) && !(rc & VM_FAULT_OOM)) {
+               __u32 flags = PCC_DETACH_FL_UNCACHE;
+
                CDEBUG(D_CACHE, "PCC fault failed: fid = "DFID" rc = %d\n",
                       PFID(ll_inode2fid(inode)), rc);
-               (void) pcc_ioctl_detach(inode, PCC_DETACH_FL_UNCACHE);
+               (void) pcc_ioctl_detach(inode, &flags);
                pcc_mmap_vma_reset(inode, vma);
        }
 
@@ -3491,7 +3497,7 @@ static int pcc_hsm_remove(struct inode *inode)
        RETURN(rc);
 }
 
-int pcc_ioctl_detach(struct inode *inode, __u32 flags)
+int pcc_ioctl_detach(struct inode *inode, __u32 *flags)
 {
        struct ll_inode_info *lli = ll_i2info(inode);
        struct pcc_inode *pcci;
@@ -3510,7 +3516,7 @@ int pcc_ioctl_detach(struct inode *inode, __u32 flags)
        LASSERT(atomic_read(&pcci->pcci_refcount) > 0);
 
        if (pcci->pcci_type == LU_PCC_READWRITE) {
-               if (flags & PCC_DETACH_FL_UNCACHE) {
+               if (*flags & PCC_DETACH_FL_UNCACHE) {
                        hsm_remove = true;
                        /*
                         * The file will be removed from PCC, set the flags
@@ -3524,12 +3530,14 @@ int pcc_ioctl_detach(struct inode *inode, __u32 flags)
        } else if (pcci->pcci_type == LU_PCC_READONLY) {
                pcc_inode_detach(inode);
 
-               if (flags & PCC_DETACH_FL_UNCACHE && !pcci->pcci_unlinked) {
+               if (*flags & PCC_DETACH_FL_UNCACHE && !pcci->pcci_unlinked) {
                        old_cred =  override_creds(pcc_super_cred(inode->i_sb));
                        rc = pcc_inode_remove(inode, pcci->pcci_path.dentry);
                        revert_creds(old_cred);
-                       if (!rc)
+                       if (!rc) {
                                pcci->pcci_unlinked = true;
+                               *flags |= PCC_DETACH_FL_CACHE_REMOVED;
+                       }
                }
 
                pcc_inode_put(pcci);
@@ -3540,11 +3548,13 @@ int pcc_ioctl_detach(struct inode *inode, __u32 flags)
 out_unlock:
        pcc_inode_unlock(inode);
 
-       if (hsm_remove || (flags & PCC_DETACH_FL_UNCACHE &&
-                          flags & PCC_DETACH_FL_KNOWN_READWRITE)) {
+       if (hsm_remove || (*flags & PCC_DETACH_FL_UNCACHE &&
+                          *flags & PCC_DETACH_FL_KNOWN_READWRITE)) {
                old_cred = override_creds(pcc_super_cred(inode->i_sb));
                rc = pcc_hsm_remove(inode);
                revert_creds(old_cred);
+               if (!rc)
+                       *flags |= PCC_DETACH_FL_CACHE_REMOVED;
        }
 
        RETURN(rc);

diff --git a/lustre/llite/pcc.h b/lustre/llite/pcc.h
index d15ec7e..ae48b4a 100644 (file)
--- a/lustre/llite/pcc.h
+++ b/lustre/llite/pcc.h
@@ -261,7 +261,7 @@ int pcc_readwrite_attach_fini(struct file *file, struct inode *inode,
                              bool attached);
 int pcc_ioctl_attach(struct file *file, struct inode *inode,
                     struct lu_pcc_attach *attach);
-int pcc_ioctl_detach(struct inode *inode, __u32 flags);
+int pcc_ioctl_detach(struct inode *inode, __u32 *flags);
 int pcc_ioctl_state(struct file *file, struct inode *inode,
                    struct lu_pcc_state *state);
 void pcc_file_init(struct pcc_file *pccf);

diff --git a/lustre/tests/sanity-pcc.sh b/lustre/tests/sanity-pcc.sh
index 1a7f228..827f7a9 100644 (file)
--- a/lustre/tests/sanity-pcc.sh
+++ b/lustre/tests/sanity-pcc.sh
@@ -168,6 +168,7 @@ set_hsm_param grace_delay 10
 cleanup_pcc_mapping() {
        local facet=${1:-$SINGLEAGT}
 
+       echo "Cleanup PCC backend on $MOUNT"
        do_facet $facet $LCTL pcc clear $MOUNT
 }
 
@@ -1753,7 +1754,6 @@ test_21c() {
        local mntpt="/mnt/pcc.$tdir"
        local hsm_root="$mntpt/$tdir"
        local file=$DIR/$tfile
-       local file2=$DIR2/$tfile
        local fid
 
        setup_loopdev $SINGLEAGT $loopfile $mntpt 50
@@ -1781,6 +1781,7 @@ test_21c() {
        do_facet $SINGLEAGT $LFS pcc detach -k $file ||
                error "failed to detach $file"
        check_lpcc_state $file "none"
+
        unlink $file || error "unlink $file failed"
 }
 run_test 21c "Verify HSM release works storing PCC-RO as HSM mirror component"
@@ -2818,6 +2819,46 @@ test_35() {
 }
 run_test 35 "mmap fault test"
 
+test_36_base() {
+       local loopfile="$TMP/$tfile"
+       local mntpt="/mnt/pcc.$tdir"
+       local hsm_root="$mntpt/$tdir"
+       local file=$DIR/$tfile
+       local -a lpcc_path
+       local state="readwrite"
+       local ro="$1"
+
+       [[ -z $ro ]] || state="readonly"
+       setup_loopdev $SINGLEAGT $loopfile $mntpt 50
+       copytool setup -m "$MOUNT" -a "$HSM_ARCHIVE_NUMBER"
+       setup_pcc_mapping
+
+       echo -n backend_clear_verify > $file
+       lpcc_path=$(lpcc_fid2path $hsm_root $file)
+       do_facet $SINGLEAGT $LFS pcc attach $ro -i $HSM_ARCHIVE_NUMBER $file ||
+               error "PCC attach $ro $file failed"
+       check_lpcc_state $file "$state"
+       do_facet $SINGLEAGT $LFS pcc detach -k $file ||
+               error "PCC detach -k $file failed"
+       do_facet $SINGLEAGT "[ -f $lpcc_path1 ]" ||
+               error "PCC copy $lpcc_path should retain"
+       do_facet $SINGLEAGT $LCTL pcc clear -v $MOUNT ||
+               error "lctl pcc clear -v $MOUNT failed"
+       do_facet $SINGLEAGT "[ -f $lpcc_path ]" &&
+               error "PCC copy $lpcc_path should be removed"
+       rm $file || error "rm $file failed"
+}
+
+test_36a() {
+       test_36_base
+}
+run_test 36a "Stale RW-PCC copy should be deleted after remove the PCC backend"
+
+test_36b() {
+       test_36_base "-r"
+}
+run_test 36b "Stale RO-PCC copy should be deleted after remove the PCC backend"
+
 complete $SECONDS
 check_and_cleanup_lustre
 exit_status

diff --git a/lustre/utils/liblustreapi_pcc.c b/lustre/utils/liblustreapi_pcc.c
index 24468de..1107f23 100644 (file)
--- a/lustre/utils/liblustreapi_pcc.c
+++ b/lustre/utils/liblustreapi_pcc.c
@@ -525,14 +525,16 @@ out_free_param:
 static int llapi_pcc_scan_detach(const char *pname, const char *fname,
                                 struct hsm_scan_control *hsc)
 {
+       struct lu_pcc_detach_fid detach;
+       char fullname[PATH_MAX];
        char fidstr[FID_LEN];
        const char *fidname;
-       __u32 detach_flags;
        bool lov_file;
+       int fd;
        int rc;
 
-       detach_flags = PCC_DETACH_FL_UNCACHE;
        /* It is the saved lov file when archive on HSM backend. */
+       detach.pccd_flags = PCC_DETACH_FL_UNCACHE;
        lov_file = endswith(fname, ".lov");
        if (lov_file) {
                size_t len;
@@ -548,16 +550,63 @@ static int llapi_pcc_scan_detach(const char *pname, const char *fname,
                }
                strncpy(fidstr, fname, FID_LEN);
                fidstr[len] = '\0';
-               detach_flags |= PCC_DETACH_FL_KNOWN_READWRITE;
+               detach.pccd_flags |= PCC_DETACH_FL_KNOWN_READWRITE;
                fidname = fidstr;
        } else {
                fidname = fname;
        }
 
+       rc = sscanf(fidname, SFID, RFID(&detach.pccd_fid));
+       if (rc != 3 || !fid_is_sane(&detach.pccd_fid)) {
+               llapi_err_noerrno(LLAPI_MSG_ERROR,
+                                 "bad FID format '%s', should be [seq:oid:ver] (e.g. "DFID")\n",
+                                 fidname, (unsigned long long)FID_SEQ_NORMAL,
+                                 2, 0);
+               return -EINVAL;
+       }
+
        llapi_printf(LLAPI_MSG_DEBUG, "Handle the file: %s\n", fidname);
 
-       return llapi_pcc_detach_fid_str(hsc->hsc_mntpath, fidname,
-                                       detach_flags);
+       rc = get_root_path(WANT_FD, NULL, &fd, (char *)hsc->hsc_mntpath, -1);
+       if (rc) {
+               llapi_error(LLAPI_MSG_ERROR, rc, "cannot get root path: %s",
+                           hsc->hsc_mntpath);
+               return rc;
+       }
+
+       rc = ioctl(fd, LL_IOC_PCC_DETACH_BY_FID, &detach);
+       close(fd);
+
+       if (rc) {
+               llapi_printf(LLAPI_MSG_DEBUG,
+                            "failed to detach file: %s\n", fidname);
+               return rc;
+       }
+
+       if (detach.pccd_flags & PCC_DETACH_FL_CACHE_REMOVED) {
+               llapi_printf(LLAPI_MSG_DEBUG,
+                            "Detach and remove the PCC cached file: %s\n",
+                            fidname);
+       } else {
+               snprintf(fullname, sizeof(fullname), "%s/%s", pname, fidname);
+               llapi_printf(LLAPI_MSG_DEBUG,
+                            "Remove non-cached file: %s flags: %X\n",
+                            fullname, detach.pccd_flags);
+               if (unlink(fullname) && errno != ENOENT) {
+                       rc = -errno;
+                       llapi_error(LLAPI_MSG_ERROR, rc,
+                                   "Unlink %s failed", fullname);
+               }
+
+               if (detach.pccd_flags & PCC_DETACH_FL_KNOWN_READWRITE) {
+                       snprintf(fullname, sizeof(fullname), "%s/%s",
+                                pname, fname);
+                       /* Remove *.lov file */
+                       unlink(fullname);
+               }
+       }
+
+       return rc;
 }
 
 static int llapi_pcc_del_internal(const char *mntpath, const char *pccpath,