Whamcloud - gitweb
EX-4182 sec: support of PCC-RO for encrypted files
authorSebastien Buisson <sbuisson@ddn.com>
Mon, 20 Dec 2021 16:41:56 +0000 (17:41 +0100)
committerAndreas Dilger <adilger@whamcloud.com>
Mon, 21 Mar 2022 18:48:42 +0000 (18:48 +0000)
In order to support PCC-RO for encrypted files, we decide to store
in PCC the ciphertext version of the Lustre files. We proceed to
decryption of PCC files only in the page cache, so cleartext is just
in memory. When a Lustre file is detached from PCC, or when the
encryption key is removed, we trash those PCC page cache pages.

As PCC files contain ciphertext, their sizes are aligned on
LUSTRE_ENCRYPTION_UNIT_SIZE instead of being lustre inode's clear
text size. In order to keep track of Lustre files' actual sizes,
we use a dedicated xattr on the PCC files. Its value is set at pcc
attach time, which is fine for PCC-RO.

Also add sanity-pcc test_21j to exercise this.

Signed-off-by: Sebastien Buisson <sbuisson@ddn.com>
Change-Id: Id7d96adb4eb4770c813a042acf7ed6c42224b9bf
Reviewed-on: https://review.whamcloud.com/45910
Tested-by: jenkins <devops@whamcloud.com>
Tested-by: Maloo <maloo@whamcloud.com>
Reviewed-by: Andreas Dilger <adilger@whamcloud.com>
lustre/include/lustre_crypto.h
lustre/llite/crypto.c
lustre/llite/dir.c
lustre/llite/file.c
lustre/llite/llite_lib.c
lustre/llite/namei.c
lustre/llite/pcc.c
lustre/llite/symlink.c
lustre/osc/osc_request.c
lustre/tests/sanity-pcc.sh
lustre/utils/liblustreapi_pcc.c

index b6c1c8a..e8780c5 100644 (file)
@@ -102,6 +102,22 @@ static inline int critical_decode(const u8 *src, int len, char *dst)
        return (char *)q - dst;
 }
 
+/* Used for support of PCC-RO for encrypted files.
+ * In case an encrypted file is put in PCC-RO, we want it to be in ciphertext.
+ * The way to achieve this is to set the S_PCCCOPY flag on the inode, so that
+ * we directly return -ENOKEY in this case. This makes the lustre inode be
+ * treated as if we do not have the key, hence fetching ciphertext content
+ * instead of decrypting it.
+ * S_PCCCOPY is actually S_DIRSYNC, as it is safe so far to use it on regular
+ * files in this scenario.
+ */
+#define S_PCCCOPY S_DIRSYNC
+#define IS_PCCCOPY(inode)      ((inode)->i_flags & S_PCCCOPY)
+#define ll_require_key(inode)  \
+       (IS_PCCCOPY(inode) ? -ENOKEY : llcrypt_require_key(inode))
+#define ll_has_encryption_key(inode)   \
+       (IS_PCCCOPY(inode) ? false : llcrypt_has_encryption_key(inode))
+
 #ifdef CONFIG_LL_ENCRYPTION
 #include <libcfs/crypto/llcrypt.h>
 #else /* !CONFIG_LL_ENCRYPTION */
index e1f95c3..b0f8e74 100644 (file)
@@ -420,7 +420,7 @@ int ll_revalidate_d_crypto(struct dentry *dentry, unsigned int flags)
 
        dir = dget_parent(dentry);
        err = llcrypt_get_encryption_info(d_inode(dir));
-       valid = !llcrypt_has_encryption_key(d_inode(dir));
+       valid = !ll_has_encryption_key(d_inode(dir));
        dput(dir);
 
        if (err < 0)
index 50c5853..799264a 100644 (file)
@@ -1857,7 +1857,7 @@ out_rmdir:
                        st.st_uid       = body->mbo_uid;
                        st.st_gid       = body->mbo_gid;
                        st.st_rdev      = body->mbo_rdev;
-                       if (llcrypt_require_key(inode) == -ENOKEY)
+                       if (ll_require_key(inode) == -ENOKEY)
                                st.st_size = round_up(st.st_size,
                                                   LUSTRE_ENCRYPTION_UNIT_SIZE);
                        else
@@ -1886,7 +1886,7 @@ out_rmdir:
                        stx.stx_ino = cl_fid_build_ino(&body->mbo_fid1,
                                                       sbi->ll_flags &
                                                       LL_SBI_32BIT_API);
-                       if (llcrypt_require_key(inode) == -ENOKEY)
+                       if (ll_require_key(inode) == -ENOKEY)
                                stx.stx_size = round_up(stx.stx_size,
                                                   LUSTRE_ENCRYPTION_UNIT_SIZE);
                        else
index f66f5a3..041951d 100644 (file)
@@ -111,11 +111,16 @@ static void ll_prepare_close(struct inode *inode, struct md_op_data *op_data,
         * stored into lli_lazysize in ll_merge_attr(), so set proper file size
         * now that we are closing.
         */
-       if (llcrypt_require_key(inode) == -ENOKEY &&
-           ll_i2info(inode)->lli_attr_valid & OBD_MD_FLLAZYSIZE)
+       if (ll_require_key(inode) == -ENOKEY &&
+           ll_i2info(inode)->lli_attr_valid & OBD_MD_FLLAZYSIZE) {
                op_data->op_attr.ia_size = ll_i2info(inode)->lli_lazysize;
-       else
+               if (IS_PCCCOPY(inode)) {
+                       inode->i_flags &= ~S_PCCCOPY;
+                       i_size_write(inode, op_data->op_attr.ia_size);
+               }
+       } else {
                op_data->op_attr.ia_size = i_size_read(inode);
+       }
        op_data->op_attr.ia_valid |= (ATTR_MODE | ATTR_ATIME | ATTR_ATIME_SET |
                                      ATTR_MTIME | ATTR_MTIME_SET |
                                      ATTR_CTIME);
@@ -461,7 +466,7 @@ static inline int ll_dom_readpage(void *data, struct page *page)
        kunmap_atomic(kaddr);
 
        if (inode && IS_ENCRYPTED(inode) && S_ISREG(inode->i_mode)) {
-               if (!llcrypt_has_encryption_key(inode))
+               if (!ll_has_encryption_key(inode))
                        CDEBUG(D_SEC, "no enc key for "DFID"\n",
                               PFID(ll_inode2fid(inode)));
                else {
@@ -1481,7 +1486,7 @@ int ll_merge_attr(const struct lu_env *env, struct inode *inode)
        CDEBUG(D_VFSTRACE, DFID" updating i_size %llu\n",
               PFID(&lli->lli_fid), attr->cat_size);
 
-       if (llcrypt_require_key(inode) == -ENOKEY) {
+       if (ll_require_key(inode) == -ENOKEY) {
                /* Without the key, round up encrypted file size to next
                 * LUSTRE_ENCRYPTION_UNIT_SIZE. Clear text size is put in
                 * lli_lazysize for proper file size setting at close time.
@@ -3736,6 +3741,7 @@ static long ll_file_unlock_lease(struct file *file, struct ll_ioc_lease *ioc,
                if (ioc->lil_count != 1)
                        RETURN(-EINVAL);
 
+               /* PCC-RW is not supported for encrypted files. */
                if (IS_ENCRYPTED(inode))
                        RETURN(-EOPNOTSUPP);
 
@@ -4326,9 +4332,6 @@ out_ladvise:
                if (!pcc_inode_permission(inode))
                        RETURN(-EPERM);
 
-               if (IS_ENCRYPTED(inode))
-                       RETURN(-EOPNOTSUPP);
-
                OBD_ALLOC_PTR(attach);
                if (attach == NULL)
                        RETURN(-ENOMEM);
@@ -4338,6 +4341,14 @@ out_ladvise:
                                   sizeof(*attach)))
                        GOTO(out_pcc, rc = -EFAULT);
 
+               /* We only support pcc for encrypted files if we have the
+                * encryption key and if it is PCC-RO.
+                */
+               if (IS_ENCRYPTED(inode) &&
+                   (!llcrypt_has_encryption_key(inode) ||
+                    attach->pcca_type != LU_PCC_READONLY))
+                       GOTO(out_pcc, rc = -EOPNOTSUPP);
+
                rc = pcc_ioctl_attach(file, inode, attach);
 out_pcc:
                OBD_FREE_PTR(attach);
@@ -4473,7 +4484,7 @@ loff_t ll_lseek(struct file *file, loff_t offset, int whence)
        /* Without the key, SEEK_HOLE return value has to be
         * rounded up to next LUSTRE_ENCRYPTION_UNIT_SIZE.
         */
-       if (llcrypt_require_key(inode) == -ENOKEY && whence == SEEK_HOLE)
+       if (ll_require_key(inode) == -ENOKEY && whence == SEEK_HOLE)
                retval = round_up(retval, LUSTRE_ENCRYPTION_UNIT_SIZE);
 
        RETURN(retval);
index 26065ad..c525ab1 100644 (file)
@@ -2169,7 +2169,7 @@ int ll_setattr_raw(struct dentry *dentry, struct iattr *attr,
                                if (filename_is_volatile(dentry->d_name.name,
                                                         dentry->d_name.len,
                                                         NULL) &&
-                                   llcrypt_require_key(inode) == -ENOKEY) {
+                                   ll_require_key(inode) == -ENOKEY) {
                                        struct file *ref_file;
                                        struct inode *ref_inode;
                                        struct ll_inode_info *ref_lli;
@@ -2575,7 +2575,7 @@ int ll_update_inode(struct inode *inode, struct lustre_md *md)
         * we will need it in ll_prepare_close().
         */
        if (lli->lli_attr_valid & OBD_MD_FLLAZYSIZE && lli->lli_lazysize &&
-           llcrypt_require_key(inode) == -ENOKEY)
+           ll_require_key(inode) == -ENOKEY)
                lli->lli_attr_valid = body->mbo_valid | OBD_MD_FLLAZYSIZE;
        else
                lli->lli_attr_valid = body->mbo_valid;
index aff3086..ec66bd7 100644 (file)
@@ -754,7 +754,7 @@ static int ll_lookup_it_finish(struct ptlrpc_request *request,
                        rc = llcrypt_get_encryption_info(inode);
                        if (rc)
                                GOTO(out, rc);
-                       if (!llcrypt_has_encryption_key(inode))
+                       if (!ll_has_encryption_key(inode))
                                GOTO(out, rc = -ENOKEY);
                }
        } else if (!it_disposition(it, DISP_OPEN_CREATE)) {
@@ -1239,7 +1239,7 @@ static int ll_atomic_open(struct inode *dir, struct dentry *dentry,
                        /* For migration or mirroring without enc key, we still
                         * need to be able to create a volatile file.
                         */
-                       if (!llcrypt_has_encryption_key(dir) &&
+                       if (!ll_has_encryption_key(dir) &&
                            (!filename_is_volatile(dentry->d_name.name,
                                                   dentry->d_name.len, NULL) ||
                            (open_flags & O_FILE_ENC) != O_FILE_ENC ||
index 0424147..2253c29 100644 (file)
@@ -1353,8 +1353,18 @@ static void pcc_inode_get(struct pcc_inode *pcci)
 
 static void pcc_inode_put(struct pcc_inode *pcci)
 {
-       if (atomic_dec_and_test(&pcci->pcci_refcount))
+       if (atomic_dec_and_test(&pcci->pcci_refcount)) {
+               struct inode *inode = &pcci->pcci_lli->lli_vfs_inode;
+               struct inode *pcc_inode = pcci->pcci_path.dentry->d_inode;
+
+               if (inode && IS_ENCRYPTED(inode) && pcc_inode)
+                       /* get rid of all page cache pages for this inode,
+                        * as they contain clear text data
+                        */
+                       truncate_inode_pages_final(pcc_inode->i_mapping);
+
                pcc_inode_fini(pcci);
+       }
 }
 
 void pcc_inode_free(struct inode *inode)
@@ -1449,6 +1459,38 @@ static int pcc_layout_xattr_set(struct pcc_inode *pcci, __u32 gen)
        RETURN(rc);
 }
 
+/* xattr to store encrypted file's size
+ *
+ * This is required because in case of encrypted inode, the PCC file contains
+ * the ciphertext. This means its size is aligned on LUSTRE_ENCRYPTION_UNIT_SIZE
+ * instead of being lustre inode's clear text size.
+ */
+static const char pcc_xattr_encsize[] = XATTR_USER_PREFIX "PCC.encsize";
+
+static int pcc_encsize_xattr_set(struct pcc_inode *pcci)
+{
+       struct dentry *pcc_dentry = pcci->pcci_path.dentry;
+       struct inode *inode = &pcci->pcci_lli->lli_vfs_inode;
+       loff_t size;
+       int rc;
+
+       ENTRY;
+
+       if (!IS_ENCRYPTED(inode))
+               RETURN(0);
+
+       if (ll_require_key(inode) == -ENOKEY &&
+           pcci->pcci_lli->lli_attr_valid & OBD_MD_FLLAZYSIZE)
+               size = pcci->pcci_lli->lli_lazysize;
+       else
+               size = inode->i_size;
+
+       rc = ll_vfs_setxattr(pcc_dentry, pcc_dentry->d_inode, pcc_xattr_encsize,
+                            &size, sizeof(size), 0);
+
+       RETURN(rc);
+}
+
 static int pcc_get_layout_info(struct inode *inode, struct cl_layout *clt)
 {
        struct lu_env *env;
@@ -2304,12 +2346,17 @@ int pcc_file_open(struct inode *inode, struct file *file)
        if (!S_ISREG(inode->i_mode))
                RETURN(0);
 
-       if (IS_ENCRYPTED(inode))
-               RETURN(0);
-
        pcc_inode_lock(inode);
        pcci = ll_i2pcci(inode);
 
+       /* We only support pcc for encrypted files if we have the encryption key
+        * and if it is PCC-RO.
+        */
+       if (IS_ENCRYPTED(inode) &&
+           (!llcrypt_has_encryption_key(inode) ||
+            (pcci && pcci->pcci_type != LU_PCC_READONLY)))
+               GOTO(out_unlock, rc = 0);
+
        if (lli->lli_pcc_state & PCC_STATE_FL_ATTACHING) {
                pcc_file_fallback_set(lli, pccf);
                GOTO(out_unlock, rc = 0);
@@ -2436,7 +2483,10 @@ ssize_t pcc_file_read_iter(struct kiocb *iocb,
        struct file *file = iocb->ki_filp;
        struct inode *inode = file_inode(file);
        struct pcc_file *pccf = ll_file2pccf(file);
-       ssize_t result;
+       unsigned int blockbits = 0, blocksize = 0;
+       pgoff_t start_index, end_index, index;
+       ssize_t result = 0;
+       int rc = 0;
 
        ENTRY;
 
@@ -2449,18 +2499,104 @@ ssize_t pcc_file_read_iter(struct kiocb *iocb,
        if (!*cached)
                RETURN(0);
 
-       /* Fake I/O error on RO-PCC */
+       /* Fake I/O error on PCC-RO */
        if (OBD_FAIL_CHECK(OBD_FAIL_LLITE_PCC_FAKE_ERROR))
                GOTO(out, result = -EIO);
 
        iocb->ki_filp = pccf->pccf_file;
-       /* generic_file_aio_read does not support ext4-dax,
-        * __pcc_file_read_iter uses ->aio_read hook directly
-        * to add support for ext4-dax.
-        */
+       if (!IS_ENCRYPTED(inode)) {
+               /* generic_file_aio_read does not support ext4-dax,
+                * __pcc_file_read_iter uses ->aio_read hook directly
+                * to add support for ext4-dax.
+                */
+               result = __pcc_file_read_iter(iocb, iter);
+               GOTO(out, result);
+       }
+
+       /* from this point, we are dealing with an encrypted inode */
+       blockbits = inode->i_blkbits;
+       blocksize = 1 << blockbits;
+       start_index = iocb->ki_pos >> PAGE_SHIFT;
+       if (i_size_read(inode) == 0)
+               end_index = (iocb->ki_pos + (loff_t)iov_iter_count(iter) - 1)
+                       >> PAGE_SHIFT;
+       else
+               end_index = (min(iocb->ki_pos + (loff_t)iov_iter_count(iter),
+                                i_size_read(inode)) - 1) >> PAGE_SHIFT;
+
+       /* Proceed to decryption of PCC-RO page cache pages */
+       for (index = start_index; index <= end_index; index++) {
+               struct address_space *mapping;
+               struct page *vmpage = NULL;
+               unsigned int offs = 0;
+
+               mapping = file_inode(pccf->pccf_file)->i_mapping;
+               vmpage = grab_cache_page(mapping, index);
+               if (vmpage == NULL)
+                       continue;
+
+               /* vmpage has already been decrypted */
+               if (PagePrivate2(vmpage))
+                       goto out_pageprivate2;
+
+               if (PageDirty(vmpage))
+                       /* this should not happen with PCC-RO */
+                       GOTO(out_pageprivate2, rc = -EIO);
+               if (!PageUptodate(vmpage)) {
+                       rc = mapping->a_ops->readpage(pccf->pccf_file, vmpage);
+                       if (rc) {
+                               put_page(vmpage);
+                               continue;
+                       }
+                       lock_page(vmpage);
+                       if (!PageUptodate(vmpage))
+                               GOTO(out_pageprivate2, rc = -EIO);
+               }
+
+               while (offs < PAGE_SIZE) {
+                       u64 lblk_num = ((u64)vmpage->index <<
+                                       (PAGE_SHIFT - blockbits)) +
+                                      (offs >> blockbits);
+                       unsigned int i;
+
+                       /* do not decrypt if page is all 0s */
+                       if (memchr_inv(page_address(vmpage) + offs, 0,
+                                      LUSTRE_ENCRYPTION_UNIT_SIZE) ==
+                           NULL)
+                               break;
+
+                       for (i = offs;
+                            i < offs + LUSTRE_ENCRYPTION_UNIT_SIZE;
+                            i += blocksize, lblk_num++) {
+                               rc = llcrypt_decrypt_block_inplace(inode,
+                                                                  vmpage,
+                                                                  blocksize, i,
+                                                                  lblk_num);
+                               if (rc)
+                                       break;
+                       }
+                       if (rc)
+                               GOTO(out_pageprivate2, rc);
+
+                       offs += LUSTRE_ENCRYPTION_UNIT_SIZE;
+               }
+               /* set PagePrivate2 flag so that we know
+                * this page is now decrypted
+                */
+               SetPagePrivate2(vmpage);
+
+out_pageprivate2:
+               unlock_page(vmpage);
+               put_page(vmpage);
+
+       }
+
        result = __pcc_file_read_iter(iocb, iter);
-       iocb->ki_filp = file;
+       if (iocb->ki_pos > i_size_read(inode) && result > 0)
+               result -= iocb->ki_pos - i_size_read(inode);
+
 out:
+       iocb->ki_filp = file;
        pcc_io_fini(inode, PIT_READ, result, cached);
        RETURN(result);
 }
@@ -2580,7 +2716,9 @@ int pcc_inode_getattr(struct inode *inode, u32 request_mask,
 {
        struct ll_inode_info *lli = ll_i2info(inode);
        const struct cred *old_cred;
+       struct pcc_inode *pcci;
        struct kstat stat;
+       loff_t size;
        s64 atime;
        s64 mtime;
        s64 ctime;
@@ -2598,7 +2736,8 @@ int pcc_inode_getattr(struct inode *inode, u32 request_mask,
                RETURN(0);
 
        old_cred = override_creds(pcc_super_cred(inode->i_sb));
-       rc = ll_vfs_getattr(&ll_i2pcci(inode)->pcci_path, &stat, request_mask,
+       pcci = ll_i2pcci(inode);
+       rc = ll_vfs_getattr(&pcci->pcci_path, &stat, request_mask,
                            flags);
        revert_creds(old_cred);
        if (rc)
@@ -2625,7 +2764,19 @@ int pcc_inode_getattr(struct inode *inode, u32 request_mask,
        if (mtime < stat.mtime.tv_sec)
                mtime = stat.mtime.tv_sec;
 
-       i_size_write(inode, stat.size);
+       size = stat.size;
+       /* The pcc_xattr_encsize xattr is only valid for PCC-RO. */
+       if (IS_ENCRYPTED(inode) && pcci->pcci_type == LU_PCC_READONLY) {
+               loff_t encsize;
+
+               rc = ll_vfs_getxattr(pcci->pcci_path.dentry,
+                                    pcci->pcci_path.dentry->d_inode,
+                                    pcc_xattr_encsize,
+                                    &encsize, sizeof(encsize));
+               if (rc > 0)
+                       size = encsize;
+       }
+       i_size_write(inode, size);
        inode->i_blocks = stat.blocks;
 
        inode->i_atime.tv_sec = atime;
@@ -2687,9 +2838,9 @@ int pcc_fsync(struct file *file, loff_t start, loff_t end,
        }
 
        /**
-        * After the file is attached into RO-PCC, its dirty pages on this
+        * After the file is attached into PCC-RO, its dirty pages on this
         * client may not be flushed. So fsync() should fall back to normal
-        * Lustre I/O path flushing dirty data to OSTs. And flush on RO-PCC
+        * Lustre I/O path flushing dirty data to OSTs. And flush on PCC-RO
         * copy is meaningless.
         */
        if (pccf->pccf_type == LU_PCC_READONLY) {
@@ -3164,7 +3315,7 @@ int pcc_fault(struct vm_area_struct *vma, struct vm_fault *vmf,
        if (!*cached)
                RETURN(0);
 
-       /* Tolerate the mmap read failure for RO-PCC */
+       /* Tolerate the mmap read failure for PCC-RO */
        if (OBD_FAIL_CHECK(OBD_FAIL_LLITE_PCC_FAKE_ERROR))
                GOTO(out, rc = VM_FAULT_SIGBUS);
 
@@ -3468,6 +3619,8 @@ int pcc_inode_create_fini(struct inode *inode, struct pcc_create_attach *pca)
        /* Set the layout generation of newly created file with 0 */
        pcc_layout_gen_set(pcci, 0);
 
+       rc = pcc_encsize_xattr_set(pcci);
+
 out_put:
        if (rc) {
                (void) pcc_inode_remove(inode, pcc_dentry);
@@ -3530,6 +3683,7 @@ static ssize_t pcc_copy_data(struct file *src, struct file *dst)
        ssize_t rc2;
        loff_t pos, offset = 0;
        size_t buf_len = 1048576;
+       struct inode *inode = file_inode(src);
        void *buf;
 
        ENTRY;
@@ -3552,6 +3706,13 @@ static ssize_t pcc_copy_data(struct file *src, struct file *dst)
                        GOTO(out_free, rc = -EINTR);
 
                pos = offset;
+               if (inode && IS_ENCRYPTED(inode))
+                       /* Setting the S_PCCCOPY flag prevents the Lustre file
+                        * from being decrypted in the OSC layer, so that the
+                        * PCC file contains ciphertext data.
+                        * S_PCCCOPY flag is removed in ll_prepare_close().
+                        */
+                       inode->i_flags |= S_PCCCOPY;
                rc2 = cfs_kernel_read(src, buf, buf_len, &pos);
                if (rc2 < 0)
                        GOTO(out_free, rc = rc2);
@@ -3581,6 +3742,7 @@ static int pcc_attach_data_archive(struct file *file, struct inode *inode,
        struct path path;
        ktime_t kstart = ktime_get();
        ssize_t ret;
+       int flags = O_WRONLY | O_LARGEFILE;
        int rc;
 
        ENTRY;
@@ -3592,7 +3754,14 @@ static int pcc_attach_data_archive(struct file *file, struct inode *inode,
 
        path.mnt = dataset->pccd_path.mnt;
        path.dentry = *dentry;
-       pcc_filp = dentry_open(&path, O_WRONLY | O_LARGEFILE, current_cred());
+       /* If the inode is encrypted, we want the PCC file to be synced to the
+        * storage. This is necessary as we are going to decrypt the page cache
+        * pages of the PCC inode later in pcc_file_read_iter(), but still we
+        * need to keep the ciphertext version on disk.
+        */
+       if (IS_ENCRYPTED(inode))
+               flags |= O_SYNC;
+       pcc_filp = dentry_open(&path, flags, current_cred());
        if (IS_ERR_OR_NULL(pcc_filp)) {
                rc = pcc_filp == NULL ? -EINVAL : PTR_ERR(pcc_filp);
                GOTO(out_dentry, rc);
@@ -3912,6 +4081,8 @@ static int pcc_readonly_attach(struct file *file,
        }
 
        pcc_layout_gen_set(pcci, gen);
+
+       rc = pcc_encsize_xattr_set(pcci);
 out_put_unlock:
        if (rc) {
                if (!unlinked)
index 37e26da..989be43 100644 (file)
@@ -130,7 +130,7 @@ static int ll_readlink_internal(struct inode *inode,
                /* Do not cache symlink targets encoded without the key,
                 * since those become outdated once the key is added.
                 */
-               if (!llcrypt_has_encryption_key(inode))
+               if (!ll_has_encryption_key(inode))
                        RETURN(0);
        }
 #endif
index 4a05698..4b85e9b 100644 (file)
@@ -1536,7 +1536,7 @@ retry_encrypt:
                if (attr.cat_size)
                        oa->o_size = attr.cat_size;
        } else if (opc == OST_READ && inode && IS_ENCRYPTED(inode) &&
-                  llcrypt_has_encryption_key(inode)) {
+                  ll_has_encryption_key(inode)) {
                for (i = 0; i < page_count; i++) {
                        struct brw_page *pg = pga[i];
                        u32 nunits = (pg->off & ~PAGE_MASK) + pg->count;
@@ -1568,7 +1568,7 @@ retry_encrypt:
        for (i = 0; i < page_count; i++) {
                short_io_size += pga[i]->count;
                if (!inode || !IS_ENCRYPTED(inode) ||
-                   !llcrypt_has_encryption_key(inode)) {
+                   !ll_has_encryption_key(inode)) {
                        pga[i]->bp_count_diff = 0;
                        pga[i]->bp_off_diff = 0;
                }
@@ -2180,7 +2180,7 @@ static int osc_brw_fini_request(struct ptlrpc_request *req, int rc)
        if (inode && IS_ENCRYPTED(inode)) {
                int idx;
 
-               if (!llcrypt_has_encryption_key(inode)) {
+               if (!ll_has_encryption_key(inode)) {
                        CDEBUG(D_SEC, "no enc key for ino %lu\n", inode->i_ino);
                        GOTO(out, rc);
                }
index d469ff8..feb5dbf 100644 (file)
@@ -114,7 +114,7 @@ lpcc_fid2path()
 {
        local hsm_root="$1"
        local lustre_path="$2"
-       local fid=$(path2fid $lustre_path)
+       local fid=${3:-$(path2fid $lustre_path)}
 
        local seq=$(echo $fid | awk -F ':' '{print $1}')
        local oid=$(echo $fid | awk -F ':' '{print $2}')
@@ -241,6 +241,45 @@ setup_loopdev_project() {
        do_facet $facet mount | grep $mntpt
 }
 
+setup_dummy_key() {
+       local mode='\x00\x00\x00\x00'
+       local raw="$(printf ""\\\\x%02x"" {0..63})"
+       local size
+       local key
+
+       [[ $(lscpu) =~ Byte\ Order.*Little ]] && size='\x40\x00\x00\x00' ||
+               size='\x00\x00\x00\x40'
+       key="${mode}${raw}${size}"
+       do_facet $SINGLEAGT "echo -en '${key}' |
+               keyctl padd logon fscrypt:4242424242424242 @u"
+}
+
+setup_for_enc_tests() {
+       local agthost=${SINGLEAGT}_HOST
+
+       # remount client with test_dummy_encryption option
+       zconf_umount ${!agthost} $MOUNT ||
+               error "umount $SINGLEAGT $MOUNT failed"
+       zconf_mount ${!agthost} $MOUNT ${MOUNT_OPTS},test_dummy_encryption ||
+               error "mount $SINGLEAGT with '-o test_dummy_encryption' failed"
+
+       setup_dummy_key
+
+       # this directory will be encrypted, because of dummy mode
+       do_facet $SINGLEAGT mkdir $DIR/$tdir || error "mkdir $DIR/$tdir failed"
+}
+
+cleanup_for_enc_tests() {
+       local agthost=${SINGLEAGT}_HOST
+
+       do_facet $SINGLEAGT rm -rf $DIR/$tdir
+       # remount client normally
+       zconf_umount ${!agthost} $MOUNT ||
+               error "umount $SINGLEAGT $MOUNT failed"
+       zconf_mount ${!agthost} $MOUNT ||
+               error "remount $SINGLEAGT failed"
+}
+
 lpcc_rw_test() {
        local restore="$1"
        local project="$2"
@@ -2057,6 +2096,79 @@ test_21i() {
 }
 run_test 21i "HSM release increase layout gen, should invalidate PCC-RO cache"
 
+test_21j() {
+       local loopfile="$TMP/$tfile"
+       local mntpt="/mnt/pcc.$tdir"
+       local hsm_root="$mntpt/$tdir"
+       local tmpfile=$TMP/abc
+       local file=$DIR/$tdir/$tfile
+       local scrambledfile
+       local lpcc_path
+       local size
+       local fid
+       local key
+
+       $LCTL get_param -n mdc.*.connect_flags | grep -q pcc_ro ||
+               skip "Server does not support PCC-RO"
+
+       $LCTL get_param mdc.*.import | grep -q client_encryption ||
+               skip "client encryption not supported"
+
+       mount.lustre --help |& grep -q "test_dummy_encryption:" ||
+               skip "need dummy encryption support"
+
+       setup_loopdev $SINGLEAGT $loopfile $mntpt 50
+       stack_trap cleanup_for_enc_tests EXIT
+       setup_for_enc_tests
+       copytool setup -m "$MOUNT" -a "$HSM_ARCHIVE_NUMBER"
+       setup_pcc_mapping
+
+       do_facet $SINGLEAGT "yes 1 | dd of=$tmpfile bs=1 count=5000 conv=fsync"
+       do_facet $SINGLEAGT cp $tmpfile $file
+       do_facet $SINGLEAGT $LFS getstripe $file
+       do_facet $SINGLEAGT $LFS pcc attach -r -i $HSM_ARCHIVE_NUMBER $file ||
+               error "failed to PCC-RO attach file $file"
+       check_lpcc_state $file "readonly"
+       echo "PCC-RO attach '$file':"
+       do_facet $SINGLEAGT $LFS getstripe -v $file
+
+       do_facet $SINGLEAGT cmp -bl $tmpfile $file ||
+               error "file $file is corrupted (1)"
+       fid=$(do_facet $SINGLEAGT lfs path2fid $file | tr -d '[]')
+       lpcc_path=$(lpcc_fid2path $hsm_root $file $fid)
+       do_facet $SINGLEAGT cmp -bl -n 4096 $tmpfile $lpcc_path ||
+               error "file $lpcc_path is corrupted (2)"
+
+       do_facet $SINGLEAGT $LFS pcc detach -k $file ||
+               error "failed to PCC-RO detach file $file"
+       do_facet $SINGLEAGT cmp -s -n 4096 $tmpfile $lpcc_path &&
+               error "file $lpcc_path is corrupted (3)"
+
+       size=$(do_facet $SINGLEAGT stat "--printf=%s" $lpcc_path)
+       [ $size == 8192 ] || error "PCC file $lpcc_path incorrect size $size"
+
+       do_facet $SINGLEAGT cmp -bl $tmpfile $file ||
+               error "file $file is corrupted (4)"
+       do_facet $SINGLEAGT cmp -bl -n 4096 $tmpfile $lpcc_path ||
+               error "file $lpcc_path is corrupted (5)"
+
+       # remove fscrypt key from keyring
+       key=$(do_facet $SINGLEAGT keyctl show |
+                                 awk '$7 ~ "^fscrypt:" {print $1}')
+       [ -n "$key" ] || error "fscrypt key empty on $SINGLEAGT"
+       do_facet $SINGLEAGT keyctl revoke $key
+       do_facet $SINGLEAGT keyctl reap
+       do_facet $SINGLEAGT $LCTL set_param -n ldlm.namespaces.*.lru_size=clear
+
+       scrambledfile=$(do_facet $SINGLEAGT find $DIR/$tdir/ \
+                                           -maxdepth 1 -mindepth 1 -type f)
+       do_facet $SINGLEAGT $LFS pcc detach -k $scrambledfile ||
+               error "failed to PCC-RO detach file $scrambledfile (2)"
+
+       do_facet $SINGLEAGT rm -f $tmpfile
+}
+run_test 21j "PCC-RO for encrypted file"
+
 test_22() {
        local loopfile="$TMP/$tfile"
        local mntpt="/mnt/pcc.$tdir"
@@ -2726,7 +2838,7 @@ test_31() {
 
        file=$DIR/$tdir/roattach
        echo -n backend_del_roattach_rm > $file
-       lpcc_path3=$(lpcc_fid2path $hsm_root $file "readonly")
+       lpcc_path3=$(lpcc_fid2path $hsm_root $file)
        do_facet $SINGLEAGT $LFS pcc attach -r -i $HSM_ARCHIVE_NUMBER $file ||
                error "RO-PCC attach $file failed"
        check_lpcc_state $file "readonly"
@@ -2764,7 +2876,7 @@ test_32() {
                "projid={100}\ rwid=$HSM_ARCHIVE_NUMBER\ auto_attach=0"
 
        do_facet $SINGLEAGT echo -n roattach_removed > $file
-       lpcc_path=$(lpcc_fid2path $hsm_root $file "readonly")
+       lpcc_path=$(lpcc_fid2path $hsm_root $file)
        do_facet $SINGLEAGT $LFS pcc attach -r -i $HSM_ARCHIVE_NUMBER $file ||
                error "RO-PCC attach $file failed"
        rmultiop_start $agt_host $file o_rc || error "multiop $file failed"
index faf8da3..8e63193 100644 (file)
@@ -365,7 +365,10 @@ int llapi_pcc_detach_file(const char *path, __u32 flags)
        int rc;
        int fd;
 
-       fd = open(path, O_RDWR | O_NONBLOCK);
+       /* Specify O_FILE_ENC | O_DIRECT flags to allow pcc detach
+        * on encrypted file without the key.
+        */
+       fd = open(path, O_RDWR | O_NONBLOCK | O_FILE_ENC | O_DIRECT);
        if (fd < 0) {
                rc = -errno;
                llapi_error(LLAPI_MSG_ERROR, rc, "cannot open '%s'",