From 9e37963fa05b66652610771740512971c65a47ef Mon Sep 17 00:00:00 2001 From: Sebastien Buisson Date: Mon, 20 Dec 2021 17:41:56 +0100 Subject: [PATCH] LU-10499 sec: support of PCC-RO for encrypted files In order to support PCC-RO for encrypted files, we decide to store in PCC the ciphertext version of the Lustre files. We proceed to decryption of PCC files only in the page cache, so cleartext is just in memory. When a Lustre file is detached from PCC, or when the encryption key is removed, we trash those PCC page cache pages. As PCC files contain ciphertext, their sizes are aligned on LUSTRE_ENCRYPTION_UNIT_SIZE instead of being lustre inode's clear text size. In order to keep track of Lustre files' actual sizes, we use a dedicated xattr on the PCC files. Its value is set at pcc attach time, which is fine for PCC-RO. Also add sanity-pcc test_21j to exercise this. EX-5427 sec: fix pcc detach of encrypted file In case of 'lfs pcc detach' of an encrypted file, its page cache pages must be trashed as they might contain cipher text because of the pcc file being detached. Improve sanity-pcc test_21j to exercise this use case. Was-Change-Id: Ice13c5b4205c074d9b46e2175d18f8743dbe9c58 EX-bug-id: EX-4182 EX-5427 Signed-off-by: Sebastien Buisson Change-Id: Id7d96adb4eb4770c813a042acf7ed6c42224b9bf Reviewed-on: https://review.whamcloud.com/c/fs/lustre-release/+/54459 Tested-by: jenkins Tested-by: Maloo Reviewed-by: Andreas Dilger Reviewed-by: James Simmons Reviewed-by: Oleg Drokin --- lustre/include/lustre_crypto.h | 16 +++ lustre/llite/crypto.c | 2 +- lustre/llite/dir.c | 4 +- lustre/llite/file.c | 26 +++-- lustre/llite/llite_lib.c | 4 +- lustre/llite/pcc.c | 218 ++++++++++++++++++++++++++++++++++++---- lustre/llite/symlink.c | 2 +- lustre/osc/osc_request.c | 6 +- lustre/tests/sanity-pcc.sh | 132 +++++++++++++++++++++++- lustre/utils/liblustreapi_pcc.c | 5 +- 10 files changed, 378 insertions(+), 37 deletions(-) diff --git a/lustre/include/lustre_crypto.h b/lustre/include/lustre_crypto.h index b02cd64..7f3453e 100644 --- a/lustre/include/lustre_crypto.h +++ b/lustre/include/lustre_crypto.h @@ -244,4 +244,20 @@ static inline int critical_decode(const u8 *src, int len, char *dst) return (char *)q - dst; } +/* Used for support of PCC-RO for encrypted files. + * In case an encrypted file is put in PCC-RO, we want it to be in ciphertext. + * The way to achieve this is to set the S_PCCCOPY flag on the inode, so that + * we directly return -ENOKEY in this case. This makes the lustre inode be + * treated as if we do not have the key, hence fetching ciphertext content + * instead of decrypting it. + * S_PCCCOPY is actually S_DIRSYNC, as it is safe so far to use it on regular + * files in this scenario. + */ +#define S_PCCCOPY S_DIRSYNC +#define IS_PCCCOPY(inode) ((inode)->i_flags & S_PCCCOPY) +#define ll_require_key(inode) \ + (IS_PCCCOPY(inode) ? -ENOKEY : llcrypt_require_key(inode)) +#define ll_has_encryption_key(inode) \ + (IS_PCCCOPY(inode) ? false : llcrypt_has_encryption_key(inode)) + #endif /* _LUSTRE_CRYPTO_H_ */ diff --git a/lustre/llite/crypto.c b/lustre/llite/crypto.c index 32fcd5b..f3e41a7 100644 --- a/lustre/llite/crypto.c +++ b/lustre/llite/crypto.c @@ -635,7 +635,7 @@ int llcrypt_d_revalidate(struct dentry *dentry, unsigned int flags) dir = dget_parent(dentry); err = llcrypt_prepare_readdir(d_inode(dir)); - valid = !llcrypt_has_encryption_key(d_inode(dir)); + valid = !ll_has_encryption_key(d_inode(dir)); dput(dir); if (err < 0) diff --git a/lustre/llite/dir.c b/lustre/llite/dir.c index c8ac8ed..77d5b93 100644 --- a/lustre/llite/dir.c +++ b/lustre/llite/dir.c @@ -2381,7 +2381,7 @@ out_rmdir: st.st_uid = body->mbo_uid; st.st_gid = body->mbo_gid; st.st_rdev = body->mbo_rdev; - if (llcrypt_require_key(inode) == -ENOKEY) + if (ll_require_key(inode) == -ENOKEY) st.st_size = round_up(st.st_size, LUSTRE_ENCRYPTION_UNIT_SIZE); else @@ -2408,7 +2408,7 @@ out_rmdir: stx.stx_mode = body->mbo_mode; stx.stx_ino = cl_fid_build_ino(&body->mbo_fid1, api32); - if (llcrypt_require_key(inode) == -ENOKEY) + if (ll_require_key(inode) == -ENOKEY) stx.stx_size = round_up(stx.stx_size, LUSTRE_ENCRYPTION_UNIT_SIZE); else diff --git a/lustre/llite/file.c b/lustre/llite/file.c index c6f777c..f1ccf66 100644 --- a/lustre/llite/file.c +++ b/lustre/llite/file.c @@ -116,11 +116,16 @@ static void ll_prepare_close(struct inode *inode, struct md_op_data *op_data, * stored into lli_lazysize in ll_merge_attr(), so set proper file size * now that we are closing. */ - if (llcrypt_require_key(inode) == -ENOKEY && - ll_i2info(inode)->lli_attr_valid & OBD_MD_FLLAZYSIZE) + if (ll_require_key(inode) == -ENOKEY && + ll_i2info(inode)->lli_attr_valid & OBD_MD_FLLAZYSIZE) { op_data->op_attr.ia_size = ll_i2info(inode)->lli_lazysize; - else + if (IS_PCCCOPY(inode)) { + inode->i_flags &= ~S_PCCCOPY; + i_size_write(inode, op_data->op_attr.ia_size); + } + } else { op_data->op_attr.ia_size = i_size_read(inode); + } op_data->op_attr.ia_valid |= (ATTR_MODE | ATTR_ATIME | ATTR_ATIME_SET | ATTR_MTIME | ATTR_MTIME_SET | ATTR_CTIME); @@ -490,7 +495,7 @@ static inline int ll_dom_readpage(void *data, struct page *page) kunmap_atomic(kaddr); if (inode && IS_ENCRYPTED(inode) && S_ISREG(inode->i_mode)) { - if (!llcrypt_has_encryption_key(inode)) { + if (!ll_has_encryption_key(inode)) { CDEBUG(D_SEC, "no enc key for "DFID"\n", PFID(ll_inode2fid(inode))); rc = -ENOKEY; @@ -1530,7 +1535,7 @@ static int ll_merge_attr_nolock(const struct lu_env *env, struct inode *inode) CDEBUG(D_VFSTRACE, DFID" updating i_size %llu i_blocks %llu\n", PFID(&lli->lli_fid), attr->cat_size, attr->cat_blocks); - if (llcrypt_require_key(inode) == -ENOKEY) { + if (ll_require_key(inode) == -ENOKEY) { /* Without the key, round up encrypted file size to next * LUSTRE_ENCRYPTION_UNIT_SIZE. Clear text size is put in * lli_lazysize for proper file size setting at close time. @@ -4317,6 +4322,7 @@ static long ll_file_unlock_lease(struct file *file, struct ll_ioc_lease *ioc, if (ioc->lil_count != 1) RETURN(-EINVAL); + /* PCC-RW is not supported for encrypted files. */ if (IS_ENCRYPTED(inode)) RETURN(-EOPNOTSUPP); @@ -4917,6 +4923,14 @@ out_ladvise: sizeof(*attach))) GOTO(out_pcc, rc = -EFAULT); + /* We only support pcc for encrypted files if we have the + * encryption key and if it is PCC-RO. + */ + if (IS_ENCRYPTED(inode) && + (!llcrypt_has_encryption_key(inode) || + attach->pcca_type != LU_PCC_READONLY)) + GOTO(out_pcc, rc = -EOPNOTSUPP); + rc = pcc_ioctl_attach(file, inode, attach); out_pcc: OBD_FREE_PTR(attach); @@ -5023,7 +5037,7 @@ static loff_t ll_lseek(struct file *file, loff_t offset, int whence) /* Without the key, SEEK_HOLE return value has to be * rounded up to next LUSTRE_ENCRYPTION_UNIT_SIZE. */ - if (llcrypt_require_key(inode) == -ENOKEY && whence == SEEK_HOLE) + if (ll_require_key(inode) == -ENOKEY && whence == SEEK_HOLE) retval = round_up(retval, LUSTRE_ENCRYPTION_UNIT_SIZE); RETURN(retval); diff --git a/lustre/llite/llite_lib.c b/lustre/llite/llite_lib.c index 23b5e0d..b677b0f 100644 --- a/lustre/llite/llite_lib.c +++ b/lustre/llite/llite_lib.c @@ -2430,7 +2430,7 @@ int ll_setattr_raw(struct dentry *dentry, struct iattr *attr, if (filename_is_volatile(dentry->d_name.name, dentry->d_name.len, NULL) && - llcrypt_require_key(inode) == -ENOKEY) { + ll_require_key(inode) == -ENOKEY) { struct file *ref_file; struct inode *ref_inode; struct ll_inode_info *ref_lli; @@ -2899,7 +2899,7 @@ int ll_update_inode(struct inode *inode, struct lustre_md *md) * we will need it in ll_prepare_close(). */ if (lli->lli_attr_valid & OBD_MD_FLLAZYSIZE && lli->lli_lazysize && - llcrypt_require_key(inode) == -ENOKEY) + ll_require_key(inode) == -ENOKEY) lli->lli_attr_valid = body->mbo_valid | OBD_MD_FLLAZYSIZE; else lli->lli_attr_valid = body->mbo_valid; diff --git a/lustre/llite/pcc.c b/lustre/llite/pcc.c index 30fbdf5..e0531f1 100644 --- a/lustre/llite/pcc.c +++ b/lustre/llite/pcc.c @@ -1350,8 +1350,24 @@ static void pcc_inode_get(struct pcc_inode *pcci) static void pcc_inode_put(struct pcc_inode *pcci) { - if (atomic_dec_and_test(&pcci->pcci_refcount)) + if (atomic_dec_and_test(&pcci->pcci_refcount)) { + struct inode *inode = &pcci->pcci_lli->lli_vfs_inode; + struct inode *pcc_inode = pcci->pcci_path.dentry->d_inode; + + if (inode && IS_ENCRYPTED(inode) && pcc_inode) { + /* get rid of all page cache pages for this pcc inode, + * as they contain clear text data + */ + truncate_inode_pages_final(pcc_inode->i_mapping); + /* also get rid of pages cache pages for this Lustre + * inode, as they might contain cipher text because + * of the pcc file + */ + truncate_inode_pages_final(inode->i_mapping); + } + pcc_inode_fini(pcci); + } } void pcc_inode_free(struct inode *inode) @@ -1448,6 +1464,38 @@ static int pcc_layout_xattr_set(struct pcc_inode *pcci, __u32 gen) RETURN(rc); } +/* xattr to store encrypted file's size + * + * This is required because in case of encrypted inode, the PCC file contains + * the ciphertext. This means its size is aligned on LUSTRE_ENCRYPTION_UNIT_SIZE + * instead of being lustre inode's clear text size. + */ +static const char pcc_xattr_encsize[] = XATTR_USER_PREFIX "PCC.encsize"; + +static int pcc_encsize_xattr_set(struct pcc_inode *pcci) +{ + struct dentry *pcc_dentry = pcci->pcci_path.dentry; + struct inode *inode = &pcci->pcci_lli->lli_vfs_inode; + loff_t size; + int rc; + + ENTRY; + + if (!IS_ENCRYPTED(inode)) + RETURN(0); + + if (ll_require_key(inode) == -ENOKEY && + pcci->pcci_lli->lli_attr_valid & OBD_MD_FLLAZYSIZE) + size = pcci->pcci_lli->lli_lazysize; + else + size = inode->i_size; + + rc = ll_vfs_setxattr(pcc_dentry, pcc_dentry->d_inode, pcc_xattr_encsize, + &size, sizeof(size), 0); + + RETURN(rc); +} + static int pcc_get_layout_info(struct inode *inode, struct cl_layout *clt) { struct lu_env *env; @@ -2335,12 +2383,17 @@ int pcc_file_open(struct inode *inode, struct file *file) if (!S_ISREG(inode->i_mode)) RETURN(0); - if (IS_ENCRYPTED(inode)) - RETURN(0); - pcc_inode_lock(inode); pcci = ll_i2pcci(inode); + /* We only support pcc for encrypted files if we have the encryption key + * and if it is PCC-RO. + */ + if (IS_ENCRYPTED(inode) && + (!llcrypt_has_encryption_key(inode) || + (pcci && pcci->pcci_type != LU_PCC_READONLY))) + GOTO(out_unlock, rc = 0); + if (lli->lli_pcc_state & PCC_STATE_FL_ATTACHING) { pcc_file_fallback_set(lli, pccf); GOTO(out_unlock, rc = 0); @@ -2371,7 +2424,7 @@ int pcc_file_open(struct inode *inode, struct file *file) CDEBUG(D_CACHE, "opening pcc file '%pd' - %pd\n", path->dentry, file->f_path.dentry); - pcc_file = dentry_open(path, file->f_flags, + pcc_file = dentry_open(path, file->f_flags & ~O_DIRECT, pcc_super_cred(inode->i_sb)); if (IS_ERR_OR_NULL(pcc_file)) { rc = pcc_file == NULL ? -EINVAL : PTR_ERR(pcc_file); @@ -2467,7 +2520,10 @@ ssize_t pcc_file_read_iter(struct kiocb *iocb, struct file *file = iocb->ki_filp; struct inode *inode = file_inode(file); struct pcc_file *pccf = ll_file2pccf(file); - ssize_t result; + unsigned int blockbits = 0, blocksize = 0; + pgoff_t start_index, end_index, index; + ssize_t result = 0; + int rc = 0; ENTRY; file->f_ra.ra_pages = 0; @@ -2480,18 +2536,109 @@ ssize_t pcc_file_read_iter(struct kiocb *iocb, if (!*cached) RETURN(0); - /* Fake I/O error on RO-PCC */ + /* Fake I/O error on PCC-RO */ if (CFS_FAIL_CHECK(OBD_FAIL_LLITE_PCC_FAKE_ERROR)) GOTO(out, result = -EIO); iocb->ki_filp = pccf->pccf_file; - /* generic_file_aio_read does not support ext4-dax, - * __pcc_file_read_iter uses ->aio_read hook directly - * to add support for ext4-dax. - */ + if (!IS_ENCRYPTED(inode)) { + /* generic_file_aio_read does not support ext4-dax, + * __pcc_file_read_iter uses ->aio_read hook directly + * to add support for ext4-dax. + */ + result = __pcc_file_read_iter(iocb, iter); + GOTO(out, result); + } + + /* from this point, we are dealing with an encrypted inode */ + blockbits = inode->i_blkbits; + blocksize = 1 << blockbits; + start_index = iocb->ki_pos >> PAGE_SHIFT; + if (i_size_read(inode) == 0) + end_index = (iocb->ki_pos + (loff_t)iov_iter_count(iter) - 1) + >> PAGE_SHIFT; + else + end_index = (min(iocb->ki_pos + (loff_t)iov_iter_count(iter), + i_size_read(inode)) - 1) >> PAGE_SHIFT; + + /* Proceed to decryption of PCC-RO page cache pages */ + for (index = start_index; index <= end_index; index++) { + struct address_space *mapping; + struct page *vmpage = NULL; + unsigned int offs = 0; + + mapping = file_inode(pccf->pccf_file)->i_mapping; + vmpage = grab_cache_page(mapping, index); + if (vmpage == NULL) + continue; + + /* vmpage has already been decrypted */ + if (PagePrivate2(vmpage)) + goto out_pageprivate2; + + if (PageDirty(vmpage)) + /* this should not happen with PCC-RO */ + GOTO(out_pageprivate2, rc = -EIO); + if (!PageUptodate(vmpage)) { +#ifdef HAVE_AOPS_READ_FOLIO + rc = mapping->a_ops->read_folio(pccf->pccf_file, + page_folio(vmpage)); +#else + rc = mapping->a_ops->readpage(pccf->pccf_file, vmpage); +#endif + if (rc) { + put_page(vmpage); + continue; + } + lock_page(vmpage); + if (!PageUptodate(vmpage)) + GOTO(out_pageprivate2, rc = -EIO); + } + + while (offs < PAGE_SIZE) { + u64 lblk_num = ((u64)vmpage->index << + (PAGE_SHIFT - blockbits)) + + (offs >> blockbits); + unsigned int i; + + /* do not decrypt if page is all 0s */ + if (memchr_inv(page_address(vmpage) + offs, 0, + LUSTRE_ENCRYPTION_UNIT_SIZE) == + NULL) + break; + + for (i = offs; + i < offs + LUSTRE_ENCRYPTION_UNIT_SIZE; + i += blocksize, lblk_num++) { + rc = llcrypt_decrypt_block_inplace(inode, + vmpage, + blocksize, i, + lblk_num); + if (rc) + break; + } + if (rc) + GOTO(out_pageprivate2, rc); + + offs += LUSTRE_ENCRYPTION_UNIT_SIZE; + } + /* set PagePrivate2 flag so that we know + * this page is now decrypted + */ + SetPagePrivate2(vmpage); + +out_pageprivate2: + unlock_page(vmpage); + put_page(vmpage); + + } + result = __pcc_file_read_iter(iocb, iter); - iocb->ki_filp = file; + if (iocb->ki_pos > i_size_read(inode) && result > 0) + result -= iocb->ki_pos - i_size_read(inode); + out: + iocb->ki_filp = file; pcc_io_fini(inode, PIT_READ, result, cached); RETURN(result); } @@ -2610,7 +2757,9 @@ int pcc_inode_getattr(struct inode *inode, u32 request_mask, { struct ll_inode_info *lli = ll_i2info(inode); const struct cred *old_cred; + struct pcc_inode *pcci; struct kstat stat; + loff_t size; s64 atime; s64 mtime; s64 ctime; @@ -2628,7 +2777,8 @@ int pcc_inode_getattr(struct inode *inode, u32 request_mask, RETURN(0); old_cred = override_creds(pcc_super_cred(inode->i_sb)); - rc = ll_vfs_getattr(&ll_i2pcci(inode)->pcci_path, &stat, request_mask, + pcci = ll_i2pcci(inode); + rc = ll_vfs_getattr(&pcci->pcci_path, &stat, request_mask, flags); revert_creds(old_cred); if (rc) @@ -2655,7 +2805,19 @@ int pcc_inode_getattr(struct inode *inode, u32 request_mask, if (mtime < stat.mtime.tv_sec) mtime = stat.mtime.tv_sec; - i_size_write(inode, stat.size); + size = stat.size; + /* The pcc_xattr_encsize xattr is only valid for PCC-RO. */ + if (IS_ENCRYPTED(inode) && pcci->pcci_type == LU_PCC_READONLY) { + loff_t encsize; + + rc = ll_vfs_getxattr(pcci->pcci_path.dentry, + pcci->pcci_path.dentry->d_inode, + pcc_xattr_encsize, + &encsize, sizeof(encsize)); + if (rc > 0) + size = encsize; + } + i_size_write(inode, size); inode->i_blocks = stat.blocks; inode_set_atime(inode, atime, 0); @@ -2722,9 +2884,9 @@ int pcc_fsync(struct file *file, loff_t start, loff_t end, } /* - * After the file is attached into RO-PCC, its dirty pages on this + * After the file is attached into PCC-RO, its dirty pages on this * client may not be flushed. So fsync() should fall back to normal - * Lustre I/O path flushing dirty data to OSTs. And flush on RO-PCC + * Lustre I/O path flushing dirty data to OSTs. And flush on PCC-RO * copy is meaningless. */ if (pccf->pccf_type == LU_PCC_READONLY) { @@ -3215,7 +3377,7 @@ int pcc_fault(struct vm_area_struct *vma, struct vm_fault *vmf, if (!*cached) RETURN(0); - /* Tolerate the mmap read failure for RO-PCC */ + /* Tolerate the mmap read failure for PCC-RO */ if (CFS_FAIL_CHECK(OBD_FAIL_LLITE_PCC_FAKE_ERROR)) GOTO(out, rc = VM_FAULT_SIGBUS); @@ -3540,6 +3702,8 @@ int pcc_inode_create_fini(struct inode *inode, struct pcc_create_attach *pca) /* Set the layout generation of newly created file with 0 */ pcc_layout_gen_set(pcci, 0); + rc = pcc_encsize_xattr_set(pcci); + out_put: if (rc) { (void) pcc_inode_remove(inode, pcc_dentry); @@ -3605,6 +3769,7 @@ static ssize_t pcc_copy_data(struct file *src, struct file *dst) ssize_t rc2; loff_t pos, offset = 0; size_t buf_len = 1048576; + struct inode *inode = file_inode(src); void *buf; ENTRY; @@ -3627,6 +3792,13 @@ static ssize_t pcc_copy_data(struct file *src, struct file *dst) GOTO(out_free, rc = -EINTR); pos = offset; + if (inode && IS_ENCRYPTED(inode)) + /* Setting the S_PCCCOPY flag prevents the Lustre file + * from being decrypted in the OSC layer, so that the + * PCC file contains ciphertext data. + * S_PCCCOPY flag is removed in ll_prepare_close(). + */ + inode->i_flags |= S_PCCCOPY; rc2 = cfs_kernel_read(src, buf, buf_len, &pos); if (rc2 < 0) GOTO(out_free, rc = rc2); @@ -3655,6 +3827,7 @@ static int pcc_attach_data_archive(struct file *file, struct inode *inode, bool direct = false; struct path path; ssize_t ret; + int flags = O_WRONLY | O_LARGEFILE; int rc; ENTRY; @@ -3666,7 +3839,14 @@ static int pcc_attach_data_archive(struct file *file, struct inode *inode, path.mnt = dataset->pccd_path.mnt; path.dentry = *dentry; - pcc_filp = dentry_open(&path, O_WRONLY | O_LARGEFILE, current_cred()); + /* If the inode is encrypted, we want the PCC file to be synced to the + * storage. This is necessary as we are going to decrypt the page cache + * pages of the PCC inode later in pcc_file_read_iter(), but still we + * need to keep the ciphertext version on disk. + */ + if (IS_ENCRYPTED(inode)) + flags |= O_SYNC; + pcc_filp = dentry_open(&path, flags, current_cred()); if (IS_ERR_OR_NULL(pcc_filp)) { rc = pcc_filp == NULL ? -EINVAL : PTR_ERR(pcc_filp); GOTO(out_dentry, rc); @@ -3990,6 +4170,8 @@ static int pcc_readonly_attach(struct file *file, } pcc_layout_gen_set(pcci, gen); + + rc = pcc_encsize_xattr_set(pcci); out_put_unlock: if (rc) { if (!unlinked) diff --git a/lustre/llite/symlink.c b/lustre/llite/symlink.c index 19a7402..7d43f71 100644 --- a/lustre/llite/symlink.c +++ b/lustre/llite/symlink.c @@ -130,7 +130,7 @@ static int ll_readlink_internal(struct inode *inode, /* Do not cache symlink targets encoded without the key, * since those become outdated once the key is added. */ - if (!llcrypt_has_encryption_key(inode)) + if (!ll_has_encryption_key(inode)) RETURN(0); } #endif diff --git a/lustre/osc/osc_request.c b/lustre/osc/osc_request.c index 88e193a..6e92742 100644 --- a/lustre/osc/osc_request.c +++ b/lustre/osc/osc_request.c @@ -1703,7 +1703,7 @@ retry_encrypt: if (attr.cat_size) oa->o_size = attr.cat_size; } else if (opc == OST_READ && inode && IS_ENCRYPTED(inode) && - llcrypt_has_encryption_key(inode)) { + ll_has_encryption_key(inode)) { for (i = 0; i < page_count; i++) { struct brw_page *pg = pga[i]; u32 nunits = (pg->bp_off & ~PAGE_MASK) + pg->bp_count; @@ -1734,7 +1734,7 @@ retry_encrypt: for (i = 0; i < page_count; i++) { short_io_size += pga[i]->bp_count; if (!inode || !IS_ENCRYPTED(inode) || - !llcrypt_has_encryption_key(inode)) { + !ll_has_encryption_key(inode)) { pga[i]->bp_count_diff = 0; pga[i]->bp_off_diff = 0; } @@ -2366,7 +2366,7 @@ static int osc_brw_fini_request(struct ptlrpc_request *req, int rc) if (inode && IS_ENCRYPTED(inode)) { int idx; - if (!llcrypt_has_encryption_key(inode)) { + if (!ll_has_encryption_key(inode)) { CDEBUG(D_SEC, "no enc key for ino %lu\n", inode->i_ino); GOTO(out, rc); } diff --git a/lustre/tests/sanity-pcc.sh b/lustre/tests/sanity-pcc.sh index 17752be..05f62cf 100755 --- a/lustre/tests/sanity-pcc.sh +++ b/lustre/tests/sanity-pcc.sh @@ -131,7 +131,7 @@ lpcc_fid2path() { local hsm_root="$1" local lustre_path="$2" - local fid=$(path2fid $lustre_path) + local fid=${3:-$(path2fid $lustre_path)} local seq=$(echo $fid | awk -F ':' '{print $1}') local oid=$(echo $fid | awk -F ':' '{print $2}') @@ -275,6 +275,45 @@ setup_loopdev_project() { do_facet $facet mount | grep $mntpt } +setup_dummy_key() { + local mode='\x00\x00\x00\x00' + local raw="$(printf ""\\\\x%02x"" {0..63})" + local size + local key + + [[ $(lscpu) =~ Byte\ Order.*Little ]] && size='\x40\x00\x00\x00' || + size='\x00\x00\x00\x40' + key="${mode}${raw}${size}" + do_facet $SINGLEAGT "echo -en '${key}' | + keyctl padd logon fscrypt:4242424242424242 @u" +} + +setup_for_enc_tests() { + local agthost=${SINGLEAGT}_HOST + + # remount client with test_dummy_encryption option + zconf_umount ${!agthost} $MOUNT || + error "umount $SINGLEAGT $MOUNT failed" + zconf_mount ${!agthost} $MOUNT ${MOUNT_OPTS},test_dummy_encryption || + error "mount $SINGLEAGT with '-o test_dummy_encryption' failed" + + setup_dummy_key + + # this directory will be encrypted, because of dummy mode + do_facet $SINGLEAGT mkdir $DIR/$tdir || error "mkdir $DIR/$tdir failed" +} + +cleanup_for_enc_tests() { + local agthost=${SINGLEAGT}_HOST + + do_facet $SINGLEAGT rm -rf $DIR/$tdir + # remount client normally + zconf_umount ${!agthost} $MOUNT || + error "umount $SINGLEAGT $MOUNT failed" + zconf_mount ${!agthost} $MOUNT || + error "remount $SINGLEAGT failed" +} + lpcc_rw_test() { local restore="$1" local project="$2" @@ -2110,6 +2149,93 @@ test_21i() { } run_test 21i "HSM release increase layout gen, should invalidate PCC-RO cache" +test_21j() { + local loopfile="$TMP/$tfile" + local mntpt="/mnt/pcc.$tdir" + local hsm_root="$mntpt/$tdir" + local tmpfile=$TMP/abc + local file=$DIR/$tdir/$tfile + local scrambledfile + local lpcc_path + local size + local fid + local key + + $LCTL get_param -n mdc.*.connect_flags | grep -q pcc_ro || + skip "Server does not support PCC-RO" + + $LCTL get_param mdc.*.import | grep -q client_encryption || + skip "client encryption not supported" + + mount.lustre --help |& grep -q "test_dummy_encryption:" || + skip "need dummy encryption support" + + setup_loopdev $SINGLEAGT $loopfile $mntpt 50 + stack_trap cleanup_for_enc_tests EXIT + setup_for_enc_tests + copytool setup -m "$MOUNT" -a "$HSM_ARCHIVE_NUMBER" + setup_pcc_mapping + + do_facet $SINGLEAGT "yes 1 | dd of=$tmpfile bs=1 count=5000 conv=fsync" + do_facet $SINGLEAGT cp $tmpfile $file + do_facet $SINGLEAGT $LFS getstripe $file + do_facet $SINGLEAGT $LFS pcc attach -r -i $HSM_ARCHIVE_NUMBER $file || + error "failed to PCC-RO attach file $file" + check_lpcc_state $file "readonly" + echo "PCC-RO attach '$file':" + do_facet $SINGLEAGT $LFS getstripe -v $file + + do_facet $SINGLEAGT cmp -bl $tmpfile $file || + error "file $file is corrupted (1)" + fid=$(do_facet $SINGLEAGT lfs path2fid $file | tr -d '[]') + lpcc_path=$(lpcc_fid2path $hsm_root $file $fid) + do_facet $SINGLEAGT cmp -bl -n 4096 $tmpfile $lpcc_path || + error "file $lpcc_path is corrupted (2)" + + do_facet $SINGLEAGT $LFS pcc detach -k $file || + error "failed to PCC-RO detach file $file" + do_facet $SINGLEAGT cmp -s -n 4096 $tmpfile $lpcc_path && + error "file $lpcc_path is corrupted (3)" + + size=$(do_facet $SINGLEAGT stat "--printf=%s" $lpcc_path) + [ $size == 8192 ] || error "PCC file $lpcc_path incorrect size $size" + + do_facet $SINGLEAGT cmp -bl $tmpfile $file || + error "file $file is corrupted (4)" + do_facet $SINGLEAGT cmp -bl -n 4096 $tmpfile $lpcc_path || + error "file $lpcc_path is corrupted (5)" + + do_facet $SINGLEAGT cp $tmpfile ${file}_2 + do_facet $SINGLEAGT $LFS getstripe ${file}_2 + do_facet $SINGLEAGT $LFS pcc attach -r -i $HSM_ARCHIVE_NUMBER ${file}_2 || + error "failed to PCC-RO attach file ${file}_2" + check_lpcc_state ${file}_2 "readonly" + echo "PCC-RO attach '${file}_2':" + do_facet $SINGLEAGT $LFS getstripe -v ${file}_2 + + do_facet $SINGLEAGT $LFS pcc detach ${file}_2 || + error "failed to PCC-RO detach file ${file}_2" + do_facet $SINGLEAGT cmp -bl $tmpfile ${file}_2 || + error "file ${file}_2 is corrupted (6)" + rm -f ${file}_2 + + # remove fscrypt key from keyring + key=$(do_facet $SINGLEAGT keyctl show | + awk '$7 ~ "^fscrypt:" {print $1}') + [ -n "$key" ] || error "fscrypt key empty on $SINGLEAGT" + do_facet $SINGLEAGT keyctl revoke $key + do_facet $SINGLEAGT keyctl reap + do_facet $SINGLEAGT $LCTL set_param -n ldlm.namespaces.*.lru_size=clear + + scrambledfile=$(do_facet $SINGLEAGT find $DIR/$tdir/ \ + -maxdepth 1 -mindepth 1 -type f) + do_facet $SINGLEAGT $LFS pcc detach -k $scrambledfile || + error "failed to PCC-RO detach file $scrambledfile (2)" + + do_facet $SINGLEAGT rm -f $tmpfile +} +run_test 21j "PCC-RO for encrypted file" + test_22() { local loopfile="$TMP/$tfile" local mntpt="/mnt/pcc.$tdir" @@ -2779,7 +2905,7 @@ test_31() { file=$DIR/$tdir/roattach echo -n backend_del_roattach_rm > $file - lpcc_path3=$(lpcc_fid2path $hsm_root $file "readonly") + lpcc_path3=$(lpcc_fid2path $hsm_root $file) do_facet $SINGLEAGT $LFS pcc attach -r -i $HSM_ARCHIVE_NUMBER $file || error "RO-PCC attach $file failed" check_lpcc_state $file "readonly" @@ -2817,7 +2943,7 @@ test_32() { "projid={100}\ rwid=$HSM_ARCHIVE_NUMBER\ auto_attach=0" do_facet $SINGLEAGT echo -n roattach_removed > $file - lpcc_path=$(lpcc_fid2path $hsm_root $file "readonly") + lpcc_path=$(lpcc_fid2path $hsm_root $file) do_facet $SINGLEAGT $LFS pcc attach -r -i $HSM_ARCHIVE_NUMBER $file || error "RO-PCC attach $file failed" rmultiop_start $agt_host $file o_rc || error "multiop $file failed" diff --git a/lustre/utils/liblustreapi_pcc.c b/lustre/utils/liblustreapi_pcc.c index f1ddba5..6636d90 100644 --- a/lustre/utils/liblustreapi_pcc.c +++ b/lustre/utils/liblustreapi_pcc.c @@ -369,7 +369,10 @@ int llapi_pcc_detach_file(const char *path, __u32 flags) int rc; int fd; - fd = open(path, O_RDWR | O_NONBLOCK); + /* Specify O_CIPHERTEXT | O_DIRECT flags to allow pcc detach + * on encrypted file without the key. + */ + fd = open(path, O_RDWR | O_NONBLOCK | O_CIPHERTEXT | O_DIRECT); if (fd < 0) { rc = -errno; llapi_error(LLAPI_MSG_ERROR, rc, "cannot open '%s'", -- 1.8.3.1