From: Sebastien Buisson Date: Fri, 22 May 2020 07:27:48 +0000 (+0000) Subject: LU-12275 sec: encryption support for DoM files X-Git-Tag: 2.13.55~5 X-Git-Url: https://git.whamcloud.com/?p=fs%2Flustre-release.git;a=commitdiff_plain;h=a71586d4ee8d6f039a413e2a0fd791db847a3c19 LU-12275 sec: encryption support for DoM files On client side, data read from DoM files do not go through the OSC layer. So implement file decryption in ll_dom_finish_open() right after file data has been put in cache pages. On server side, DoM file size needs to be properly set on MDT when content is encrypted. Pages are full of encrypted data, but inode size must be apparent, clear text object size. For reads of DoM encrypted files to work proprely, we also need to make sure we send whole encryption units to client side. Also add sanity-sec test_50 to exercise encryption of DoM files. Test-Parameters: testlist=sanity-sec envdefinitions=ONLY="36 37 38 39 40 41 42 43 44 45 46 47 48 49 50" clientdistro=el8.1 fstype=ldiskfs mdscount=2 mdtcount=4 Test-Parameters: testlist=sanity-sec envdefinitions=ONLY="36 37 38 39 40 41 42 43 44 45 46 47 48 49 50" clientdistro=el8.1 fstype=zfs mdscount=2 mdtcount=4 Signed-off-by: Sebastien Buisson Change-Id: I7721ca4085373a7a01b2062c37458a7136e646e0 Reviewed-on: https://review.whamcloud.com/38702 Reviewed-by: Andreas Dilger Reviewed-by: Mike Pershin Tested-by: jenkins Tested-by: Maloo Reviewed-by: Oleg Drokin --- diff --git a/lustre/llite/crypto.c b/lustre/llite/crypto.c index 110928a..7e32fa4 100644 --- a/lustre/llite/crypto.c +++ b/lustre/llite/crypto.c @@ -32,18 +32,13 @@ static int ll_get_context(struct inode *inode, void *ctx, size_t len) { - struct dentry *dentry; + struct dentry *dentry = d_find_any_alias(inode); int rc; - if (hlist_empty(&inode->i_dentry)) - return -ENODATA; - - hlist_for_each_entry(dentry, &inode->i_dentry, d_alias) { - break; - } - rc = ll_vfs_getxattr(dentry, inode, LL_XATTR_NAME_ENCRYPTION_CONTEXT, ctx, len); + if (dentry) + dput(dentry); /* used as encryption unit size */ if (S_ISREG(inode->i_mode)) diff --git a/lustre/llite/file.c b/lustre/llite/file.c index 92bef32..f411a71 100644 --- a/lustre/llite/file.c +++ b/lustre/llite/file.c @@ -423,6 +423,9 @@ static inline int ll_dom_readpage(void *data, struct page *page) { struct niobuf_local *lnb = data; void *kaddr; + int rc = 0; + + struct inode *inode = page2inode(page); kaddr = kmap_atomic(page); memcpy(kaddr, lnb->lnb_data, lnb->lnb_len); @@ -432,9 +435,22 @@ static inline int ll_dom_readpage(void *data, struct page *page) flush_dcache_page(page); SetPageUptodate(page); kunmap_atomic(kaddr); + + if (inode && IS_ENCRYPTED(inode) && S_ISREG(inode->i_mode)) { + if (!llcrypt_has_encryption_key(inode)) + CDEBUG(D_SEC, "no enc key for "DFID"\n", + PFID(ll_inode2fid(inode))); + /* decrypt only if page is not empty */ + else if (memcmp(page_address(page), + page_address(ZERO_PAGE(0)), + PAGE_SIZE) != 0) + rc = llcrypt_decrypt_pagecache_blocks(page, + PAGE_SIZE, + 0); + } unlock_page(page); - return 0; + return rc; } void ll_dom_finish_open(struct inode *inode, struct ptlrpc_request *req, @@ -475,7 +491,8 @@ void ll_dom_finish_open(struct inode *inode, struct ptlrpc_request *req, * buffer, in both cases total size should be equal to the file size. */ body = req_capsule_server_get(&req->rq_pill, &RMF_MDT_BODY); - if (rnb->rnb_offset + rnb->rnb_len != body->mbo_dom_size) { + if (rnb->rnb_offset + rnb->rnb_len != body->mbo_dom_size && + !(inode && IS_ENCRYPTED(inode))) { CERROR("%s: server returns off/len %llu/%u but size %llu\n", ll_i2sbi(inode)->ll_fsname, rnb->rnb_offset, rnb->rnb_len, body->mbo_dom_size); diff --git a/lustre/llite/namei.c b/lustre/llite/namei.c index ed6eb70..9b7241a 100644 --- a/lustre/llite/namei.c +++ b/lustre/llite/namei.c @@ -634,6 +634,36 @@ static int ll_lookup_it_finish(struct ptlrpc_request *request, if (rc) RETURN(rc); + /* If encryption context was returned by MDT, put it in + * inode now to save an extra getxattr and avoid deadlock. + */ + if (body->mbo_valid & OBD_MD_ENCCTX) { + encctx = req_capsule_server_get(pill, &RMF_FILE_ENCCTX); + encctxlen = req_capsule_get_size(pill, + &RMF_FILE_ENCCTX, + RCL_SERVER); + + if (encctxlen) { + CDEBUG(D_SEC, + "server returned encryption ctx for "DFID"\n", + PFID(ll_inode2fid(inode))); + rc = ll_xattr_cache_insert(inode, + LL_XATTR_NAME_ENCRYPTION_CONTEXT, + encctx, encctxlen); + if (rc) + CWARN("%s: cannot set enc ctx for "DFID": rc = %d\n", + ll_i2sbi(inode)->ll_fsname, + PFID(ll_inode2fid(inode)), rc); + else if (encrypt) { + rc = llcrypt_get_encryption_info(inode); + if (rc) + CDEBUG(D_SEC, + "cannot get enc info for "DFID": rc = %d\n", + PFID(ll_inode2fid(inode)), rc); + } + } + } + if (it->it_op & IT_OPEN) ll_dom_finish_open(inode, request, it); @@ -678,29 +708,6 @@ static int ll_lookup_it_finish(struct ptlrpc_request *request, PFID(ll_inode2fid(inode)), rc); } - - /* If encryption context was returned by MDT, put it in - * inode now to save an extra getxattr and avoid deadlock. - */ - if (body->mbo_valid & OBD_MD_ENCCTX) { - encctx = req_capsule_server_get(pill, &RMF_FILE_ENCCTX); - encctxlen = req_capsule_get_size(pill, - &RMF_FILE_ENCCTX, - RCL_SERVER); - - if (encctxlen) { - CDEBUG(D_SEC, - "server returned encryption ctx for "DFID"\n", - PFID(ll_inode2fid(inode))); - rc = ll_xattr_cache_insert(inode, - LL_XATTR_NAME_ENCRYPTION_CONTEXT, - encctx, encctxlen); - if (rc) - CWARN("%s: cannot set enc ctx for "DFID": rc = %d\n", - ll_i2sbi(inode)->ll_fsname, - PFID(ll_inode2fid(inode)), rc); - } - } } /* Only hash *de if it is unhashed (new dentry). diff --git a/lustre/mdt/mdt_io.c b/lustre/mdt/mdt_io.c index 04cfe5d..b89d8fb 100644 --- a/lustre/mdt/mdt_io.c +++ b/lustre/mdt/mdt_io.c @@ -586,9 +586,9 @@ static int mdt_commitrw_read(const struct lu_env *env, struct mdt_device *mdt, static int mdt_commitrw_write(const struct lu_env *env, struct obd_export *exp, struct mdt_device *mdt, struct mdt_object *mo, - struct lu_attr *la, int objcount, int niocount, - struct niobuf_local *lnb, unsigned long granted, - int old_rc) + struct lu_attr *la, struct obdo *oa, int objcount, + int niocount, struct niobuf_local *lnb, + unsigned long granted, int old_rc) { struct dt_device *dt = mdt->mdt_bottom; struct dt_object *dob; @@ -654,7 +654,7 @@ retry: GOTO(out_stop, rc); dt_write_lock(env, dob, 0); - rc = dt_write_commit(env, dob, lnb, niocount, th, 0); + rc = dt_write_commit(env, dob, lnb, niocount, th, oa->o_size); if (rc) GOTO(unlock, rc); @@ -745,7 +745,7 @@ int mdt_obd_commitrw(const struct lu_env *env, int cmd, struct obd_export *exp, la_from_obdo(la, oa, valid); - rc = mdt_commitrw_write(env, exp, mdt, mo, la, objcount, + rc = mdt_commitrw_write(env, exp, mdt, mo, la, oa, objcount, npages, lnb, oa->o_grant_used, old_rc); if (rc == 0) obdo_from_la(oa, la, VALID_FLAGS | LA_GID | LA_UID); @@ -1408,6 +1408,7 @@ int mdt_dom_read_on_open(struct mdt_thread_info *mti, struct mdt_device *mdt, int rc; loff_t offset; unsigned int len, copied = 0; + __u64 real_dom_size; int lnbs, nr_local, i; bool dom_lock = false; @@ -1441,8 +1442,18 @@ int mdt_dom_read_on_open(struct mdt_thread_info *mti, struct mdt_device *mdt, if (!dom_lock || !mdt->mdt_opts.mo_dom_read_open) RETURN(0); + /* if DoM object holds encrypted content, we need to make sure we + * send whole encryption units, or client will read corrupted content + */ + if (mbo->mbo_valid & LA_FLAGS && mbo->mbo_flags & LUSTRE_ENCRYPT_FL && + mbo->mbo_dom_size & ~LUSTRE_ENCRYPTION_MASK) + real_dom_size = (mbo->mbo_dom_size & LUSTRE_ENCRYPTION_MASK) + + LUSTRE_ENCRYPTION_UNIT_SIZE; + else + real_dom_size = mbo->mbo_dom_size; + CDEBUG(D_INFO, "File size %llu, reply sizes %d/%d\n", - mbo->mbo_dom_size, req->rq_reqmsg->lm_repsize, req->rq_replen); + real_dom_size, req->rq_reqmsg->lm_repsize, req->rq_replen); len = req->rq_reqmsg->lm_repsize - req->rq_replen; /* NB: at this moment we have the following sizes: @@ -1461,11 +1472,11 @@ int mdt_dom_read_on_open(struct mdt_thread_info *mti, struct mdt_device *mdt, * 1) try to fit into the buffer we have * 2) return just file tail otherwise. */ - if (mbo->mbo_dom_size <= len) { + if (real_dom_size <= len) { /* can fit whole data */ - len = mbo->mbo_dom_size; + len = real_dom_size; offset = 0; - } else if (mbo->mbo_dom_size < + } else if (real_dom_size < mdt_lmm_dom_stripesize(mti->mti_attr.ma_lmm)) { int tail, pgbits; @@ -1484,14 +1495,14 @@ int mdt_dom_read_on_open(struct mdt_thread_info *mti, struct mdt_device *mdt, } pgbits = max_t(int, PAGE_SHIFT, req->rq_export->exp_target_data.ted_pagebits); - tail = mbo->mbo_dom_size % (1 << pgbits); + tail = real_dom_size % (1 << pgbits); /* no partial tail or tail can't fit in reply */ if (tail == 0 || len < tail) RETURN(0); len = tail; - offset = mbo->mbo_dom_size - len; + offset = real_dom_size - len; } else { /* DOM stripe is fully written, so don't expect its tail * will be used by append. diff --git a/lustre/tests/sanity-sec.sh b/lustre/tests/sanity-sec.sh index 5f6ea06..6344ebe 100755 --- a/lustre/tests/sanity-sec.sh +++ b/lustre/tests/sanity-sec.sh @@ -3537,7 +3537,7 @@ trace_cmd() { local cmd="$@" local xattr_name="security.c" - sync ; sync ; echo 3 > /proc/sys/vm/drop_caches + cancel_lru_locks osc ; cancel_lru_locks mdc $LCTL set_param debug=+info $LCTL clear @@ -3588,6 +3588,143 @@ test_49() { } run_test 49 "Avoid getxattr for encryption context" +test_50() { + local testfile=$DIR/$tdir/$tfile + local tmpfile=$TMP/abc + local pagesz=$(getconf PAGESIZE) + local sz + + $LCTL get_param mdc.*.import | grep -q client_encryption || + skip "client encryption not supported" + + mount.lustre --help |& grep -q "test_dummy_encryption:" || + skip "need dummy encryption support" + + stack_trap cleanup_for_enc_tests EXIT + setup_for_enc_tests + + # write small file, data on MDT only + tr '\0' '1' < /dev/zero | + dd of=$tmpfile bs=1 count=5000 conv=fsync + $LFS setstripe -E 1M -L mdt -E EOF $testfile + cp $tmpfile $testfile + + # check that in-memory representation of file is correct + cmp -bl $tmpfile $testfile || + error "file $testfile is corrupted in memory" + + cancel_lru_locks osc ; cancel_lru_locks mdc + + # check that file read from server is correct + cmp -bl $tmpfile $testfile || + error "file $testfile is corrupted on server" + + # decrease size: truncate to PAGE_SIZE + $TRUNCATE $tmpfile $pagesz + $TRUNCATE $testfile $pagesz + cancel_lru_locks osc ; cancel_lru_locks mdc + cmp -bl $tmpfile $testfile || + error "file $testfile is corrupted (1)" + + # increase size: truncate to 2 x PAGE_SIZE + sz=$((pagesz*2)) + $TRUNCATE $tmpfile $sz + $TRUNCATE $testfile $sz + cancel_lru_locks osc ; cancel_lru_locks mdc + cmp -bl $tmpfile $testfile || + error "file $testfile is corrupted (2)" + + # truncate to PAGE_SIZE / 2 + sz=$((pagesz/2)) + $TRUNCATE $tmpfile $sz + $TRUNCATE $testfile $sz + cancel_lru_locks osc ; cancel_lru_locks mdc + cmp -bl $tmpfile $testfile || + error "file $testfile is corrupted (3)" + + # truncate to a smaller, non-multiple of PAGE_SIZE, non-multiple of 16 + sz=$((sz-7)) + $TRUNCATE $tmpfile $sz + $TRUNCATE $testfile $sz + cancel_lru_locks osc ; cancel_lru_locks mdc + cmp -bl $tmpfile $testfile || + error "file $testfile is corrupted (4)" + + # truncate to a larger, non-multiple of PAGE_SIZE, non-multiple of 16 + sz=$((sz+18)) + $TRUNCATE $tmpfile $sz + $TRUNCATE $testfile $sz + cancel_lru_locks osc ; cancel_lru_locks mdc + cmp -bl $tmpfile $testfile || + error "file $testfile is corrupted (5)" + + # truncate to a larger, non-multiple of PAGE_SIZE, in a different page + sz=$((sz+pagesz+30)) + $TRUNCATE $tmpfile $sz + $TRUNCATE $testfile $sz + cancel_lru_locks osc ; cancel_lru_locks mdc + cmp -bl $tmpfile $testfile || + error "file $testfile is corrupted (6)" + + rm -f $testfile + cancel_lru_locks osc ; cancel_lru_locks mdc + + # write hole in file, data spread on MDT and OST + tr '\0' '2' < /dev/zero | + dd of=$tmpfile bs=1 count=1539 seek=1539074 conv=fsync,notrunc + $LFS setstripe -E 1M -L mdt -E EOF $testfile + cp --sparse=always $tmpfile $testfile + + # check that in-memory representation of file is correct + cmp -bl $tmpfile $testfile || + error "file $testfile is corrupted in memory" + + cancel_lru_locks osc ; cancel_lru_locks mdc + + # check that file read from server is correct + cmp -bl $tmpfile $testfile || + error "file $testfile is corrupted on server" + + # truncate to a smaller, non-multiple of PAGE_SIZE, non-multiple of 16, + # inside OST part of data + sz=$((1024*1024+13)) + $TRUNCATE $tmpfile $sz + $TRUNCATE $testfile $sz + cancel_lru_locks osc ; cancel_lru_locks mdc + cmp -bl $tmpfile $testfile || + error "file $testfile is corrupted (7)" + + # truncate to a smaller, non-multiple of PAGE_SIZE, non-multiple of 16, + # inside MDT part of data + sz=7 + $TRUNCATE $tmpfile $sz + $TRUNCATE $testfile $sz + cancel_lru_locks osc ; cancel_lru_locks mdc + cmp -bl $tmpfile $testfile || + error "file $testfile is corrupted (8)" + + # truncate to a larger, non-multiple of PAGE_SIZE, non-multiple of 16, + # inside MDT part of data + sz=$((1024*1024-13)) + $TRUNCATE $tmpfile $sz + $TRUNCATE $testfile $sz + cancel_lru_locks osc ; cancel_lru_locks mdc + cmp -bl $tmpfile $testfile || + error "file $testfile is corrupted (9)" + + # truncate to a larger, non-multiple of PAGE_SIZE, non-multiple of 16, + # inside OST part of data + sz=$((1024*1024+7)) + $TRUNCATE $tmpfile $sz + $TRUNCATE $testfile $sz + cancel_lru_locks osc ; cancel_lru_locks mdc + cmp -bl $tmpfile $testfile || + error "file $testfile is corrupted (10)" + + rm -f $tmpfile +} +run_test 50 "DoM encrypted file" + log "cleanup: ======================================================" sec_unsetup() {