From ac5fcdce025b4825500c0308d89dfdab1faece51 Mon Sep 17 00:00:00 2001 From: Sebastien Buisson Date: Wed, 8 Jul 2020 16:19:08 +0000 Subject: [PATCH] LU-12275 sec: encryption with different client PAGE_SIZE In order to properly handle encryption/decryption on clients that have a PAGE_SIZE != LUSTRE_ENCRYPTION_UNIT_SIZE (typically aarch64/ppc64), a few adjustements are necessary: - when encrypting, do not proceed with PAGE_SIZE as encryption length. Instead, round up to a multiple of LUSTRE_ENCRYPTION_UNIT_SIZE. On aarch64/ppc64, it avoids encrypting way beyond LUSTRE_ENCRYPTION_UNIT_SIZE when the page is not full. - when decrypting, do not proceed with PAGE_SIZE as decryption length. Instead, do LUSTRE_ENCRYPTION_UNIT_SIZE length at a time. It enables proper detection of 'all 0s' sent by servers for content beyond file size. Regarding tests, add sanity-sec test_53 to exercise encryption from clients with different PAGE_SIZE. The trick to achieve this with AT is to expect the client to have 64KB PAGE_SIZE, and the servers to have 4KB PAGE_SIZE, and then mount a client from the MDS node. This also means code running on server side needs to have client encryption support enabled, so CentOS/RHEL 8 at least. Test-Parameters: trivial Test-Parameters: clientarch=aarch64 clientcount=1 clientdistro=el8.1 serverdistro=el8.1 testlist=sanity-sec env=ONLY="36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 52 53" fstype=ldiskfs mdscount=2 mdtcount=4 Test-Parameters: clientarch=aarch64 clientcount=1 clientdistro=el8.1 serverdistro=el8.1 testlist=sanity-sec env=ONLY="36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 52 53" fstype=zfs mdscount=2 mdtcount=4 Test-Parameters: clientarch=x86_64 clientcount=1 clientdistro=el8.1 serverdistro=el8.1 testlist=sanity-sec env=ONLY="36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 52 53" fstype=ldiskfs mdscount=2 mdtcount=4 Test-Parameters: clientarch=x86_64 clientcount=1 clientdistro=el8.1 serverdistro=el8.1 testlist=sanity-sec env=ONLY="36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 52 53" fstype=zfs mdscount=2 mdtcount=4 Test-Parameters: clientdistro=el8.1 testlist=sanity-sec env=ONLY="36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52" mdscount=2 mdtcount=4 Signed-off-by: Sebastien Buisson Change-Id: Iee4b4d9e70c2e8c8e12061c39400cf6a8c03bac3 Reviewed-on: https://review.whamcloud.com/39315 Tested-by: jenkins Tested-by: Maloo Reviewed-by: Andreas Dilger Reviewed-by: Wang Shilong --- lustre/llite/file.c | 26 +++++++--- lustre/osc/osc_request.c | 79 ++++++++++++++++++------------ lustre/tests/sanity-sec.sh | 119 +++++++++++++++++++++++++++++++++++++++++++-- 3 files changed, 183 insertions(+), 41 deletions(-) diff --git a/lustre/llite/file.c b/lustre/llite/file.c index 9cb9d00..d68c232 100644 --- a/lustre/llite/file.c +++ b/lustre/llite/file.c @@ -440,13 +440,25 @@ static inline int ll_dom_readpage(void *data, struct page *page) if (!llcrypt_has_encryption_key(inode)) CDEBUG(D_SEC, "no enc key for "DFID"\n", PFID(ll_inode2fid(inode))); - /* decrypt only if page is not empty */ - else if (memcmp(page_address(page), - page_address(ZERO_PAGE(0)), - PAGE_SIZE) != 0) - rc = llcrypt_decrypt_pagecache_blocks(page, - PAGE_SIZE, - 0); + else { + unsigned int offs = 0; + + while (offs < PAGE_SIZE) { + /* decrypt only if page is not empty */ + if (memcmp(page_address(page) + offs, + page_address(ZERO_PAGE(0)), + LUSTRE_ENCRYPTION_UNIT_SIZE) == 0) + break; + + rc = llcrypt_decrypt_pagecache_blocks(page, + LUSTRE_ENCRYPTION_UNIT_SIZE, + offs); + if (rc) + break; + + offs += LUSTRE_ENCRYPTION_UNIT_SIZE; + } + } } unlock_page(page); diff --git a/lustre/osc/osc_request.c b/lustre/osc/osc_request.c index c9db263..1acf2d1 100644 --- a/lustre/osc/osc_request.c +++ b/lustre/osc/osc_request.c @@ -1406,8 +1406,12 @@ osc_brw_prep_request(int cmd, struct client_obd *cli, struct obdo *oa, struct page *data_page = NULL; bool retried = false; bool lockedbymyself; + u32 nunits = (pg->off & ~PAGE_MASK) + pg->count; retry_encrypt: + if (nunits & ~LUSTRE_ENCRYPTION_MASK) + nunits = (nunits & LUSTRE_ENCRYPTION_MASK) + + LUSTRE_ENCRYPTION_UNIT_SIZE; /* The page can already be locked when we arrive here. * This is possible when cl_page_assume/vvp_page_assume * is stuck on wait_on_page_writeback with page lock @@ -1420,7 +1424,7 @@ retry_encrypt: lockedbymyself = trylock_page(pg->pg); data_page = llcrypt_encrypt_pagecache_blocks(pg->pg, - PAGE_SIZE, 0, + nunits, 0, GFP_NOFS); if (lockedbymyself) unlock_page(pg->pg); @@ -1442,24 +1446,29 @@ retry_encrypt: oa->o_size = oap->oap_count + oap->oap_obj_off + oap->oap_page_off; } - /* len is forced to PAGE_SIZE, and poff to 0 + /* len is forced to nunits, and relative offset to 0 * so store the old, clear text info */ - pg->bp_count_diff = PAGE_SIZE - pg->count; - pg->count = PAGE_SIZE; + pg->bp_count_diff = nunits - pg->count; + pg->count = nunits; pg->bp_off_diff = pg->off & ~PAGE_MASK; pg->off = pg->off & PAGE_MASK; } } else if (opc == OST_READ && inode && IS_ENCRYPTED(inode)) { for (i = 0; i < page_count; i++) { struct brw_page *pg = pga[i]; - - /* count/off are forced to cover the whole page so that - * all encrypted data is stored on the OST, so adjust - * bp_{count,off}_diff for the size of the clear text. + u32 nunits = (pg->off & ~PAGE_MASK) + pg->count; + + if (nunits & ~LUSTRE_ENCRYPTION_MASK) + nunits = (nunits & LUSTRE_ENCRYPTION_MASK) + + LUSTRE_ENCRYPTION_UNIT_SIZE; + /* count/off are forced to cover the whole encryption + * unit size so that all encrypted data is stored on the + * OST, so adjust bp_{count,off}_diff for the size of + * the clear text. */ - pg->bp_count_diff = PAGE_SIZE - pg->count; - pg->count = PAGE_SIZE; + pg->bp_count_diff = nunits - pg->count; + pg->count = nunits; pg->bp_off_diff = pg->off & ~PAGE_MASK; pg->off = pg->off & PAGE_MASK; } @@ -2070,30 +2079,38 @@ static int osc_brw_fini_request(struct ptlrpc_request *req, int rc) } for (idx = 0; idx < aa->aa_page_count; idx++) { struct brw_page *pg = aa->aa_ppga[idx]; + unsigned int offs = 0; + + while (offs < PAGE_SIZE) { + /* do not decrypt if page is all 0s */ + if (memchr_inv(page_address(pg->pg) + offs, 0, + LUSTRE_ENCRYPTION_UNIT_SIZE) == NULL) { + /* if page is empty forward info to + * upper layers (ll_io_zero_page) by + * clearing PagePrivate2 + */ + if (!offs) + ClearPagePrivate2(pg->pg); + break; + } - /* do not decrypt if page is all 0s */ - if (memchr_inv(page_address(pg->pg), 0, - PAGE_SIZE) == NULL) { - /* if page is empty forward info to upper layers - * (ll_io_zero_page) by clearing PagePrivate2 + /* The page is already locked when we arrive here, + * except when we deal with a twisted page for + * specific Direct IO support, in which case + * PageChecked flag is set on page. */ - ClearPagePrivate2(pg->pg); - continue; + if (PageChecked(pg->pg)) + lock_page(pg->pg); + rc = llcrypt_decrypt_pagecache_blocks(pg->pg, + LUSTRE_ENCRYPTION_UNIT_SIZE, + offs); + if (PageChecked(pg->pg)) + unlock_page(pg->pg); + if (rc) + GOTO(out, rc); + + offs += LUSTRE_ENCRYPTION_UNIT_SIZE; } - - /* The page is already locked when we arrive here, - * except when we deal with a twisted page for - * specific Direct IO support, in which case - * PageChecked flag is set on page. - */ - if (PageChecked(pg->pg)) - lock_page(pg->pg); - rc = llcrypt_decrypt_pagecache_blocks(pg->pg, - PAGE_SIZE, 0); - if (PageChecked(pg->pg)) - unlock_page(pg->pg); - if (rc) - GOTO(out, rc); } } diff --git a/lustre/tests/sanity-sec.sh b/lustre/tests/sanity-sec.sh index f8ca92e..b9a0e43 100755 --- a/lustre/tests/sanity-sec.sh +++ b/lustre/tests/sanity-sec.sh @@ -2787,6 +2787,7 @@ test_38() { local srvsz=0 local filesz local bsize + local pagesz=$(getconf PAGE_SIZE) $LCTL get_param mdc.*.import | grep -q client_encryption || skip "client encryption not supported" @@ -2806,6 +2807,7 @@ test_38() { dd if=$tmpfile of=$testfile bs=4 count=1 seek=$blksz \ oflag=seek_bytes conv=fsync + blksz=$(($blksz > $pagesz ? $blksz : $pagesz)) # check that in-memory representation of file is correct bsize=$(stat --format=%B $testfile) filesz=$(stat --format=%b $testfile) @@ -3153,7 +3155,7 @@ test_43() { # create file tr '\0' '1' < /dev/zero | - dd of=$tmpfile bs=$pagesz count=1 conv=fsync + dd of=$tmpfile bs=1 count=$pagesz conv=fsync $LFS setstripe -c1 -i0 -S 256k $testfile cp $tmpfile $testfile @@ -3431,7 +3433,7 @@ test_48a() { # create file, 4 x PAGE_SIZE long tr '\0' '1' < /dev/zero | - dd of=$tmpfile bs=4x$pagesz count=1 conv=fsync + dd of=$tmpfile bs=1 count=4x$pagesz conv=fsync $LFS setstripe -c1 -i0 $testfile cp $tmpfile $testfile echo "abc" > $tmpfile2 @@ -3842,7 +3844,7 @@ test_52() { error "mirror 2 is corrupted" tr '\0' '2' < /dev/zero | - dd of=$tmpfile bs=9000 count=1 conv=fsync + dd of=$tmpfile bs=1 count=9000 conv=fsync $LFS mirror write -N 1 -i $tmpfile $testfile || error "could not write to mirror 1" @@ -3854,6 +3856,117 @@ test_52() { } run_test 52 "Mirrored encrypted file" +test_53() { + local testfile=$DIR/$tdir/$tfile + local testfile2=$DIR2/$tdir/$tfile + local tmpfile=$TMP/$tfile.tmp + local resfile=$TMP/$tfile.res + local pagesz + local filemd5 + + $LCTL get_param mdc.*.import | grep -q client_encryption || + skip "client encryption not supported" + + mount.lustre --help |& grep -q "test_dummy_encryption:" || + skip "need dummy encryption support" + + pagesz=$(getconf PAGESIZE) + [[ $pagesz == 65536 ]] || skip "Need 64K PAGE_SIZE client" + + do_node $mds1_HOST \ + "mount.lustre --help |& grep -q 'test_dummy_encryption:'" || + skip "need dummy encryption support on MDS client mount" + + # this test is probably useless now, but may turn out to be useful when + # Lustre supports servers with PAGE_SIZE != 4KB + pagesz=$(do_node $mds1_HOST getconf PAGESIZE) + [[ $pagesz == 4096 ]] || skip "Need 4K PAGE_SIZE MDS client" + + stack_trap cleanup_for_enc_tests EXIT + stack_trap "zconf_umount $mds1_HOST $MOUNT2" EXIT + setup_for_enc_tests + + $LFS setstripe -c1 -i0 $testfile + + # write from 1st client + cat /dev/urandom | tr -dc 'a-zA-Z0-9' | + dd of=$tmpfile bs=$((pagesz+3)) count=2 conv=fsync + dd if=$tmpfile of=$testfile bs=$((pagesz+3)) count=2 conv=fsync || + error "could not write to $testfile (1)" + + # read from 2nd client + # mount and IOs must be done in the same shell session, otherwise + # encryption key in session keyring is missing + do_node $mds1_HOST "mkdir -p $MOUNT2" + do_node $mds1_HOST \ + "$MOUNT_CMD -o ${MOUNT_OPTS},test_dummy_encryption \ + $MGSNID:/$FSNAME $MOUNT2 && \ + dd if=$testfile2 of=$resfile bs=$((pagesz+3)) count=2" || + error "could not read from $testfile2 (1)" + + # compare + filemd5=$(do_node $mds1_HOST md5sum $resfile | awk '{print $1}') + [ $filemd5 = $(md5sum $tmpfile | awk '{print $1}') ] || + error "file is corrupted (1)" + do_node $mds1_HOST rm -f $resfile + cancel_lru_locks + + # truncate from 2nd client + $TRUNCATE $tmpfile $((pagesz+3)) + zconf_umount $mds1_HOST $MOUNT2 || + error "umount $mds1_HOST $MOUNT2 failed (1)" + do_node $mds1_HOST "$MOUNT_CMD -o ${MOUNT_OPTS},test_dummy_encryption \ + $MGSNID:/$FSNAME $MOUNT2 && \ + $TRUNCATE $testfile2 $((pagesz+3))" || + error "could not truncate $testfile2 (1)" + + # compare + cmp -bl $tmpfile $testfile || + error "file is corrupted (2)" + rm -f $tmpfile $testfile + cancel_lru_locks + zconf_umount $mds1_HOST $MOUNT2 || + error "umount $mds1_HOST $MOUNT2 failed (2)" + + # do conversly + do_node $mds1_HOST \ + dd if=/dev/urandom of=$tmpfile bs=$((pagesz+3)) count=2 conv=fsync + # write from 2nd client + do_node $mds1_HOST \ + "$MOUNT_CMD -o ${MOUNT_OPTS},test_dummy_encryption \ + $MGSNID:/$FSNAME $MOUNT2 && \ + dd if=$tmpfile of=$testfile2 bs=$((pagesz+3)) count=2 conv=fsync" || + error "could not write to $testfile2 (2)" + + # read from 1st client + dd if=$testfile of=$resfile bs=$((pagesz+3)) count=2 || + error "could not read from $testfile (2)" + + # compare + filemd5=$(do_node $mds1_HOST md5sum -b $tmpfile | awk '{print $1}') + [ $filemd5 = $(md5sum -b $resfile | awk '{print $1}') ] || + error "file is corrupted (3)" + rm -f $resfile + cancel_lru_locks + + # truncate from 1st client + do_node $mds1_HOST "$TRUNCATE $tmpfile $((pagesz+3))" + $TRUNCATE $testfile $((pagesz+3)) || + error "could not truncate $testfile (2)" + + # compare + zconf_umount $mds1_HOST $MOUNT2 || + error "umount $mds1_HOST $MOUNT2 failed (3)" + do_node $mds1_HOST "$MOUNT_CMD -o ${MOUNT_OPTS},test_dummy_encryption \ + $MGSNID:/$FSNAME $MOUNT2 && \ + cmp -bl $tmpfile $testfile2" || + error "file is corrupted (4)" + + do_node $mds1_HOST rm -f $tmpfile + rm -f $tmpfile +} +run_test 53 "Mixed PAGE_SIZE clients" + log "cleanup: ======================================================" sec_unsetup() { -- 1.8.3.1