Whamcloud - gitweb
LU-12275 sec: encryption with different client PAGE_SIZE 15/39315/15
authorSebastien Buisson <sbuisson@ddn.com>
Wed, 8 Jul 2020 16:19:08 +0000 (16:19 +0000)
committerOleg Drokin <green@whamcloud.com>
Sat, 12 Sep 2020 15:43:50 +0000 (15:43 +0000)
In order to properly handle encryption/decryption on clients that have
a PAGE_SIZE != LUSTRE_ENCRYPTION_UNIT_SIZE (typically aarch64/ppc64),
a few adjustements are necessary:
- when encrypting, do not proceed with PAGE_SIZE as encryption length.
  Instead, round up to a multiple of LUSTRE_ENCRYPTION_UNIT_SIZE.
  On aarch64/ppc64, it avoids encrypting way beyond
  LUSTRE_ENCRYPTION_UNIT_SIZE when the page is not full.
- when decrypting, do not proceed with PAGE_SIZE as decryption length.
  Instead, do LUSTRE_ENCRYPTION_UNIT_SIZE length at a time. It enables
  proper detection of 'all 0s' sent by servers for content beyond file
  size.

Regarding tests, add sanity-sec test_53 to exercise encryption from
clients with different PAGE_SIZE.
The trick to achieve this with AT is to expect the client to have 64KB
PAGE_SIZE, and the servers to have 4KB PAGE_SIZE, and then mount a
client from the MDS node.
This also means code running on server side needs to have client
encryption support enabled, so CentOS/RHEL 8 at least.

Test-Parameters: trivial
Test-Parameters: clientarch=aarch64 clientcount=1 clientdistro=el8.1 serverdistro=el8.1 testlist=sanity-sec env=ONLY="36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 52 53" fstype=ldiskfs mdscount=2 mdtcount=4
Test-Parameters: clientarch=aarch64 clientcount=1 clientdistro=el8.1 serverdistro=el8.1 testlist=sanity-sec env=ONLY="36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 52 53" fstype=zfs mdscount=2 mdtcount=4
Test-Parameters: clientarch=x86_64 clientcount=1 clientdistro=el8.1 serverdistro=el8.1 testlist=sanity-sec env=ONLY="36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 52 53" fstype=ldiskfs mdscount=2 mdtcount=4
Test-Parameters: clientarch=x86_64 clientcount=1 clientdistro=el8.1 serverdistro=el8.1 testlist=sanity-sec env=ONLY="36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 52 53" fstype=zfs mdscount=2 mdtcount=4
Test-Parameters: clientdistro=el8.1 testlist=sanity-sec env=ONLY="36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52" mdscount=2 mdtcount=4
Signed-off-by: Sebastien Buisson <sbuisson@ddn.com>
Change-Id: Iee4b4d9e70c2e8c8e12061c39400cf6a8c03bac3
Reviewed-on: https://review.whamcloud.com/39315
Tested-by: jenkins <devops@whamcloud.com>
Tested-by: Maloo <maloo@whamcloud.com>
Reviewed-by: Andreas Dilger <adilger@whamcloud.com>
Reviewed-by: Wang Shilong <wshilong@whamcloud.com>
lustre/llite/file.c
lustre/osc/osc_request.c
lustre/tests/sanity-sec.sh

index 9cb9d00..d68c232 100644 (file)
@@ -440,13 +440,25 @@ static inline int ll_dom_readpage(void *data, struct page *page)
                if (!llcrypt_has_encryption_key(inode))
                        CDEBUG(D_SEC, "no enc key for "DFID"\n",
                               PFID(ll_inode2fid(inode)));
-               /* decrypt only if page is not empty */
-               else if (memcmp(page_address(page),
-                               page_address(ZERO_PAGE(0)),
-                               PAGE_SIZE) != 0)
-                       rc = llcrypt_decrypt_pagecache_blocks(page,
-                                                             PAGE_SIZE,
-                                                             0);
+               else {
+                       unsigned int offs = 0;
+
+                       while (offs < PAGE_SIZE) {
+                               /* decrypt only if page is not empty */
+                               if (memcmp(page_address(page) + offs,
+                                          page_address(ZERO_PAGE(0)),
+                                          LUSTRE_ENCRYPTION_UNIT_SIZE) == 0)
+                                       break;
+
+                               rc = llcrypt_decrypt_pagecache_blocks(page,
+                                                   LUSTRE_ENCRYPTION_UNIT_SIZE,
+                                                                     offs);
+                               if (rc)
+                                       break;
+
+                               offs += LUSTRE_ENCRYPTION_UNIT_SIZE;
+                       }
+               }
        }
        unlock_page(page);
 
index c9db263..1acf2d1 100644 (file)
@@ -1406,8 +1406,12 @@ osc_brw_prep_request(int cmd, struct client_obd *cli, struct obdo *oa,
                        struct page *data_page = NULL;
                        bool retried = false;
                        bool lockedbymyself;
+                       u32 nunits = (pg->off & ~PAGE_MASK) + pg->count;
 
 retry_encrypt:
+                       if (nunits & ~LUSTRE_ENCRYPTION_MASK)
+                               nunits = (nunits & LUSTRE_ENCRYPTION_MASK) +
+                                       LUSTRE_ENCRYPTION_UNIT_SIZE;
                        /* The page can already be locked when we arrive here.
                         * This is possible when cl_page_assume/vvp_page_assume
                         * is stuck on wait_on_page_writeback with page lock
@@ -1420,7 +1424,7 @@ retry_encrypt:
                        lockedbymyself = trylock_page(pg->pg);
                        data_page =
                                llcrypt_encrypt_pagecache_blocks(pg->pg,
-                                                                PAGE_SIZE, 0,
+                                                                nunits, 0,
                                                                 GFP_NOFS);
                        if (lockedbymyself)
                                unlock_page(pg->pg);
@@ -1442,24 +1446,29 @@ retry_encrypt:
                                oa->o_size = oap->oap_count +
                                        oap->oap_obj_off + oap->oap_page_off;
                        }
-                       /* len is forced to PAGE_SIZE, and poff to 0
+                       /* len is forced to nunits, and relative offset to 0
                         * so store the old, clear text info
                         */
-                       pg->bp_count_diff = PAGE_SIZE - pg->count;
-                       pg->count = PAGE_SIZE;
+                       pg->bp_count_diff = nunits - pg->count;
+                       pg->count = nunits;
                        pg->bp_off_diff = pg->off & ~PAGE_MASK;
                        pg->off = pg->off & PAGE_MASK;
                }
        } else if (opc == OST_READ && inode && IS_ENCRYPTED(inode)) {
                for (i = 0; i < page_count; i++) {
                        struct brw_page *pg = pga[i];
-
-                       /* count/off are forced to cover the whole page so that
-                        * all encrypted data is stored on the OST, so adjust
-                        * bp_{count,off}_diff for the size of the clear text.
+                       u32 nunits = (pg->off & ~PAGE_MASK) + pg->count;
+
+                       if (nunits & ~LUSTRE_ENCRYPTION_MASK)
+                               nunits = (nunits & LUSTRE_ENCRYPTION_MASK) +
+                                       LUSTRE_ENCRYPTION_UNIT_SIZE;
+                       /* count/off are forced to cover the whole encryption
+                        * unit size so that all encrypted data is stored on the
+                        * OST, so adjust bp_{count,off}_diff for the size of
+                        * the clear text.
                         */
-                       pg->bp_count_diff = PAGE_SIZE - pg->count;
-                       pg->count = PAGE_SIZE;
+                       pg->bp_count_diff = nunits - pg->count;
+                       pg->count = nunits;
                        pg->bp_off_diff = pg->off & ~PAGE_MASK;
                        pg->off = pg->off & PAGE_MASK;
                }
@@ -2070,30 +2079,38 @@ static int osc_brw_fini_request(struct ptlrpc_request *req, int rc)
                }
                for (idx = 0; idx < aa->aa_page_count; idx++) {
                        struct brw_page *pg = aa->aa_ppga[idx];
+                       unsigned int offs = 0;
+
+                       while (offs < PAGE_SIZE) {
+                               /* do not decrypt if page is all 0s */
+                               if (memchr_inv(page_address(pg->pg) + offs, 0,
+                                        LUSTRE_ENCRYPTION_UNIT_SIZE) == NULL) {
+                                       /* if page is empty forward info to
+                                        * upper layers (ll_io_zero_page) by
+                                        * clearing PagePrivate2
+                                        */
+                                       if (!offs)
+                                               ClearPagePrivate2(pg->pg);
+                                       break;
+                               }
 
-                       /* do not decrypt if page is all 0s */
-                       if (memchr_inv(page_address(pg->pg), 0,
-                                      PAGE_SIZE) == NULL) {
-                               /* if page is empty forward info to upper layers
-                                * (ll_io_zero_page) by clearing PagePrivate2
+                               /* The page is already locked when we arrive here,
+                                * except when we deal with a twisted page for
+                                * specific Direct IO support, in which case
+                                * PageChecked flag is set on page.
                                 */
-                               ClearPagePrivate2(pg->pg);
-                               continue;
+                               if (PageChecked(pg->pg))
+                                       lock_page(pg->pg);
+                               rc = llcrypt_decrypt_pagecache_blocks(pg->pg,
+                                                   LUSTRE_ENCRYPTION_UNIT_SIZE,
+                                                                     offs);
+                               if (PageChecked(pg->pg))
+                                       unlock_page(pg->pg);
+                               if (rc)
+                                       GOTO(out, rc);
+
+                               offs += LUSTRE_ENCRYPTION_UNIT_SIZE;
                        }
-
-                       /* The page is already locked when we arrive here,
-                        * except when we deal with a twisted page for
-                        * specific Direct IO support, in which case
-                        * PageChecked flag is set on page.
-                        */
-                       if (PageChecked(pg->pg))
-                               lock_page(pg->pg);
-                       rc = llcrypt_decrypt_pagecache_blocks(pg->pg,
-                                                             PAGE_SIZE, 0);
-                       if (PageChecked(pg->pg))
-                               unlock_page(pg->pg);
-                       if (rc)
-                               GOTO(out, rc);
                }
        }
 
index f8ca92e..b9a0e43 100755 (executable)
@@ -2787,6 +2787,7 @@ test_38() {
        local srvsz=0
        local filesz
        local bsize
+       local pagesz=$(getconf PAGE_SIZE)
 
        $LCTL get_param mdc.*.import | grep -q client_encryption ||
                skip "client encryption not supported"
@@ -2806,6 +2807,7 @@ test_38() {
        dd if=$tmpfile of=$testfile bs=4 count=1 seek=$blksz \
                oflag=seek_bytes conv=fsync
 
+       blksz=$(($blksz > $pagesz ? $blksz : $pagesz))
        # check that in-memory representation of file is correct
        bsize=$(stat --format=%B $testfile)
        filesz=$(stat --format=%b $testfile)
@@ -3153,7 +3155,7 @@ test_43() {
 
        # create file
        tr '\0' '1' < /dev/zero |
-               dd of=$tmpfile bs=$pagesz count=1 conv=fsync
+               dd of=$tmpfile bs=1 count=$pagesz conv=fsync
        $LFS setstripe -c1 -i0 -S 256k $testfile
        cp $tmpfile $testfile
 
@@ -3431,7 +3433,7 @@ test_48a() {
 
        # create file, 4 x PAGE_SIZE long
        tr '\0' '1' < /dev/zero |
-               dd of=$tmpfile bs=4x$pagesz count=1 conv=fsync
+               dd of=$tmpfile bs=1 count=4x$pagesz conv=fsync
        $LFS setstripe -c1 -i0 $testfile
        cp $tmpfile $testfile
        echo "abc" > $tmpfile2
@@ -3842,7 +3844,7 @@ test_52() {
                error "mirror 2 is corrupted"
 
        tr '\0' '2' < /dev/zero |
-           dd of=$tmpfile bs=9000 count=1 conv=fsync
+           dd of=$tmpfile bs=1 count=9000 conv=fsync
 
        $LFS mirror write -N 1 -i $tmpfile $testfile ||
                error "could not write to mirror 1"
@@ -3854,6 +3856,117 @@ test_52() {
 }
 run_test 52 "Mirrored encrypted file"
 
+test_53() {
+       local testfile=$DIR/$tdir/$tfile
+       local testfile2=$DIR2/$tdir/$tfile
+       local tmpfile=$TMP/$tfile.tmp
+       local resfile=$TMP/$tfile.res
+       local pagesz
+       local filemd5
+
+       $LCTL get_param mdc.*.import | grep -q client_encryption ||
+               skip "client encryption not supported"
+
+       mount.lustre --help |& grep -q "test_dummy_encryption:" ||
+               skip "need dummy encryption support"
+
+       pagesz=$(getconf PAGESIZE)
+       [[ $pagesz == 65536 ]] || skip "Need 64K PAGE_SIZE client"
+
+       do_node $mds1_HOST \
+               "mount.lustre --help |& grep -q 'test_dummy_encryption:'" ||
+                       skip "need dummy encryption support on MDS client mount"
+
+       # this test is probably useless now, but may turn out to be useful when
+       # Lustre supports servers with PAGE_SIZE != 4KB
+       pagesz=$(do_node $mds1_HOST getconf PAGESIZE)
+       [[ $pagesz == 4096 ]] || skip "Need 4K PAGE_SIZE MDS client"
+
+       stack_trap cleanup_for_enc_tests EXIT
+       stack_trap "zconf_umount $mds1_HOST $MOUNT2" EXIT
+       setup_for_enc_tests
+
+       $LFS setstripe -c1 -i0 $testfile
+
+       # write from 1st client
+       cat /dev/urandom | tr -dc 'a-zA-Z0-9' |
+               dd of=$tmpfile bs=$((pagesz+3)) count=2 conv=fsync
+       dd if=$tmpfile of=$testfile bs=$((pagesz+3)) count=2 conv=fsync ||
+               error "could not write to $testfile (1)"
+
+       # read from 2nd client
+       # mount and IOs must be done in the same shell session, otherwise
+       # encryption key in session keyring is missing
+       do_node $mds1_HOST "mkdir -p $MOUNT2"
+       do_node $mds1_HOST \
+               "$MOUNT_CMD -o ${MOUNT_OPTS},test_dummy_encryption \
+                $MGSNID:/$FSNAME $MOUNT2 && \
+                dd if=$testfile2 of=$resfile bs=$((pagesz+3)) count=2" ||
+               error "could not read from $testfile2 (1)"
+
+       # compare
+       filemd5=$(do_node $mds1_HOST md5sum $resfile | awk '{print $1}')
+       [ $filemd5 = $(md5sum $tmpfile | awk '{print $1}') ] ||
+               error "file is corrupted (1)"
+       do_node $mds1_HOST rm -f $resfile
+       cancel_lru_locks
+
+       # truncate from 2nd client
+       $TRUNCATE $tmpfile $((pagesz+3))
+       zconf_umount $mds1_HOST $MOUNT2 ||
+               error "umount $mds1_HOST $MOUNT2 failed (1)"
+       do_node $mds1_HOST "$MOUNT_CMD -o ${MOUNT_OPTS},test_dummy_encryption \
+                          $MGSNID:/$FSNAME $MOUNT2 && \
+                          $TRUNCATE $testfile2 $((pagesz+3))" ||
+               error "could not truncate $testfile2 (1)"
+
+       # compare
+       cmp -bl $tmpfile $testfile ||
+               error "file is corrupted (2)"
+       rm -f $tmpfile $testfile
+       cancel_lru_locks
+       zconf_umount $mds1_HOST $MOUNT2 ||
+               error "umount $mds1_HOST $MOUNT2 failed (2)"
+
+       # do conversly
+       do_node $mds1_HOST \
+             dd if=/dev/urandom of=$tmpfile bs=$((pagesz+3)) count=2 conv=fsync
+       # write from 2nd client
+       do_node $mds1_HOST \
+          "$MOUNT_CMD -o ${MOUNT_OPTS},test_dummy_encryption \
+           $MGSNID:/$FSNAME $MOUNT2 && \
+           dd if=$tmpfile of=$testfile2 bs=$((pagesz+3)) count=2 conv=fsync" ||
+               error "could not write to $testfile2 (2)"
+
+       # read from 1st client
+       dd if=$testfile of=$resfile bs=$((pagesz+3)) count=2 ||
+               error "could not read from $testfile (2)"
+
+       # compare
+       filemd5=$(do_node $mds1_HOST md5sum -b $tmpfile | awk '{print $1}')
+       [ $filemd5 = $(md5sum -b $resfile | awk '{print $1}') ] ||
+               error "file is corrupted (3)"
+       rm -f $resfile
+       cancel_lru_locks
+
+       # truncate from 1st client
+       do_node $mds1_HOST "$TRUNCATE $tmpfile $((pagesz+3))"
+       $TRUNCATE $testfile $((pagesz+3)) ||
+               error "could not truncate $testfile (2)"
+
+       # compare
+       zconf_umount $mds1_HOST $MOUNT2 ||
+               error "umount $mds1_HOST $MOUNT2 failed (3)"
+       do_node $mds1_HOST "$MOUNT_CMD -o ${MOUNT_OPTS},test_dummy_encryption \
+                          $MGSNID:/$FSNAME $MOUNT2 && \
+                          cmp -bl $tmpfile $testfile2" ||
+               error "file is corrupted (4)"
+
+       do_node $mds1_HOST rm -f $tmpfile
+       rm -f $tmpfile
+}
+run_test 53 "Mixed PAGE_SIZE clients"
+
 log "cleanup: ======================================================"
 
 sec_unsetup() {