LU-12358 pcc: add project quota support on PCC backend

Current PCC can enforce a quota limitation of the capacity usage
for each user and group to provide cache isolation. An admin
can specify the quota enforcement on the local PCC file system.

Users can perform PCC-cached I/O on files until they receive a
return value -ENOSPC of -EDQUOT, which means that they hit the
quota limit or that there is no free capacity left on the local
PCC backend fs during I/O or the attach process. At this time,
I/O will fall back to the normal I/O path.

This patch adds project quota on the PCC backend file system
along with user/group quota.

With this feature, it can have multiple PCC backends on a single
client with different caching rules, so we can define upfront
how much of the client FS can be used for each cache.

Test-Parameters: clientcount=3 testlist=sanity-pcc,sanity-pcc,sanity-pcc
Signed-off-by: Qian Yingjin <qian@ddn.com>
Change-Id: Ib93da953d4a3a7091f62094f8175bde91e819895
Reviewed-on: https://review.whamcloud.com/c/fs/lustre-release/+/39831
Tested-by: jenkins <devops@whamcloud.com>
Tested-by: Maloo <maloo@whamcloud.com>
Reviewed-by: Andreas Dilger <adilger@whamcloud.com>
Reviewed-by: James Simmons <jsimmons@infradead.org>
Reviewed-by: Li Xi <lixi@ddn.com>
Reviewed-by: Oleg Drokin <green@whamcloud.com>

diff --git a/lustre/llite/pcc.c b/lustre/llite/pcc.c
index 5f4cf07..49189d7 100644 (file)
--- a/lustre/llite/pcc.c
+++ b/lustre/llite/pcc.c
@@ -110,6 +110,10 @@
 #include <lustre_compat.h>
 #include "llite_internal.h"
 
+#ifdef HAVE_FILEATTR_GET
+#include <linux/fileattr.h>
+#endif
+
 struct kmem_cache *pcc_inode_slab;
 
 int pcc_super_init(struct pcc_super *super)
@@ -619,41 +623,53 @@ pcc_parse_value_pair(struct pcc_cmd *cmd, char *buffer)
                        return -EINVAL;
                cmd->u.pccc_add.pccc_roid = id;
        } else if (strcmp(key, "auto_attach") == 0) {
-               rc = kstrtoul(val, 10, &id);
+               rc = kstrtobool(val, &enable);
                if (rc)
                        return rc;
-               if (id == 0)
+               if (enable)
+                       cmd->u.pccc_add.pccc_flags |= PCC_DATASET_AUTO_ATTACH;
+               else
                        cmd->u.pccc_add.pccc_flags &= ~PCC_DATASET_AUTO_ATTACH;
        } else if (strcmp(key, "open_attach") == 0) {
-               rc = kstrtoul(val, 10, &id);
+               rc = kstrtobool(val, &enable);
                if (rc)
                        return rc;
-               if (id == 0)
+               if (enable)
+                       cmd->u.pccc_add.pccc_flags |= PCC_DATASET_OPEN_ATTACH;
+               else
                        cmd->u.pccc_add.pccc_flags &= ~PCC_DATASET_OPEN_ATTACH;
        } else if (strcmp(key, "io_attach") == 0) {
-               rc = kstrtoul(val, 10, &id);
+               rc = kstrtobool(val, &enable);
                if (rc)
                        return rc;
-               if (id == 0)
+               if (enable)
+                       cmd->u.pccc_add.pccc_flags |= PCC_DATASET_IO_ATTACH;
+               else
                        cmd->u.pccc_add.pccc_flags &= ~PCC_DATASET_IO_ATTACH;
        } else if (strcmp(key, "stat_attach") == 0) {
-               rc = kstrtoul(val, 10, &id);
+               rc = kstrtobool(val, &enable);
                if (rc)
                        return rc;
-               if (id == 0)
+               if (enable)
+                       cmd->u.pccc_add.pccc_flags |= PCC_DATASET_STAT_ATTACH;
+               else
                        cmd->u.pccc_add.pccc_flags &= ~PCC_DATASET_STAT_ATTACH;
        } else if (strcmp(key, "rwpcc") == 0 || strcmp(key, "pccrw") == 0) {
-               rc = kstrtoul(val, 10, &id);
+               rc = kstrtobool(val, &enable);
                if (rc)
                        return rc;
-               if (id > 0)
+               if (enable)
                        cmd->u.pccc_add.pccc_flags |= PCC_DATASET_PCCRW;
+               else
+                       cmd->u.pccc_add.pccc_flags &= ~PCC_DATASET_PCCRW;
        } else if (strcmp(key, "ropcc") == 0 || strcmp(key, "pccro") == 0) {
-               rc = kstrtoul(val, 10, &id);
+               rc = kstrtobool(val, &enable);
                if (rc)
                        return rc;
-               if (id > 0)
+               if (enable)
                        cmd->u.pccc_add.pccc_flags |= PCC_DATASET_PCCRO;
+               else
+                       cmd->u.pccc_add.pccc_flags &= ~PCC_DATASET_PCCRO;
        } else if (strcmp(key, "mmap_conv") == 0) {
                rc = kstrtobool(val, &enable);
                if (rc)
@@ -666,6 +682,14 @@ pcc_parse_value_pair(struct pcc_cmd *cmd, char *buffer)
 #endif
                else
                        cmd->u.pccc_add.pccc_flags &= ~PCC_DATASET_MMAP_CONV;
+       } else if (strcmp(key, "proj_quota") == 0) {
+               rc = kstrtobool(val, &enable);
+               if (rc)
+                       return rc;
+               if (enable)
+                       cmd->u.pccc_add.pccc_flags |= PCC_DATASET_PROJ_QUOTA;
+               else
+                       cmd->u.pccc_add.pccc_flags &= ~PCC_DATASET_PROJ_QUOTA;
        } else if (strcmp(key, "hsmtool") == 0) {
                cmd->u.pccc_add.pccc_hsmtool_type = hsmtool_string2type(val);
                if (cmd->u.pccc_add.pccc_hsmtool_type != HSMTOOL_POSIX_V1 &&
@@ -688,8 +712,9 @@ pcc_parse_value_pairs(struct pcc_cmd *cmd, char *buffer)
        switch (cmd->pccc_cmd) {
        case PCC_ADD_DATASET:
                cmd->u.pccc_add.pccc_hsmtool_type = HSMTOOL_UNKNOWN;
-               /* Enable auto attach by default */
-               cmd->u.pccc_add.pccc_flags |= PCC_DATASET_AUTO_ATTACH;
+               /* Enable these features by default */
+               cmd->u.pccc_add.pccc_flags |= PCC_DATASET_AUTO_ATTACH |
+                                             PCC_DATASET_PROJ_QUOTA;
                break;
        case PCC_DEL_DATASET:
        case PCC_CLEAR_ALL:
@@ -1789,12 +1814,17 @@ static int pcc_try_auto_attach(struct inode *inode, bool *cached,
                RETURN(-EINVAL);
        }
 
-       if (clt.cl_is_released)
+       if (clt.cl_is_released) {
                rc = pcc_try_datasets_attach(inode, iot, clt.cl_layout_gen,
                                             LU_PCC_READWRITE, cached);
-       else if (clt.cl_is_rdonly)
+       } else if (clt.cl_is_rdonly) {
+               /* Not try read-only attach for data modification operations */
+               if (iot == PIT_WRITE || iot == PIT_SETATTR)
+                       RETURN(0);
+
                rc = pcc_try_datasets_attach(inode, iot, clt.cl_layout_gen,
                                             LU_PCC_READONLY, cached);
+       }
 
        RETURN(rc);
 }
@@ -3039,7 +3069,6 @@ out:
 
 /*
  * Reset uid, gid or size for the PCC copy masked by @valid.
- * TODO: Set the project ID for PCC copy.
  */
 static int pcc_inode_reset_iattr(struct dentry *dentry, unsigned int valid,
                                 kuid_t uid, kgid_t gid, loff_t size)
@@ -3062,6 +3091,84 @@ static int pcc_inode_reset_iattr(struct dentry *dentry, unsigned int valid,
        RETURN(rc);
 }
 
+static int __pcc_file_reset_projid(struct file *file, __u32 projid)
+{
+#ifdef HAVE_FILEATTR_GET
+       struct fileattr fa = { .fsx_projid = projid };
+       struct dentry *dentry = file->f_path.dentry;
+       struct inode *inode = d_inode(dentry);
+       int rc;
+
+       /* project quota not supported on backing filesystem */
+       if (!inode->i_op->fileattr_set)
+               return -EOPNOTSUPP;
+
+       rc = inode->i_op->fileattr_set(&init_user_ns, dentry, &fa);
+#else
+       struct fsxattr fsx = { .fsx_projid = projid };
+       mm_segment_t old_fs;
+       int rc;
+
+       /* project quota not supported on backing filesystem */
+       if (!file->f_op->unlocked_ioctl)
+               return -EOPNOTSUPP;
+
+       old_fs = get_fs();
+       set_fs(KERNEL_DS);
+       rc = file->f_op->unlocked_ioctl(file, FS_IOC_FSSETXATTR,
+                                       (unsigned long)&fsx);
+       set_fs(old_fs);
+#endif
+       return rc;
+}
+
+/* Set the project ID for PCC copy.*/
+static int pcc_file_reset_projid(struct pcc_dataset *dataset, struct file *file,
+                                __u32 projid)
+{
+       int rc;
+
+       ENTRY;
+
+       if (!(dataset->pccd_flags & PCC_DATASET_PROJ_QUOTA))
+               RETURN(0);
+
+       rc = __pcc_file_reset_projid(file, projid);
+       if (rc == -EOPNOTSUPP || rc == -ENOTTY) {
+               CWARN("%s: cache fs project quota off, disabling: rc = %d\n",
+                     dataset->pccd_pathname, rc);
+               dataset->pccd_flags &= ~PCC_DATASET_PROJ_QUOTA;
+               RETURN(0);
+       }
+
+       RETURN(rc);
+}
+
+static int pcc_inode_reset_projid(struct pcc_dataset *dataset,
+                                 struct dentry *dentry, __u32 projid)
+{
+       struct path path;
+       struct file *file;
+       int rc;
+
+       ENTRY;
+
+       if (!(dataset->pccd_flags & PCC_DATASET_PROJ_QUOTA))
+               RETURN(0);
+
+       path.mnt = dataset->pccd_path.mnt;
+       path.dentry = dentry;
+       file = dentry_open(&path, O_WRONLY | O_LARGEFILE, current_cred());
+       if (IS_ERR_OR_NULL(file)) {
+               rc = file == NULL ? -EINVAL : PTR_ERR(file);
+               RETURN(rc);
+       }
+
+       rc = pcc_file_reset_projid(dataset, file, projid);
+       fput(file);
+       RETURN(rc);
+}
+
 int pcc_inode_create(struct super_block *sb, struct pcc_dataset *dataset,
                     struct lu_fid *fid, struct dentry **pcc_dentry)
 {
@@ -3106,6 +3213,11 @@ int pcc_inode_create_fini(struct inode *inode, struct pcc_create_attach *pca)
        if (rc)
                GOTO(out_put, rc);
 
+       rc = pcc_inode_reset_projid(pca->pca_dataset, pcc_dentry,
+                                   ll_i2info(inode)->lli_projid);
+       if (rc)
+               GOTO(out_put, rc);
+
        pcc_inode_attach_set(super, pca->pca_dataset, ll_i2info(inode),
                             pcci, pcc_dentry, LU_PCC_READWRITE);
 
@@ -3279,6 +3391,11 @@ static int pcc_attach_data_archive(struct file *file, struct inode *inode,
        if (rc)
                GOTO(out_fput, rc);
 
+       rc = pcc_file_reset_projid(dataset, pcc_filp,
+                                   ll_i2info(inode)->lli_projid);
+       if (rc)
+               GOTO(out_fput, rc);
+
        /*
         * When attach a file at file open() time with direct I/O mode, the
         * data copy from Lustre OSTs to PCC copy in kernel will report

diff --git a/lustre/llite/pcc.h b/lustre/llite/pcc.h
index 73be3fe..6d24d6e 100644 (file)
--- a/lustre/llite/pcc.h
+++ b/lustre/llite/pcc.h
@@ -137,6 +137,8 @@ enum pcc_dataset_flags {
        PCC_DATASET_PCC_DEFAULT = PCC_DATASET_PCCRO,
        /* Move pagecache from mapping of PCC copy to Lustre file for mmap */
        PCC_DATASET_MMAP_CONV   = 0x40,
+       /* Set the project ID for the PCC copy */
+       PCC_DATASET_PROJ_QUOTA  = 0x80,
 };
 
 struct pcc_dataset {

diff --git a/lustre/tests/sanity-pcc.sh b/lustre/tests/sanity-pcc.sh
index 65d412a..21d8292 100755 (executable)
--- a/lustre/tests/sanity-pcc.sh
+++ b/lustre/tests/sanity-pcc.sh
@@ -216,7 +216,7 @@ setup_loopdev() {
        local size=${4:-50}
 
        do_facet $facet mkdir -p $mntpt || error "mkdir -p $mntpt failed"
-       stack_trap "do_facet $facet rm -rf $mntpt" EXIT
+       stack_trap "do_facet $facet rmdir $mntpt" EXIT
        do_facet $facet dd if=/dev/zero of=$file bs=1M count=$size
        stack_trap "do_facet $facet rm -f $file" EXIT
        do_facet $facet mount
@@ -230,6 +230,25 @@ setup_loopdev() {
        stack_trap "umount_loopdev $facet $mntpt" EXIT
 }
 
+setup_loopdev_project() {
+       local facet=$1
+       local file=$2
+       local mntpt=$3
+       local size=${4:-50}
+
+       do_facet $facet mkdir -p $mntpt || error "mkdir -p $mntpt failed"
+       stack_trap "do_facet $facet rmdir $mntpt" EXIT
+       do_facet $facet dd if=/dev/zero of=$file bs=1M count=$size
+       stack_trap "do_facet $facet rm -f $file" EXIT
+       do_facet $facet mkfs.ext4 -O project,quota $file ||
+               error "mkfs.ext4 -O project,quota $file failed"
+       do_facet $facet file $file
+       do_facet $facet mount -t ext4 -o loop,prjquota $file $mntpt ||
+               error "mount -o loop,prjquota $file $mntpt failed"
+       stack_trap "umount_loopdev $facet $mntpt" EXIT
+       do_facet $facet mount | grep $mntpt
+}
+
 lpcc_rw_test() {
        local restore="$1"
        local project="$2"
@@ -411,8 +430,7 @@ test_1f() {
        local hsm_root="$mntpt/$tdir"
        local file=$DIR/$tdir/$tfile
 
-       ! is_project_quota_supported &&
-               skip "project quota is not supported"
+       is_project_quota_supported || skip "project quota is not supported"
 
        enable_project_quota
        setup_loopdev $SINGLEAGT $loopfile $mntpt 50
@@ -507,8 +525,7 @@ test_2a() {
        local hsm_root="$mntpt/$tdir"
        local agt_host=$(facet_active_host $SINGLEAGT)
 
-       ! is_project_quota_supported &&
-               skip "project quota is not supported" && return
+       is_project_quota_supported || skip "project quota is not supported"
 
        enable_project_quota
        setup_loopdev $SINGLEAGT $loopfile $mntpt 50
@@ -773,8 +790,7 @@ test_4() {
        local hsm_root="$mntpt/$tdir"
        local excepts="-e 7 -e 8 -e 9"
 
-       ! is_project_quota_supported &&
-               skip "project quota is not supported" && return
+       is_project_quota_supported || skip "project quota is not supported"
 
        enable_project_quota
        setup_loopdev $SINGLEAGT $loopfile $mntpt 50
@@ -1318,8 +1334,7 @@ test_13c() {
        local mntpt="/mnt/pcc.$tdir"
        local hsm_root="$mntpt/$tdir"
 
-       ! is_project_quota_supported &&
-               echo "Skip project quota is not supported" && return 0
+       is_project_quota_supported || skip "project quota is not supported"
 
        enable_project_quota
        setup_loopdev $SINGLEAGT $loopfile $mntpt 50
@@ -2886,8 +2901,7 @@ test_34() {
        $LCTL get_param -n mdc.*.connect_flags | grep -q pcc_ro ||
                skip "Server does not support PCC-RO"
 
-       ! is_project_quota_supported &&
-               skip "project quota is not supported"
+       is_project_quota_supported || skip "project quota is not supported"
 
        enable_project_quota
        setup_loopdev $SINGLEAGT $loopfile $mntpt 50
@@ -3109,6 +3123,56 @@ test_38() {
 }
 run_test 38 "Verify LFS pcc state does not trigger prefetch for auto PCC-RO"
 
+test_39() {
+       $LCTL get_param -n mdc.*.connect_flags | grep -q pcc_ro ||
+               skip "Server does not support PCC-RO"
+
+       quotaon --help |& grep -q 'project quotas' ||
+               skip "Not support project quota on local filesystem"
+
+       is_project_quota_supported || skip "project quota is not supported"
+
+       enable_project_quota
+
+       local loopfile="$TMP/$tfile"
+       local mntpt="/mnt/pcc.$tdir"
+       local hsm_root="$mntpt/$tdir"
+       local dir=$DIR/$tdir
+       local file=$dir/$tfile
+       local id=100
+
+       setup_loopdev_project $SINGLEAGT $loopfile $mntpt 50
+       do_facet $SINGLEAGT quotaon -Ppv $mntpt
+       do_facet $SINGLEAGT setquota -P $id 0 4096 0 0 $mntpt ||
+               error "setquota -P $id on $mntpt failed"
+       do_facet $SINGLEAGT repquota -Pvs $mntpt
+
+       do_facet $SINGLEAGT mkdir $hsm_root || error "mkdir $hsm_root failed"
+       setup_pcc_mapping $SINGLEAGT \
+               "projid={$id}\ roid=$HSM_ARCHIVE_NUMBER\ proj_quota=1\ pccro=1"
+       do_facet $SINGLEAGT $LCTL pcc list $MOUNT
+
+       do_facet $SINGLEAGT mkdir -p $dir || error "mkdir $dir failed"
+       do_facet $SINGLEAGT dd if=/dev/zero of=$file bs=1M count=2 ||
+               error "Write $file failed"
+       $LFS project -p $id $file || error "failed to set project for $file"
+       $LFS project -d $file
+       do_facet $SINGLEAGT dd if=$file of=/dev/null bs=1M count=2 ||
+               error "Read $file failed"
+       do_facet $SINGLEAGT $LFS pcc state $file
+       check_lpcc_state $file "readonly"
+       do_facet $SINGLEAGT repquota -Pvs $mntpt
+       do_facet $SINGLEAGT dd if=/dev/zero of=$file bs=1M count=5 ||
+               error "Write $file failed"
+       check_lpcc_state $file "none"
+       do_facet $SINGLEAGT dd if=$file of=/dev/null bs=1M count=5 ||
+               error "Read $file failed"
+       do_facet $SINGLEAGT repquota -Pvs $mntpt
+       do_facet $SINGLEAGT $LFS pcc state $file
+       check_lpcc_state $file "none"
+}
+run_test 39 "Test Project quota on loop PCC device"
+
 test_41() {
        local loopfile="$TMP/$tfile"
        local mntpt="/mnt/pcc.$tdir"