LU-12358 pcc: add project quota support on PCC backend

Current PCC can enforce a quota limitation of the capacity usage
for each user and group to provide cache isolation. An admin
can specify the quota enforcement on the local PCC file system.

Users can perform PCC-cached I/O on files until they receive a
return value -ENOSPC of -EDQUOT, which means that they hit the
quota limit or that there is no free capacity left on the local
PCC backend fs during I/O or the attach process. At this time,
I/O will fall back to the normal I/O path.

This patch adds project quota on the PCC backend file system
along with user/group quota.

With this feature, it can have multiple PCC backends on a single
client with different caching rules, so we can define upfront
how much of the client FS can be used for each cache.

Test-Parameters: clientcount=3 testlist=sanity-pcc,sanity-pcc,sanity-pcc
Signed-off-by: Qian Yingjin <qian@ddn.com>
Change-Id: Ib93da953d4a3a7091f62094f8175bde91e819895
Reviewed-on: https://review.whamcloud.com/41928
Tested-by: jenkins <devops@whamcloud.com>
Tested-by: Maloo <maloo@whamcloud.com>
Reviewed-by: Andreas Dilger <adilger@whamcloud.com>
Reviewed-by: Wang Shilong <wshilong@whamcloud.com>

diff --git a/lustre/llite/pcc.c b/lustre/llite/pcc.c
index 9f815b6..289bff6 100644 (file)
--- a/lustre/llite/pcc.c
+++ b/lustre/llite/pcc.c
@@ -570,6 +570,7 @@ pcc_parse_value_pair(struct pcc_cmd *cmd, char *buffer)
 {
        char *key, *val;
        unsigned long id;
+       bool enable;
        int rc;
 
        val = buffer;
@@ -593,49 +594,69 @@ pcc_parse_value_pair(struct pcc_cmd *cmd, char *buffer)
                        return -EINVAL;
                cmd->u.pccc_add.pccc_roid = id;
        } else if (strcmp(key, "auto_attach") == 0) {
-               rc = kstrtoul(val, 10, &id);
+               rc = kstrtobool(val, &enable);
                if (rc)
                        return rc;
-               if (id == 0)
+               if (enable)
+                       cmd->u.pccc_add.pccc_flags |= PCC_DATASET_AUTO_ATTACH;
+               else
                        cmd->u.pccc_add.pccc_flags &= ~PCC_DATASET_AUTO_ATTACH;
        } else if (strcmp(key, "open_attach") == 0) {
-               rc = kstrtoul(val, 10, &id);
+               rc = kstrtobool(val, &enable);
                if (rc)
                        return rc;
-               if (id == 0)
+               if (enable)
+                       cmd->u.pccc_add.pccc_flags |= PCC_DATASET_OPEN_ATTACH;
+               else
                        cmd->u.pccc_add.pccc_flags &= ~PCC_DATASET_OPEN_ATTACH;
        } else if (strcmp(key, "io_attach") == 0) {
-               rc = kstrtoul(val, 10, &id);
+               rc = kstrtobool(val, &enable);
                if (rc)
                        return rc;
-               if (id == 0)
+               if (enable)
+                       cmd->u.pccc_add.pccc_flags |= PCC_DATASET_IO_ATTACH;
+               else
                        cmd->u.pccc_add.pccc_flags &= ~PCC_DATASET_IO_ATTACH;
        } else if (strcmp(key, "stat_attach") == 0) {
-               rc = kstrtoul(val, 10, &id);
+               rc = kstrtobool(val, &enable);
                if (rc)
                        return rc;
-               if (id == 0)
+               if (enable)
+                       cmd->u.pccc_add.pccc_flags |= PCC_DATASET_STAT_ATTACH;
+               else
                        cmd->u.pccc_add.pccc_flags &= ~PCC_DATASET_STAT_ATTACH;
-       } else if (strcmp(key, "rwpcc") == 0) {
-               rc = kstrtoul(val, 10, &id);
+       } else if (strcmp(key, "rwpcc") == 0 || strcmp(key, "pccrw") == 0) {
+               rc = kstrtobool(val, &enable);
                if (rc)
                        return rc;
-               if (id > 0)
+               if (enable)
                        cmd->u.pccc_add.pccc_flags |= PCC_DATASET_RWPCC;
-       } else if (strcmp(key, "ropcc") == 0) {
-               rc = kstrtoul(val, 10, &id);
+               else
+                       cmd->u.pccc_add.pccc_flags &= ~PCC_DATASET_RWPCC;
+       } else if (strcmp(key, "ropcc") == 0 || strcmp(key, "pccro") == 0) {
+               rc = kstrtobool(val, &enable);
                if (rc)
                        return rc;
-               if (id > 0)
+               if (enable)
                        cmd->u.pccc_add.pccc_flags |= PCC_DATASET_ROPCC;
+               else
+                       cmd->u.pccc_add.pccc_flags &= ~PCC_DATASET_ROPCC;
        } else if (strcmp(key, "mmap_conv") == 0) {
-               rc = kstrtoul(val, 10, &id);
+               rc = kstrtobool(val, &enable);
                if (rc)
                        return rc;
-               if (id > 0)
+               if (enable)
                        cmd->u.pccc_add.pccc_flags |= PCC_DATASET_MMAP_CONV;
-               else if (id == 0)
+               else
                        cmd->u.pccc_add.pccc_flags &= ~PCC_DATASET_MMAP_CONV;
+       } else if (strcmp(key, "proj_quota") == 0) {
+               rc = kstrtobool(val, &enable);
+               if (rc)
+                       return rc;
+               if (enable)
+                       cmd->u.pccc_add.pccc_flags |= PCC_DATASET_PROJ_QUOTA;
+               else
+                       cmd->u.pccc_add.pccc_flags &= ~PCC_DATASET_PROJ_QUOTA;
        } else {
                return -EINVAL;
        }
@@ -652,9 +673,10 @@ pcc_parse_value_pairs(struct pcc_cmd *cmd, char *buffer)
 
        switch (cmd->pccc_cmd) {
        case PCC_ADD_DATASET:
-               /* Enable auto attach and mmap pagecache convert by default */
+               /* Enable these features by default */
                cmd->u.pccc_add.pccc_flags |= PCC_DATASET_AUTO_ATTACH |
-                                             PCC_DATASET_MMAP_CONV;
+                                             PCC_DATASET_MMAP_CONV |
+                                             PCC_DATASET_PROJ_QUOTA;
                break;
        case PCC_DEL_DATASET:
        case PCC_CLEAR_ALL:
@@ -2890,7 +2912,6 @@ out:
 
 /*
  * Reset uid, gid or size for the PCC copy masked by @valid.
- * TODO: Set the project ID for PCC copy.
  */
 int pcc_inode_reset_iattr(struct dentry *dentry, unsigned int valid,
                          kuid_t uid, kgid_t gid, loff_t size)
@@ -2913,6 +2934,67 @@ int pcc_inode_reset_iattr(struct dentry *dentry, unsigned int valid,
        RETURN(rc);
 }
 
+/* Set the project ID for PCC copy.*/
+int pcc_file_reset_projid(struct pcc_dataset *dataset, struct file *file,
+                          __u32 projid)
+{
+       struct fsxattr fsx = { .fsx_projid = projid };
+       mm_segment_t old_fs;
+       int rc;
+
+       ENTRY;
+
+       if (!(dataset->pccd_flags & PCC_DATASET_PROJ_QUOTA))
+               RETURN(0);
+
+       /* project quota not supported on backing filesystem */
+       if (!file->f_op->unlocked_ioctl) {
+               CWARN("%s: cache fs project quota absent, disabling: rc = %d\n",
+                     dataset->pccd_pathname, -ENOIOCTLCMD);
+               dataset->pccd_flags &= ~PCC_DATASET_PROJ_QUOTA;
+               RETURN(0);
+       }
+
+       old_fs = get_fs();
+       set_fs(KERNEL_DS);
+       rc = file->f_op->unlocked_ioctl(file, FS_IOC_FSSETXATTR,
+                                       (unsigned long)&fsx);
+       set_fs(old_fs);
+       if (rc == -EOPNOTSUPP || rc == -ENOTTY) {
+               CWARN("%s: cache fs project quota off, disabling: rc = %d\n",
+                     dataset->pccd_pathname, rc);
+               dataset->pccd_flags &= ~PCC_DATASET_PROJ_QUOTA;
+               RETURN(0);
+       }
+
+       RETURN(rc);
+}
+
+int pcc_inode_reset_projid(struct pcc_dataset *dataset, struct dentry *dentry,
+                          __u32 projid)
+{
+       struct path path;
+       struct file *file;
+       int rc;
+
+       ENTRY;
+
+       if (!(dataset->pccd_flags & PCC_DATASET_PROJ_QUOTA))
+               RETURN(0);
+
+       path.mnt = dataset->pccd_path.mnt;
+       path.dentry = dentry;
+       file = dentry_open(&path, O_WRONLY | O_LARGEFILE, current_cred());
+       if (IS_ERR_OR_NULL(file)) {
+               rc = file == NULL ? -EINVAL : PTR_ERR(file);
+               RETURN(rc);
+       }
+
+       rc = pcc_file_reset_projid(dataset, file, projid);
+       fput(file);
+       RETURN(rc);
+}
+
 int pcc_inode_create(struct super_block *sb, struct pcc_dataset *dataset,
                     struct lu_fid *fid, struct dentry **pcc_dentry)
 {
@@ -2955,6 +3037,11 @@ int pcc_inode_create_fini(struct inode *inode, struct pcc_create_attach *pca)
        if (rc)
                GOTO(out_put, rc);
 
+       rc = pcc_inode_reset_projid(pca->pca_dataset, pcc_dentry,
+                                   ll_i2info(inode)->lli_projid);
+       if (rc)
+               GOTO(out_put, rc);
+
        pcc_inode_attach_set(super, pca->pca_dataset, ll_i2info(inode),
                             pcci, pcc_dentry, LU_PCC_READWRITE);
 
@@ -3124,6 +3211,11 @@ static int pcc_attach_data_archive(struct file *file, struct inode *inode,
        if (rc)
                GOTO(out_fput, rc);
 
+       rc = pcc_file_reset_projid(dataset, pcc_filp,
+                                   ll_i2info(inode)->lli_projid);
+       if (rc)
+               GOTO(out_fput, rc);
+
        /*
         * When attach a file at file open() time with direct I/O mode, the
         * data copy from Lustre OSTs to PCC copy in kernel will report

diff --git a/lustre/llite/pcc.h b/lustre/llite/pcc.h
index ae48b4a..6eaa2f0 100644 (file)
--- a/lustre/llite/pcc.h
+++ b/lustre/llite/pcc.h
@@ -132,6 +132,8 @@ enum pcc_dataset_flags {
        PCC_DATASET_PCC_ALL     = PCC_DATASET_RWPCC | PCC_DATASET_ROPCC,
        /* Move pagecache from mapping of PCC copy to Lustre file for mmap */
        PCC_DATASET_MMAP_CONV   = 0x40,
+       /* Set the project ID for the PCC copy */
+       PCC_DATASET_PROJ_QUOTA  = 0x80,
 };
 
 struct pcc_dataset {

diff --git a/lustre/tests/sanity-pcc.sh b/lustre/tests/sanity-pcc.sh
index 5e7934f..b5489d5 100644 (file)
--- a/lustre/tests/sanity-pcc.sh
+++ b/lustre/tests/sanity-pcc.sh
@@ -201,7 +201,7 @@ setup_loopdev() {
        local size=${4:-50}
 
        do_facet $facet mkdir -p $mntpt || error "mkdir -p $mntpt failed"
-       stack_trap "do_facet $facet rm -rf $mntpt" EXIT
+       stack_trap "do_facet $facet rmdir $mntpt" EXIT
        do_facet $facet dd if=/dev/zero of=$file bs=1M count=$size
        stack_trap "do_facet $facet rm -f $file" EXIT
        do_facet $facet mount
@@ -215,6 +215,25 @@ setup_loopdev() {
        stack_trap "umount_loopdev $facet $mntpt" EXIT
 }
 
+setup_loopdev_project() {
+       local facet=$1
+       local file=$2
+       local mntpt=$3
+       local size=${4:-50}
+
+       do_facet $facet mkdir -p $mntpt || error "mkdir -p $mntpt failed"
+       stack_trap "do_facet $facet rmdir $mntpt" EXIT
+       do_facet $facet dd if=/dev/zero of=$file bs=1M count=$size
+       stack_trap "do_facet $facet rm -f $file" EXIT
+       do_facet $facet mkfs.ext4 -O project,quota $file ||
+               error "mkfs.ext4 -O project,quota $file failed"
+       do_facet $facet file $file
+       do_facet $facet mount -t ext4 -o loop,prjquota $file $mntpt ||
+               error "mount -o loop,prjquota $file $mntpt failed"
+       stack_trap "umount_loopdev $facet $mntpt" EXIT
+       do_facet $facet mount | grep $mntpt
+}
+
 lpcc_rw_test() {
        local restore="$1"
        local project="$2"
@@ -396,8 +415,7 @@ test_1f() {
        local hsm_root="$mntpt/$tdir"
        local file=$DIR/$tdir/$tfile
 
-       ! is_project_quota_supported &&
-               skip "project quota is not supported"
+       is_project_quota_supported || skip "project quota is not supported"
 
        enable_project_quota
        setup_loopdev $SINGLEAGT $loopfile $mntpt 50
@@ -492,8 +510,7 @@ test_2a() {
        local hsm_root="$mntpt/$tdir"
        local agt_host=$(facet_active_host $SINGLEAGT)
 
-       ! is_project_quota_supported &&
-               skip "project quota is not supported" && return
+       is_project_quota_supported || skip "project quota is not supported"
 
        enable_project_quota
        setup_loopdev $SINGLEAGT $loopfile $mntpt 50
@@ -755,8 +772,7 @@ test_4() {
        local hsm_root="$mntpt/$tdir"
        local excepts="-e 7 -e 8 -e 9"
 
-       ! is_project_quota_supported &&
-               skip "project quota is not supported" && return
+       is_project_quota_supported || skip "project quota is not supported"
 
        enable_project_quota
        setup_loopdev $SINGLEAGT $loopfile $mntpt 50
@@ -1286,8 +1302,7 @@ test_13c() {
        local mntpt="/mnt/pcc.$tdir"
        local hsm_root="$mntpt/$tdir"
 
-       ! is_project_quota_supported &&
-               echo "Skip project quota is not supported" && return 0
+       is_project_quota_supported || skip "project quota is not supported"
 
        enable_project_quota
        setup_loopdev $SINGLEAGT $loopfile $mntpt 50
@@ -2759,8 +2774,7 @@ test_34() {
        local hsm_root="$mntpt/$tdir"
        local file=$DIR/$tfile
 
-       ! is_project_quota_supported &&
-               skip "project quota is not supported"
+       is_project_quota_supported || skip "project quota is not supported"
 
        enable_project_quota
        setup_loopdev $SINGLEAGT $loopfile $mntpt 50
@@ -2949,8 +2963,7 @@ test_38() {
        local dir=$DIR/$tdir
        local file=$dir/$tfile
 
-       ! is_project_quota_supported &&
-               skip "project quota is not supported"
+       is_project_quota_supported || skip "project quota is not supported"
 
        enable_project_quota
        mkdir $dir || error "mkdir $dir failed"
@@ -2974,6 +2987,53 @@ test_38() {
 }
 run_test 38 "Verify LFS pcc state does not trigger prefetch for auto PCC-RO"
 
+test_39() {
+       quotaon --help |& grep -q 'project quotas' ||
+               skip "Not support project quota on local filesystem"
+
+       is_project_quota_supported || skip "project quota is not supported"
+
+       enable_project_quota
+
+       local loopfile="$TMP/$tfile"
+       local mntpt="/mnt/pcc.$tdir"
+       local hsm_root="$mntpt/$tdir"
+       local dir=$DIR/$tdir
+       local file=$dir/$tfile
+       local id=100
+
+       setup_loopdev_project $SINGLEAGT $loopfile $mntpt 50
+       do_facet $SINGLEAGT quotaon -Ppv $mntpt
+       do_facet $SINGLEAGT setquota -P $id 0 4096 0 0 $mntpt ||
+               error "setquota -P $id on $mntpt failed"
+       do_facet $SINGLEAGT repquota -Pvs $mntpt
+
+       do_facet $SINGLEAGT mkdir $hsm_root || error "mkdir $hsm_root failed"
+       setup_pcc_mapping $SINGLEAGT \
+               "projid={$id}\ roid=$HSM_ARCHIVE_NUMBER\ proj_quota=1\ ropcc=1"
+       do_facet $SINGLEAGT $LCTL pcc list $MOUNT
+
+       do_facet $SINGLEAGT mkdir -p $dir || error "mkdir $dir failed"
+       do_facet $SINGLEAGT dd if=/dev/zero of=$file bs=1M count=2 ||
+               error "Write $file failed"
+       $LFS project -p $id $file || error "failed to set project for $file"
+       $LFS project -d $file
+       do_facet $SINGLEAGT dd if=$file of=/dev/null bs=1M count=2 ||
+               error "Read $file failed"
+       do_facet $SINGLEAGT $LFS pcc state $file
+       check_lpcc_state $file "readonly"
+       do_facet $SINGLEAGT repquota -Pvs $mntpt
+       do_facet $SINGLEAGT dd if=/dev/zero of=$file bs=1M count=5 ||
+               error "Write $file failed"
+       check_lpcc_state $file "none"
+       do_facet $SINGLEAGT dd if=$file of=/dev/null bs=1M count=5 ||
+               error "Read $file failed"
+       do_facet $SINGLEAGT repquota -Pvs $mntpt
+       do_facet $SINGLEAGT $LFS pcc state $file
+       check_lpcc_state $file "none"
+}
+run_test 39 "Test Project quota on loop PCC device"
+
 complete $SECONDS
 check_and_cleanup_lustre
 exit_status