LU-12358 pcc: add project quota support on PCC backend

Current PCC can enforce a quota limitation of the capacity usage
for each user and group to provide cache isolation. An
administrator can specify the quota enforcement on the local PCC
file system.

Users can perform PCC-cached I/O on files until they receive a
return value -ENOSPC of -EDQUOT, which means that they hit the
quota limit or that there is no free capacity left on the local
PCC backend fs during I/O or the attach process. At this time,
I/O will fall back to the normal I/O path.

This patch adds projecct quota on the PCC backend file system
along with user/group quota.

With this feature, it can have multiple PCC backends on a single
client with different caching rules, so we can define upfront how
much of the client FS can be used for each cache.

Signed-off-by: Qian Yingjin <qian@ddn.com>
Change-Id: Ib93da953d4a3a7091f62094f8175bde91e819895

diff --git a/lustre/llite/pcc.c b/lustre/llite/pcc.c
index 521e564..561b238 100644 (file)
--- a/lustre/llite/pcc.c
+++ b/lustre/llite/pcc.c
@@ -628,6 +628,12 @@ pcc_parse_value_pair(struct pcc_cmd *cmd, char *buffer)
                        return rc;
                if (id > 0)
                        cmd->u.pccc_add.pccc_flags |= PCC_DATASET_ROPCC;
+       } else if (strcmp(key, "proj_quota") == 0) {
+               rc = kstrtoul(val, 10, &id);
+               if (rc)
+                       return rc;
+               if (id > 0)
+                       cmd->u.pccc_add.pccc_flags |= PCC_DATASET_PROJ_QUOTA;
        } else {
                return -EINVAL;
        }
@@ -2563,7 +2569,6 @@ out:
 
 /*
  * Reset uid, gid or size for the PCC copy masked by @valid.
- * TODO: Set the project ID for PCC copy.
  */
 int pcc_inode_reset_iattr(struct dentry *dentry, unsigned int valid,
                          kuid_t uid, kgid_t gid, loff_t size)
@@ -2586,6 +2591,58 @@ int pcc_inode_reset_iattr(struct dentry *dentry, unsigned int valid,
        RETURN(rc);
 }
 
+/* Set the project ID for PCC copy.*/
+int pcc_file_reset_projid(struct pcc_dataset *dataset, struct file *file,
+                          __u32 projid)
+{
+       struct fsxattr fsx;
+       mm_segment_t old_fs;
+       int rc;
+
+       ENTRY;
+
+       if (!(dataset->pccd_flags & PCC_DATASET_PROJ_QUOTA))
+               RETURN(0);
+
+       if (!file->f_op->unlocked_ioctl)
+               RETURN(-ENOIOCTLCMD);
+
+       memset(&fsx, 0, sizeof(struct fsxattr));
+       fsx.fsx_projid = projid;
+       old_fs = get_fs();
+       set_fs(get_ds());
+       rc = file->f_op->unlocked_ioctl(file, FS_IOC_FSSETXATTR,
+                                       (unsigned long)&fsx);
+       set_fs(old_fs);
+
+       RETURN(rc);
+}
+
+int pcc_inode_reset_projid(struct pcc_dataset *dataset, struct dentry *dentry,
+                          __u32 projid)
+{
+       struct path path;
+       struct file *file;
+       int rc;
+
+       ENTRY;
+
+       if (!(dataset->pccd_flags & PCC_DATASET_PROJ_QUOTA))
+               RETURN(0);
+
+       path.mnt = dataset->pccd_path.mnt;
+       path.dentry = dentry;
+       file = dentry_open(&path, O_WRONLY | O_LARGEFILE, current_cred());
+       if (IS_ERR_OR_NULL(file)) {
+               rc = file == NULL ? -EINVAL : PTR_ERR(file);
+               RETURN(rc);
+       }
+
+       rc = pcc_file_reset_projid(dataset, file, projid);
+       fput(file);
+       RETURN(rc);
+}
+
 int pcc_inode_create(struct super_block *sb, struct pcc_dataset *dataset,
                     struct lu_fid *fid, struct dentry **pcc_dentry)
 {
@@ -2628,6 +2685,11 @@ int pcc_inode_create_fini(struct inode *inode, struct pcc_create_attach *pca)
        if (rc)
                GOTO(out_put, rc);
 
+       rc = pcc_inode_reset_projid(pca->pca_dataset, pcc_dentry,
+                                   ll_i2info(inode)->lli_projid);
+       if (rc)
+               GOTO(out_put, rc);
+
        pcc_inode_attach_set(super, pca->pca_dataset, ll_i2info(inode),
                             pcci, pcc_dentry, LU_PCC_READWRITE);
 
@@ -2796,6 +2858,11 @@ static int pcc_attach_data_archive(struct file *file, struct inode *inode,
        if (rc)
                GOTO(out_fput, rc);
 
+       rc = pcc_file_reset_projid(dataset, pcc_filp,
+                                   ll_i2info(inode)->lli_projid);
+       if (rc)
+               GOTO(out_fput, rc);
+
        ret = pcc_copy_data(file, pcc_filp);
        if (ret < 0)
                GOTO(out_fput, rc = ret);

diff --git a/lustre/llite/pcc.h b/lustre/llite/pcc.h
index 16ca1b1..8ad519d 100644 (file)
--- a/lustre/llite/pcc.h
+++ b/lustre/llite/pcc.h
@@ -130,6 +130,8 @@ enum pcc_dataset_flags {
        PCC_DATASET_ROPCC       = 0x20,
        /* PCC backend provides caching services for both RW-PCC and RO-PCC */
        PCC_DATASET_PCC_ALL     = PCC_DATASET_RWPCC | PCC_DATASET_ROPCC,
+       /* Set the project ID for the PCC copy */
+       PCC_DATASET_PROJ_QUOTA  = 0x40,
 };
 
 struct pcc_dataset {

diff --git a/lustre/tests/sanity-pcc.sh b/lustre/tests/sanity-pcc.sh
index 14affb2..349e763 100644 (file)
--- a/lustre/tests/sanity-pcc.sh
+++ b/lustre/tests/sanity-pcc.sh
@@ -210,6 +210,25 @@ setup_loopdev() {
        stack_trap "umount_loopdev $facet $mntpt" EXIT
 }
 
+setup_loopdev_project() {
+       local facet=$1
+       local file=$2
+       local mntpt=$3
+       local size=${4:-50}
+
+       do_facet $facet mkdir -p $mntpt || error "mkdir -p $mntpt failed"
+       stack_trap "do_facet $facet rm -rf $mntpt" EXIT
+       do_facet $facet dd if=/dev/zero of=$file bs=1M count=$size
+       stack_trap "do_facet $facet rm -f $file" EXIT
+       do_facet $facet mkfs.ext4 -O project,quota $file ||
+               error "mkfs.ext4 -O project,quota $file failed"
+       do_facet $facet file $file
+       do_facet $facet mount -t ext4 -o loop,prjquota $file $mntpt ||
+               error "mount -o loop,prjquota $file $mntpt failed"
+       stack_trap "umount_loopdev $facet $mntpt" EXIT
+       do_facet $facet mount | grep $mntpt
+}
+
 lpcc_rw_test() {
        local restore="$1"
        local project="$2"
@@ -2783,6 +2802,52 @@ test_35() {
 }
 run_test 35 "Auto PCC-RO attach in atomic_open"
 
+test_36() {
+       quotaon --help |& grep -q 'project quotas' ||
+               skip "Not support project quota on local filesystem"
+
+       ! is_project_quota_supported &&
+               skip "project quota is not supported"
+
+       enable_project_quota
+
+       local loopfile="$TMP/$tfile"
+       local mntpt="/mnt/pcc.$tdir"
+       local hsm_root="$mntpt/$tdir"
+       local file=$DIR/$tfile
+       local id=100
+
+       setup_loopdev_project $SINGLEAGT $loopfile $mntpt 50
+       do_facet $SINGLEAGT quotaon -Ppv $mntpt
+       do_facet $SINGLEAGT setquota -P $id 0 4096 0 0 $mntpt ||
+               error "setquota -P $id on $mntpt failed"
+       do_facet $SINGLEAGT repquota -Pvs $mntpt
+
+       do_facet $SINGLEAGT mkdir $hsm_root || error "mkdir $hsm_root failed"
+       setup_pcc_mapping $SINGLEAGT \
+               "projid={$id}\ roid=$HSM_ARCHIVE_NUMBER\ proj_quota=1\ ropcc=1"
+       do_facet $SINGLEAGT $LCTL pcc list $MOUNT
+
+       do_facet $SINGLEAGT dd if=/dev/zero of=$file bs=1M count=2 ||
+               error "Write $file failed"
+       $LFS project -p $id $file || error "failed to set project for $file"
+       $LFS project -d $file
+       do_facet $SINGLEAGT dd if=$file of=/dev/null bs=1M count=2 ||
+               error "Read $file failed"
+       do_facet $SINGLEAGT $LFS pcc state $file
+       check_lpcc_state $file "readonly"
+       do_facet $SINGLEAGT repquota -Pvs $mntpt
+       do_facet $SINGLEAGT dd if=/dev/zero of=$file bs=1M count=5 ||
+               error "Write $file failed"
+       check_lpcc_state $file "none"
+       do_facet $SINGLEAGT dd if=$file of=/dev/null bs=1M count=5 ||
+               error "Read $file failed"
+       do_facet $SINGLEAGT repquota -Pvs $mntpt
+       do_facet $SINGLEAGT $LFS pcc state $file
+       check_lpcc_state $file "none"
+}
+run_test 36 "Test Project quota on loop PCC device"
+
 complete $SECONDS
 check_and_cleanup_lustre
 exit_status