From 7c5d1ec252df0a4ca4d1e4e1acb3ad5bb3467fcf Mon Sep 17 00:00:00 2001 From: Sergey Cheremencev Date: Sun, 18 Dec 2022 01:42:10 +0400 Subject: [PATCH] LU-16415 quota: enforce project quota for root Patch adds an option to enforce project quotas for root. It is disabled by default, to enable set osd-ldiskfs.*.quota_slave.root_prj_enable to 1 at each target where you need this option. Patch also adds sanity-quota_1j to test new feature. Lustre-change: https://review.whamcloud.com/49460 Lustre-commit: f147655c33ea61450105b602c82da900fd1417b5 Signed-off-by: Sergey Cheremencev Change-Id: I978dc8442235149794f85110309f63bc560defdc Reviewed-by: Hongchao Zhang Reviewed-by: Sebastien Buisson Reviewed-by: Andreas Dilger Reviewed-on: https://review.whamcloud.com/c/ex/lustre-release/+/49812 Tested-by: jenkins Tested-by: Maloo --- lustre/include/lustre_quota.h | 1 + lustre/include/obd.h | 7 ++-- lustre/include/uapi/linux/lustre/lustre_idl.h | 2 ++ lustre/mdt/mdt_io.c | 3 ++ lustre/ofd/ofd_io.c | 3 ++ lustre/osc/osc_cache.c | 2 +- lustre/osc/osc_quota.c | 1 + lustre/osd-ldiskfs/osd_io.c | 2 ++ lustre/osd-ldiskfs/osd_quota.c | 2 ++ lustre/osd-zfs/osd_io.c | 2 ++ lustre/osd-zfs/osd_quota.c | 2 ++ lustre/quota/qsd_handler.c | 3 ++ lustre/quota/qsd_internal.h | 3 +- lustre/quota/qsd_lib.c | 30 ++++++++++++++++ lustre/tests/sanity-quota.sh | 50 ++++++++++++++++++++++++++- 15 files changed, 107 insertions(+), 6 deletions(-) diff --git a/lustre/include/lustre_quota.h b/lustre/include/lustre_quota.h index 6e87a17..605c999 100644 --- a/lustre/include/lustre_quota.h +++ b/lustre/include/lustre_quota.h @@ -181,6 +181,7 @@ enum osd_quota_local_flags { QUOTA_FL_OVER_GRPQUOTA = BIT(1), QUOTA_FL_SYNC = BIT(2), QUOTA_FL_OVER_PRJQUOTA = BIT(3), + QUOTA_FL_ROOT_PRJQUOTA = BIT(4), }; struct qsd_instance *qsd_init(const struct lu_env *, char *, struct dt_device *, diff --git a/lustre/include/obd.h b/lustre/include/obd.h index 77e6205..4798197 100644 --- a/lustre/include/obd.h +++ b/lustre/include/obd.h @@ -217,7 +217,10 @@ struct client_obd { unsigned int cl_checksum:1, /* 0 = disabled, 1 = enabled */ cl_checksum_dump:1, /* same */ - cl_ocd_grant_param:1; + cl_ocd_grant_param:1, + cl_root_squash:1, /* if root squash enabled*/ + /* check prj quota for root */ + cl_root_prjquota:1; enum lustre_sec_part cl_sp_me; enum lustre_sec_part cl_sp_to; struct sptlrpc_flavor cl_flvr_mgc; /* fixed flavor of mgc->mgs */ @@ -240,8 +243,6 @@ struct client_obd { struct list_head cl_grant_chain; time64_t cl_grant_shrink_interval; /* seconds */ - int cl_root_squash; /* if root squash enabled*/ - /* A chunk is an optimal size used by osc_extent to determine * the extent size. A chunk is max(PAGE_SIZE, OST block size) */ int cl_chunkbits; diff --git a/lustre/include/uapi/linux/lustre/lustre_idl.h b/lustre/include/uapi/linux/lustre/lustre_idl.h index 0e97dbe..e7c8b77 100644 --- a/lustre/include/uapi/linux/lustre/lustre_idl.h +++ b/lustre/include/uapi/linux/lustre/lustre_idl.h @@ -1127,6 +1127,7 @@ enum obdo_flags { OBD_FL_FLUSH = 0x00200000, /* flush pages on the OST */ OBD_FL_SHORT_IO = 0x00400000, /* short io request */ OBD_FL_ROOT_SQUASH = 0x00800000, /* root squash */ + OBD_FL_ROOT_PRJQUOTA = 0x01000000, /* check prj quota for root */ /* OBD_FL_LOCAL_MASK = 0xF0000000, was local-only flags until 2.10 */ /* @@ -1423,6 +1424,7 @@ struct hsm_state_set { * space for unstable pages; asking * it to sync quickly */ #define OBD_BRW_OVER_PRJQUOTA 0x8000 /* Running out of project quota */ +#define OBD_BRW_ROOT_PRJQUOTA 0x10000 /* check project quota for root */ #define OBD_BRW_RDMA_ONLY 0x20000 /* RPC contains RDMA-only pages*/ #define OBD_BRW_SYS_RESOURCE 0x40000 /* page has CAP_SYS_RESOURCE */ diff --git a/lustre/mdt/mdt_io.c b/lustre/mdt/mdt_io.c index d09b555..1ffa1e6 100644 --- a/lustre/mdt/mdt_io.c +++ b/lustre/mdt/mdt_io.c @@ -826,6 +826,9 @@ int mdt_obd_commitrw(const struct lu_env *env, int cmd, struct obd_export *exp, oa->o_flags = OBD_FL_NO_PRJQUOTA; } + if (lnb[0].lnb_flags & OBD_BRW_ROOT_PRJQUOTA) + oa->o_flags |= OBD_FL_ROOT_PRJQUOTA; + if (root_squash) oa->o_flags |= OBD_FL_ROOT_SQUASH; diff --git a/lustre/ofd/ofd_io.c b/lustre/ofd/ofd_io.c index 1fb4dd5..caf9870 100644 --- a/lustre/ofd/ofd_io.c +++ b/lustre/ofd/ofd_io.c @@ -1527,6 +1527,9 @@ int ofd_commitrw(const struct lu_env *env, int cmd, struct obd_export *exp, oa->o_flags = OBD_FL_NO_PRJQUOTA; } + if (lnb[0].lnb_flags & OBD_BRW_ROOT_PRJQUOTA) + oa->o_flags |= OBD_FL_ROOT_PRJQUOTA; + if (root_squash) oa->o_flags |= OBD_FL_ROOT_SQUASH; diff --git a/lustre/osc/osc_cache.c b/lustre/osc/osc_cache.c index 7aec989..2fa956e 100644 --- a/lustre/osc/osc_cache.c +++ b/lustre/osc/osc_cache.c @@ -2345,7 +2345,7 @@ int osc_queue_async_io(const struct lu_env *env, struct cl_io *io, * we should bypass quota */ if ((!oio->oi_cap_sys_resource || - cli->cl_root_squash) && + cli->cl_root_squash || cli->cl_root_prjquota) && !io->ci_noquota) { struct cl_object *obj; struct cl_attr *attr; diff --git a/lustre/osc/osc_quota.c b/lustre/osc/osc_quota.c index 0f07952..e1606c5 100644 --- a/lustre/osc/osc_quota.c +++ b/lustre/osc/osc_quota.c @@ -109,6 +109,7 @@ int osc_quota_setdq(struct client_obd *cli, __u64 xid, const unsigned int qid[], mutex_lock(&cli->cl_quota_mutex); cli->cl_root_squash = !!(flags & OBD_FL_ROOT_SQUASH); + cli->cl_root_prjquota = !!(flags & OBD_FL_ROOT_PRJQUOTA); /* still mark the quots is running out for the old request, because it * could be processed after the new request at OST, the side effect is * the following request will be processed synchronously, but it will diff --git a/lustre/osd-ldiskfs/osd_io.c b/lustre/osd-ldiskfs/osd_io.c index 104b27f..c9e074c 100644 --- a/lustre/osd-ldiskfs/osd_io.c +++ b/lustre/osd-ldiskfs/osd_io.c @@ -1558,6 +1558,8 @@ out_declare: lnb[0].lnb_flags |= OBD_BRW_OVER_GRPQUOTA; if (local_flags & QUOTA_FL_OVER_PRJQUOTA) lnb[0].lnb_flags |= OBD_BRW_OVER_PRJQUOTA; + if (local_flags & QUOTA_FL_ROOT_PRJQUOTA) + lnb[0].lnb_flags |= OBD_BRW_ROOT_PRJQUOTA; if (rc == 0) rc = osd_trunc_lock(osd_dt_obj(dt), oh, true); diff --git a/lustre/osd-ldiskfs/osd_quota.c b/lustre/osd-ldiskfs/osd_quota.c index a67dc5b..b0dbf41 100644 --- a/lustre/osd-ldiskfs/osd_quota.c +++ b/lustre/osd-ldiskfs/osd_quota.c @@ -692,6 +692,8 @@ int osd_declare_inode_qid(const struct lu_env *env, qid_t uid, qid_t gid, qi->lqi_type = PRJQUOTA; rcp = osd_declare_qid(env, oh, qi, obj, true, local_flags); + if (local_flags && *local_flags & QUOTA_FL_ROOT_PRJQUOTA) + force = th->th_ignore_quota; if (force && (rcp == -EDQUOT || rcp == -EINPROGRESS)) { CDEBUG(D_QUOTA, "forced to ignore quota flags = %#x\n", local_flags ? *local_flags : -1); diff --git a/lustre/osd-zfs/osd_io.c b/lustre/osd-zfs/osd_io.c index eb40f72..cec9127 100644 --- a/lustre/osd-zfs/osd_io.c +++ b/lustre/osd-zfs/osd_io.c @@ -738,6 +738,8 @@ retry: #ifdef ZFS_PROJINHERIT if (local_flags & QUOTA_FL_OVER_PRJQUOTA) lnb[0].lnb_flags |= OBD_BRW_OVER_PRJQUOTA; + if (local_flags & QUOTA_FL_ROOT_PRJQUOTA) + lnb[0].lnb_flags |= OBD_BRW_ROOT_PRJQUOTA; #endif RETURN(rc); diff --git a/lustre/osd-zfs/osd_quota.c b/lustre/osd-zfs/osd_quota.c index 2e75b8c..f947b1d 100644 --- a/lustre/osd-zfs/osd_quota.c +++ b/lustre/osd-zfs/osd_quota.c @@ -575,6 +575,8 @@ int osd_declare_quota(const struct lu_env *env, struct osd_device *osd, qi->lqi_type = PRJQUOTA; rcp = qsd_op_begin(env, qsd, &oh->ot_quota_trans, qi, local_flags); + if (local_flags && *local_flags & QUOTA_FL_ROOT_PRJQUOTA) + force = th->th_ignore_quota; if (force && (rcp == -EDQUOT || rcp == -EINPROGRESS)) rcp = 0; } diff --git a/lustre/quota/qsd_handler.c b/lustre/quota/qsd_handler.c index aa3a700..14743960 100644 --- a/lustre/quota/qsd_handler.c +++ b/lustre/quota/qsd_handler.c @@ -884,6 +884,9 @@ int qsd_op_begin(const struct lu_env *env, struct qsd_instance *qsd, (qsd->qsd_type_array[qi->lqi_type])->qqi_acct_failed) RETURN(0); + if (local_flags && qi->lqi_id.qid_projid && qsd->qsd_root_prj_enable) + *local_flags |= QUOTA_FL_ROOT_PRJQUOTA; + LASSERTF(trans->lqt_id_cnt <= QUOTA_MAX_TRANSIDS, "id_cnt=%d\n", trans->lqt_id_cnt); /* check whether we already allocated a slot for this id */ diff --git a/lustre/quota/qsd_internal.h b/lustre/quota/qsd_internal.h index e69edc5..cd681c9 100644 --- a/lustre/quota/qsd_internal.h +++ b/lustre/quota/qsd_internal.h @@ -113,7 +113,8 @@ struct qsd_instance { qsd_exp_valid:1,/* qsd_exp is now valid */ qsd_stopping:1, /* qsd_instance is stopping */ qsd_updating:1, /* qsd is updating record */ - qsd_exclusive:1; /* upd exclusive with reint */ + qsd_exclusive:1, /* upd exclusive with reint */ + qsd_root_prj_enable:1; }; diff --git a/lustre/quota/qsd_lib.c b/lustre/quota/qsd_lib.c index 6b827cb..e29a5a7 100644 --- a/lustre/quota/qsd_lib.c +++ b/lustre/quota/qsd_lib.c @@ -267,6 +267,34 @@ qsd_timeout_seq_write(struct file *file, const char __user *buffer, } LPROC_SEQ_FOPS(qsd_timeout); +static int qsd_root_prj_enable_seq_show(struct seq_file *m, void *data) +{ + struct qsd_instance *qsd = m->private; + + LASSERT(qsd != NULL); + seq_printf(m, "%d\n", qsd->qsd_root_prj_enable); + return 0; +} + +static ssize_t +qsd_root_prj_enable_seq_write(struct file *file, const char __user *buffer, + size_t count, loff_t *off) +{ + struct seq_file *m = file->private_data; + struct qsd_instance *qsd = m->private; + bool enable; + int rc; + + LASSERT(qsd != NULL); + rc = kstrtobool_from_user(buffer, count, &enable); + if (rc) + return rc; + + qsd->qsd_root_prj_enable = enable; + return count; +} +LPROC_SEQ_FOPS(qsd_root_prj_enable); + static struct lprocfs_vars lprocfs_quota_qsd_vars[] = { { .name = "info", .fops = &qsd_state_fops }, @@ -276,6 +304,8 @@ static struct lprocfs_vars lprocfs_quota_qsd_vars[] = { .fops = &qsd_force_reint_fops }, { .name = "timeout", .fops = &qsd_timeout_fops }, + { .name = "root_prj_enable", + .fops = &qsd_root_prj_enable_fops }, { NULL } }; diff --git a/lustre/tests/sanity-quota.sh b/lustre/tests/sanity-quota.sh index cbfc4d9..5fdd6dc 100755 --- a/lustre/tests/sanity-quota.sh +++ b/lustre/tests/sanity-quota.sh @@ -542,7 +542,7 @@ test_1_check_write() { quota_error $short_qtype $TSTUSR \ "$qtype write failure, but expect success" log "Write out of block quota ..." - # this time maybe cache write, ignore it's failure + # this time maybe cache write, ignore it's failure $RUNAS $DD of=$testfile count=$((limit/2)) seek=$((limit/2)) || true # flush cache, ensure noquota flag is set on client cancel_lru_locks osc @@ -1093,6 +1093,54 @@ test_1h() { } run_test 1h "Block hard limit test using fallocate" +test_1j() { + local limit=20 # MB + local testfile="$DIR/$tdir/$tfile-0" + + (( $OST1_VERSION >= $(version_code 2.14.0.74) )) || + skip "need OST at least 2.14.0.74" + + is_project_quota_supported || + skip "skip project quota unsupported" + + setup_quota_test || error "setup quota failed with $?" + + # enable ost quota + set_ost_qtype $QTYPE || error "enable ost quota failed" + + # test for Project + log "--------------------------------------" + log "Project quota (block hardlimit:$limit mb)" + $LFS setquota -p $TSTPRJID -b 0 -B ${limit}M -i 0 -I 0 $DIR || + error "set project quota failed" + + $LFS setstripe $testfile -c 1 -i 0 || error "setstripe $testfile failed" + change_project -p $TSTPRJID $testfile + + local procf=osd-$ost1_FSTYPE.$FSNAME-OST0000.quota_slave.root_prj_enable + do_facet ost1 $LCTL set_param $procf=1 || + error "enable root quotas for project failed" + stack_trap "do_facet ost1 $LCTL set_param $procf=0" + + runas -u 0 -g 0 $DD of=$testfile count=$limit oflag=direct || true + runas -u 0 -g 0 $DD of=$testfile count=$((limit/2)) seek=$limit oflag=direct && + quota_error "project" $TSTPRJID "root write to project success" + + do_facet ost1 $LCTL set_param $procf=0 || + error "disable root quotas for project failed" + + runas -u 0 -g 0 $DD of=$testfile count=$limit seek=$limit oflag=direct || + quota_error "project" $TSTPRJID "root write to project failed" + + # cleanup + cleanup_quota_test + + used=$(getquota -p $TSTPRJID global curspace) + (( $used == 0 )) || quota_error p $TSTPRJID \ + "project quota isn't released after deletion" +} +run_test 1j "Enable project quota enforcement for root" + # test inode hardlimit test_2() { local testfile="$DIR/$tdir/$tfile-0" -- 1.8.3.1