From 25a70a88c9eb35b7e43347c0d8220e334591d25e Mon Sep 17 00:00:00 2001 From: Sergey Cheremencev Date: Thu, 10 Sep 2020 15:48:01 +0300 Subject: [PATCH] LU-13952 quota: default OST Pool Quotas Patch makes ability to set default quota limits per OST pool. Patch also adds sanity-quota_73. Test-Parameters: testlist=sanity-quota HPE-bug-id: LUS-9133 Change-Id: I9e49def231aeeed4588e5e3fbcd29fdd62a35855 Signed-off-by: Sergey Cheremencev Reviewed-on: https://review.whamcloud.com/39873 Tested-by: jenkins Reviewed-by: Andreas Dilger Tested-by: Maloo Reviewed-by: Shaun Tancheff Reviewed-by: Oleg Drokin --- lustre/include/uapi/linux/lustre/lustre_user.h | 6 ++- lustre/llite/dir.c | 2 + lustre/mdt/mdt_handler.c | 4 ++ lustre/quota/qmt_handler.c | 16 +++--- lustre/quota/qmt_pool.c | 4 +- lustre/tests/sanity-quota.sh | 68 ++++++++++++++++++-------- lustre/utils/lfs.c | 19 ++++--- 7 files changed, 82 insertions(+), 37 deletions(-) diff --git a/lustre/include/uapi/linux/lustre/lustre_user.h b/lustre/include/uapi/linux/lustre/lustre_user.h index 9297f0d..723e47f 100644 --- a/lustre/include/uapi/linux/lustre/lustre_user.h +++ b/lustre/include/uapi/linux/lustre/lustre_user.h @@ -1274,6 +1274,8 @@ static inline __u64 lustre_stoqb(__kernel_size_t space) #define LUSTRE_Q_SETQUOTAPOOL 0x800010 /* set user pool quota */ #define LUSTRE_Q_GETINFOPOOL 0x800011 /* get pool quota info */ #define LUSTRE_Q_SETINFOPOOL 0x800012 /* set pool quota info */ +#define LUSTRE_Q_GETDEFAULT_POOL 0x800013 /* get default pool quota*/ +#define LUSTRE_Q_SETDEFAULT_POOL 0x800014 /* set default pool quota */ /* In the current Lustre implementation, the grace time is either the time * or the timestamp to be used after some quota ID exceeds the soft limt, * 48 bits should be enough, its high 16 bits can be used as quota flags. @@ -1303,7 +1305,9 @@ static inline __u64 lustre_stoqb(__kernel_size_t space) (cmd == LUSTRE_Q_GETQUOTAPOOL || \ cmd == LUSTRE_Q_SETQUOTAPOOL || \ cmd == LUSTRE_Q_SETINFOPOOL || \ - cmd == LUSTRE_Q_GETINFOPOOL) + cmd == LUSTRE_Q_GETINFOPOOL || \ + cmd == LUSTRE_Q_SETDEFAULT_POOL || \ + cmd == LUSTRE_Q_GETDEFAULT_POOL) #define ALLQUOTA 255 /* set all quota */ static inline const char *qtype_name(int qtype) diff --git a/lustre/llite/dir.c b/lustre/llite/dir.c index 93c2fc0..b77a8af 100644 --- a/lustre/llite/dir.c +++ b/lustre/llite/dir.c @@ -1141,12 +1141,14 @@ int quotactl_ioctl(struct ll_sb_info *sbi, struct if_quotactl *qctl) case LUSTRE_Q_SETDEFAULT: case LUSTRE_Q_SETQUOTAPOOL: case LUSTRE_Q_SETINFOPOOL: + case LUSTRE_Q_SETDEFAULT_POOL: if (!capable(CAP_SYS_ADMIN)) RETURN(-EPERM); break; case Q_GETQUOTA: case LUSTRE_Q_GETDEFAULT: case LUSTRE_Q_GETQUOTAPOOL: + case LUSTRE_Q_GETDEFAULT_POOL: if (check_owner(type, id) && (!capable(CAP_SYS_ADMIN))) RETURN(-EPERM); diff --git a/lustre/mdt/mdt_handler.c b/lustre/mdt/mdt_handler.c index 29a4f51..aae5616 100644 --- a/lustre/mdt/mdt_handler.c +++ b/lustre/mdt/mdt_handler.c @@ -2937,6 +2937,7 @@ static int mdt_quotactl(struct tgt_session_info *tsi) case LUSTRE_Q_SETDEFAULT: case LUSTRE_Q_SETQUOTAPOOL: case LUSTRE_Q_SETINFOPOOL: + case LUSTRE_Q_SETDEFAULT_POOL: if (!nodemap_can_setquota(nodemap)) GOTO(out_nodemap, rc = -EPERM); /* fallthrough */ @@ -2945,6 +2946,7 @@ static int mdt_quotactl(struct tgt_session_info *tsi) case LUSTRE_Q_GETDEFAULT: case LUSTRE_Q_GETQUOTAPOOL: case LUSTRE_Q_GETINFOPOOL: + case LUSTRE_Q_GETDEFAULT_POOL: if (qmt == NULL) GOTO(out_nodemap, rc = -EOPNOTSUPP); /* slave quotactl */ @@ -3003,6 +3005,8 @@ static int mdt_quotactl(struct tgt_session_info *tsi) case LUSTRE_Q_GETQUOTAPOOL: case LUSTRE_Q_SETINFOPOOL: case LUSTRE_Q_GETINFOPOOL: + case LUSTRE_Q_SETDEFAULT_POOL: + case LUSTRE_Q_GETDEFAULT_POOL: /* forward quotactl request to QMT */ rc = qmt_hdls.qmth_quotactl(tsi->tsi_env, qmt, oqctl); break; diff --git a/lustre/quota/qmt_handler.c b/lustre/quota/qmt_handler.c index 3b7b6cf..5e7f2f5 100644 --- a/lustre/quota/qmt_handler.c +++ b/lustre/quota/qmt_handler.c @@ -156,7 +156,7 @@ int qmt_set_with_lqe(const struct lu_env *env, struct qmt_device *qmt, __u64 ver; bool dirtied = false; int rc = 0; - int need_id_notify = 0; + bool need_id_notify = false; ENTRY; /* need to write back to global quota file? */ @@ -206,7 +206,7 @@ int qmt_set_with_lqe(const struct lu_env *env, struct qmt_device *qmt, quota_set: /* recompute qunit in case it was never initialized */ if (qmt_revalidate(env, lqe)) - need_id_notify = 1; + need_id_notify = true; /* clear grace time */ if (lqe->lqe_softlimit == 0 || @@ -252,8 +252,8 @@ quota_set: /* compute new qunit value now that we have modified the quota * settings or clear/set edquot flag if needed */ - if (qmt_adjust_qunit(env, lqe) || qmt_adjust_edquot(lqe, now)) - need_id_notify |= 1; + need_id_notify |= qmt_adjust_qunit(env, lqe); + need_id_notify |= qmt_adjust_edquot(lqe, now); } EXIT; out: @@ -272,14 +272,14 @@ out_nolock: LQUOTA_DEBUG(lqe, "notify all lqe with default quota"); iter_data.qeid_env = env; iter_data.qeid_qmt = qmt; - cfs_hash_for_each_safe(lqe->lqe_site->lqs_hash, + cfs_hash_for_each(lqe->lqe_site->lqs_hash, qmt_entry_iter_cb, &iter_data); /* Always notify slaves with default values. Don't * care about overhead as will be sent only not changed * values(see qmt_id_lock_cb for details).*/ - need_id_notify = 1; + need_id_notify = true; } - if (need_id_notify && !is_updated) + if (need_id_notify) qmt_set_id_notify(env, qmt, lqe); } @@ -403,6 +403,7 @@ static int qmt_quotactl(const struct lu_env *env, struct lu_device *ld, break; case LUSTRE_Q_GETDEFAULT: + case LUSTRE_Q_GETDEFAULT_POOL: is_default = true; /* fallthrough */ @@ -437,6 +438,7 @@ static int qmt_quotactl(const struct lu_env *env, struct lu_device *ld, break; case LUSTRE_Q_SETDEFAULT: + case LUSTRE_Q_SETDEFAULT_POOL: is_default = true; /* fallthrough */ diff --git a/lustre/quota/qmt_pool.c b/lustre/quota/qmt_pool.c index 87207c2..0806d75 100644 --- a/lustre/quota/qmt_pool.c +++ b/lustre/quota/qmt_pool.c @@ -392,8 +392,8 @@ struct qmt_pool_info *qmt_pool_lookup(const struct lu_env *env, RETURN(ERR_PTR(-ENOENT)); } - CDEBUG(D_QUOTA, "type %d name %p index %d\n", - rtype, pool_name, idx); + CDEBUG(D_QUOTA, "type %d name %s index %d\n", + rtype, pool_name ?: "", idx); /* Now just find a pool with correct type in a list. Further we need * to go through the list and find a pool that includes requested OST * or MDT. Possibly this would return a list of pools that includes diff --git a/lustre/tests/sanity-quota.sh b/lustre/tests/sanity-quota.sh index 773c04c..b110d54 100755 --- a/lustre/tests/sanity-quota.sh +++ b/lustre/tests/sanity-quota.sh @@ -3906,7 +3906,7 @@ test_default_quota() { skip "Not supported before 2.11.51." local qtype=$1 - local qpool=$2 + local qres_type=$2 local qid=$TSTUSR local qprjid=$TSTPRJID local qdtype="-U" @@ -3914,6 +3914,7 @@ test_default_quota() { local qh="-B" local LIMIT=20480 #20M disk space local TESTFILE="$DIR/$tdir/$tfile-0" + local $qpool_cmd [ $qtype == "-p" ] && ! is_project_quota_supported && echo "Project quota is not supported" && return 0 @@ -3925,14 +3926,20 @@ test_default_quota() { qid=$qprjid } - [ $qpool == "meta" ] && { + [ $qres_type == "meta" ] && { LIMIT=10240 #10K inodes qs="-i" qh="-I" } + [ ! -z "$3" ] && { + qpool_cmd="--pool $3" + # pool quotas don't work properly without global limit + $LFS setquota $qtype $qid -B1T -b1T $DIR || + error "set global limit failed" + } setup_quota_test || error "setup quota failed with $?" - trap cleanup_quota_test EXIT + stack_trap cleanup_quota_test EXIT quota_init @@ -3941,19 +3948,19 @@ test_default_quota() { set_ost_qtype $QTYPE || error "enable ost quota failed" log "set to use default quota" - $LFS setquota $qtype $qid -d $DIR || + $LFS setquota $qtype $qid -d $qpool_cmd $DIR || error "set $qid to use default quota failed" log "set default quota" - $LFS setquota $qdtype $qs ${LIMIT} $qh ${LIMIT} $DIR || + $LFS setquota $qdtype $qpool_cmd $qs ${LIMIT} $qh ${LIMIT} $DIR || error "set $qid default quota failed" log "get default quota" $LFS quota $qdtype $DIR || error "get default quota failed" - if [ $qpool == "data" ]; then - local SLIMIT=$($LFS quota $qdtype $DIR | grep "$MOUNT" | \ - awk '{print $2}') + if [ $qres_type == "data" ]; then + local SLIMIT=$($LFS quota $qpool_cmd $qdtype $DIR | \ + grep "$MOUNT" | awk '{print $2}') [ $SLIMIT -eq $LIMIT ] || error "the returned default quota is wrong" else @@ -3967,13 +3974,14 @@ test_default_quota() { local USED=$(getquota $qtype $qid global curspace) [ $USED -ne 0 ] && error "Used space for $qid isn't 0." - $LFS setstripe $TESTFILE -c 1 || error "setstripe $TESTFILE failed" + $LFS setstripe $TESTFILE -c 1 $qpool_cmd || + error "setstripe $TESTFILE failed" chown $TSTUSR.$TSTUSR $TESTFILE || error "chown $TESTFILE failed" [ $qtype == "-p" ] && change_project -sp $TSTPRJID $DIR/$tdir log "Test not out of quota" - if [ $qpool == "data" ]; then + if [ $qres_type == "data" ]; then $RUNAS $DD of=$TESTFILE count=$((LIMIT/2 >> 10)) oflag=sync || quota_error $qtype $qid "write failed, expect succeed" else @@ -3988,7 +3996,7 @@ test_default_quota() { cancel_lru_locks osc cancel_lru_locks mdc sync; sync_all_data || true - if [ $qpool == "data" ]; then + if [ $qres_type == "data" ]; then $RUNAS $DD of=$TESTFILE count=$((LIMIT*2 >> 10)) oflag=sync && quota_error $qtype $qid "write succeed, expect EDQUOT" else @@ -3998,19 +4006,24 @@ test_default_quota() { unlinkmany $TESTFILE $((LIMIT*2)) fi + rm -f $TESTFILE + $LFS setstripe $TESTFILE -c 1 $qpool_cmd || + error "setstripe $TESTFILE failed" + chown $TSTUSR.$TSTUSR $TESTFILE || error "chown $TESTFILE failed" + log "Increase default quota" # LU-4505: sleep 5 seconds to enable quota acquire sleep 5 # increase default quota - $LFS setquota $qdtype $qs $((LIMIT*3)) $qh $((LIMIT*3)) $DIR || - error "set default quota failed" + $LFS setquota $qdtype $qpool_cmd $qs $((LIMIT*3)) \ + $qh $((LIMIT*3)) $DIR || error "set default quota failed" cancel_lru_locks osc cancel_lru_locks mdc sync; sync_all_data || true - if [ $qpool == "data" ]; then + if [ $qres_type == "data" ]; then $RUNAS $DD of=$TESTFILE count=$((LIMIT*2 >> 10)) oflag=sync || quota_error $qtype $qid "write failed, expect succeed" else @@ -4021,13 +4034,13 @@ test_default_quota() { fi log "Set quota to override default quota" - $LFS setquota $qtype $qid $qs ${LIMIT} $qh ${LIMIT} $DIR || + $LFS setquota $qtype $qid $qpool_cmd $qs ${LIMIT} $qh ${LIMIT} $DIR || error "set $qid quota failed" cancel_lru_locks osc cancel_lru_locks mdc sync; sync_all_data || true - if [ $qpool == "data" ]; then + if [ $qres_type == "data" ]; then $RUNAS $DD of=$TESTFILE count=$((LIMIT*2 >> 10)) oflag=sync && quota_error $qtype $qid "write succeed, expect EQUOT" else @@ -4042,13 +4055,13 @@ test_default_quota() { # LU-4505: sleep 5 seconds to enable quota acquire sleep 5 - $LFS setquota $qtype $qid -d $DIR || + $LFS setquota $qtype $qid -d $qpool_cmd $DIR || error "set $qid to use default quota failed" cancel_lru_locks osc cancel_lru_locks mdc sync; sync_all_data || true - if [ $qpool == "data" ]; then + if [ $qres_type == "data" ]; then $RUNAS $DD of=$TESTFILE count=$((LIMIT*2 >> 10)) oflag=sync || quota_error $qtype $qid "write failed, expect succeed" else @@ -4062,9 +4075,10 @@ test_default_quota() { rm -f $TESTFILE wait_delete_completed || error "wait_delete_completed failed" sync_all_data || true - $LFS setquota $qdtype -b 0 -B 0 -i 0 -I 0 $DIR || + + $LFS setquota $qdtype $qpool_cmd $qs 0 $qh 0 $DIR || error "reset default quota failed" - $LFS setquota $qtype $qid -b 0 -B 0 -i 0 -I 0 $DIR || + $LFS setquota $qtype $qid $qpool_cmd $qs 0 $qh 0 $DIR || error "reset quota failed" cleanup_quota_test @@ -4833,6 +4847,20 @@ test_72() } run_test 72 "lfs quota --pool prints only pool's OSTs" +test_73() +{ + local qpool="qpool1" + + mds_supports_qp + + pool_add $qpool || error "pool_add failed" + pool_add_targets $qpool 0 $((OSTCOUNT - 1)) || + error "pool_add_targets failed" + + test_default_quota "-u" "data" "qpool1" +} +run_test 73 "default limits at OST Pool Quotas" + quota_fini() { do_nodes $(comma_list $(nodes_list)) "lctl set_param debug=-quota" diff --git a/lustre/utils/lfs.c b/lustre/utils/lfs.c index 77aabd2..9ce1585 100644 --- a/lustre/utils/lfs.c +++ b/lustre/utils/lfs.c @@ -7255,8 +7255,6 @@ quota_type: case LFS_POOL_OPT: if (lfs_verify_poolarg(optarg)) return -1; - fprintf(stdout, - "Trying to set grace for pool %s\n", optarg); strncpy(qctl->qc_poolname, optarg, LOV_MAXPOOLNAME); qctl->qc_cmd = LUSTRE_Q_SETINFOPOOL; break; @@ -7457,10 +7455,10 @@ quota_type_def: rc = -1; goto out; } - fprintf(stdout, - "Trying to set quota for pool %s\n", optarg); strncpy(qctl->qc_poolname, optarg, LOV_MAXPOOLNAME); - qctl->qc_cmd = LUSTRE_Q_SETQUOTAPOOL; + qctl->qc_cmd = qctl->qc_cmd == LUSTRE_Q_SETDEFAULT ? + LUSTRE_Q_SETDEFAULT_POOL : + LUSTRE_Q_SETQUOTAPOOL; break; default: fprintf(stderr, @@ -7521,6 +7519,9 @@ quota_type_def: dqb->dqb_itime = 0; dqb->dqb_btime = 0; dqb->dqb_valid |= QIF_LIMITS | QIF_TIMES; + /* do not set inode limits for Pool Quotas */ + if (qctl->qc_cmd == LUSTRE_Q_SETDEFAULT_POOL) + dqb->dqb_valid ^= QIF_ILIMITS | QIF_ITIME; } else if ((!(limit_mask & BHLIMIT) ^ !(limit_mask & BSLIMIT)) || (!(limit_mask & IHLIMIT) ^ !(limit_mask & ISLIMIT))) { /* sigh, we can't just set blimits/ilimits */ @@ -7675,7 +7676,8 @@ static void print_quota(char *mnt, struct if_quotactl *qctl, int type, if (qctl->qc_cmd == LUSTRE_Q_GETQUOTA || qctl->qc_cmd == Q_GETOQUOTA || qctl->qc_cmd == LUSTRE_Q_GETQUOTAPOOL || - qctl->qc_cmd == LUSTRE_Q_GETDEFAULT) { + qctl->qc_cmd == LUSTRE_Q_GETDEFAULT || + qctl->qc_cmd == LUSTRE_Q_GETDEFAULT_POOL) { int bover = 0, iover = 0; struct obd_dqblk *dqb = &qctl->qc_dqblk; char numbuf[3][STRBUF_LEN]; @@ -7943,6 +7945,7 @@ static int get_print_quota(char *mnt, char *name, struct if_quotactl *qctl, if ((qctl->qc_cmd == LUSTRE_Q_GETQUOTA || qctl->qc_cmd == LUSTRE_Q_GETQUOTAPOOL || + qctl->qc_cmd == LUSTRE_Q_GETDEFAULT_POOL || qctl->qc_cmd == LUSTRE_Q_GETDEFAULT) && !quiet) print_quota_title(name, qctl, human_readable, show_default); @@ -8323,7 +8326,9 @@ quota_type: } } else { qctl->qc_valid = QC_GENERAL; - qctl->qc_cmd = LUSTRE_Q_GETDEFAULT; + qctl->qc_cmd = qctl->qc_cmd == LUSTRE_Q_GETQUOTAPOOL ? + LUSTRE_Q_GETDEFAULT_POOL : + LUSTRE_Q_GETDEFAULT; qctl->qc_id = 0; } -- 1.8.3.1