From: Sergey Cheremencev Date: Tue, 7 Jul 2020 20:19:48 +0000 (+0300) Subject: LU-13359 quota: make used for pool correct X-Git-Tag: 2.13.56~115 X-Git-Url: https://git.whamcloud.com/?p=fs%2Flustre-release.git;a=commitdiff_plain;h=6b9f849fd5f49ce68e6102f9c79f52eb11bf949d LU-13359 quota: make used for pool correct Before this patch used space for quota pool was a sum of a space used by user at all OSTs in a system. Now it is fixed and lfs quota --pool takes into account only OSTs form the pool. With option -v it also shows only OSTs from the pool. Change-Id: Idf1c8ed66fca7caec70246ea4182df883bcef23c Signed-off-by: Sergey Cheremencev Reviewed-on: https://review.whamcloud.com/39298 Reviewed-by: Petros Koutoupis Tested-by: jenkins Reviewed-by: Andreas Dilger Tested-by: Maloo Reviewed-by: Oleg Drokin --- diff --git a/lustre/llite/dir.c b/lustre/llite/dir.c index 4518d16..efc1dbd 100644 --- a/lustre/llite/dir.c +++ b/lustre/llite/dir.c @@ -1213,13 +1213,21 @@ static int quotactl_ioctl(struct ll_sb_info *sbi, struct if_quotactl *qctl) if ((cmd == Q_GETQUOTA || cmd == LUSTRE_Q_GETQUOTAPOOL) && !(oqctl->qc_dqblk.dqb_valid & QIF_SPACE) && !oqctl->qc_dqblk.dqb_curspace) { - struct obd_quotactl *oqctl_tmp; - - OBD_ALLOC_PTR(oqctl_tmp); - if (oqctl_tmp == NULL) - GOTO(out, rc = -ENOMEM); - - oqctl_tmp->qc_cmd = Q_GETOQUOTA; + struct obd_quotactl *oqctl_tmp; + int qctl_len = sizeof(*oqctl_tmp) + LOV_MAXPOOLNAME + 1; + + OBD_ALLOC(oqctl_tmp, qctl_len); + if (oqctl_tmp == NULL) + GOTO(out, rc = -ENOMEM); + + if (cmd == LUSTRE_Q_GETQUOTAPOOL) { + oqctl_tmp->qc_cmd = LUSTRE_Q_GETQUOTAPOOL; + memcpy(oqctl_tmp->qc_poolname, + qctl->qc_poolname, + LOV_MAXPOOLNAME + 1); + } else { + oqctl_tmp->qc_cmd = Q_GETOQUOTA; + } oqctl_tmp->qc_id = oqctl->qc_id; oqctl_tmp->qc_type = oqctl->qc_type; @@ -1232,21 +1240,22 @@ static int quotactl_ioctl(struct ll_sb_info *sbi, struct if_quotactl *qctl) oqctl->qc_dqblk.dqb_valid |= QIF_SPACE; } - /* collect space & inode usage from MDTs */ - oqctl_tmp->qc_dqblk.dqb_curspace = 0; - oqctl_tmp->qc_dqblk.dqb_curinodes = 0; - rc = obd_quotactl(sbi->ll_md_exp, oqctl_tmp); - if (!rc || rc == -EREMOTEIO) { - oqctl->qc_dqblk.dqb_curspace += - oqctl_tmp->qc_dqblk.dqb_curspace; - oqctl->qc_dqblk.dqb_curinodes = - oqctl_tmp->qc_dqblk.dqb_curinodes; - oqctl->qc_dqblk.dqb_valid |= QIF_INODES; - } else { - oqctl->qc_dqblk.dqb_valid &= ~QIF_SPACE; - } + /* collect space & inode usage from MDTs */ + oqctl_tmp->qc_cmd = Q_GETOQUOTA; + oqctl_tmp->qc_dqblk.dqb_curspace = 0; + oqctl_tmp->qc_dqblk.dqb_curinodes = 0; + rc = obd_quotactl(sbi->ll_md_exp, oqctl_tmp); + if (!rc || rc == -EREMOTEIO) { + oqctl->qc_dqblk.dqb_curspace += + oqctl_tmp->qc_dqblk.dqb_curspace; + oqctl->qc_dqblk.dqb_curinodes = + oqctl_tmp->qc_dqblk.dqb_curinodes; + oqctl->qc_dqblk.dqb_valid |= QIF_INODES; + } else { + oqctl->qc_dqblk.dqb_valid &= ~QIF_SPACE; + } - OBD_FREE_PTR(oqctl_tmp); + OBD_FREE(oqctl_tmp, qctl_len); } out: QCTL_COPY(qctl, oqctl); diff --git a/lustre/lod/lod_pool.c b/lustre/lod/lod_pool.c index 55932d5..b124a7b 100644 --- a/lustre/lod/lod_pool.c +++ b/lustre/lod/lod_pool.c @@ -122,7 +122,7 @@ static int pool_cmpfn(struct rhashtable_compare_arg *arg, const void *obj) return strcmp(pool_name, pool->pool_name); } -static const struct rhashtable_params pools_hash_params = { +const struct rhashtable_params pools_hash_params = { .key_len = 1, /* actually variable */ .key_offset = offsetof(struct pool_desc, pool_name), .head_offset = offsetof(struct pool_desc, pool_hash), @@ -130,6 +130,7 @@ static const struct rhashtable_params pools_hash_params = { .obj_cmpfn = pool_cmpfn, .automatic_shrinking = true, }; +EXPORT_SYMBOL(pools_hash_params); /* * Methods for /proc seq_file iteration of the defined pools. diff --git a/lustre/lov/lov_internal.h b/lustre/lov/lov_internal.h index d7ca81f..1a89d3f 100644 --- a/lustre/lov/lov_internal.h +++ b/lustre/lov/lov_internal.h @@ -388,4 +388,7 @@ static inline void lov_lsm2layout(struct lov_stripe_md *lsm, ol->ol_comp_id = 0; } } + +extern const struct rhashtable_params pools_hash_params; +extern void lov_pool_putref(struct pool_desc *pool); #endif diff --git a/lustre/lov/lov_obd.c b/lustre/lov/lov_obd.c index e1eeceb..317bd2b 100644 --- a/lustre/lov/lov_obd.c +++ b/lustre/lov/lov_obd.c @@ -1262,20 +1262,38 @@ __releases(&md->lsm_lock) } static int lov_quotactl(struct obd_device *obd, struct obd_export *exp, - struct obd_quotactl *oqctl) + struct obd_quotactl *oqctl) { - struct lov_obd *lov = &obd->u.lov; - struct lov_tgt_desc *tgt; - __u64 curspace = 0; - __u64 bhardlimit = 0; - int i, rc = 0; - ENTRY; + struct lov_obd *lov = &obd->u.lov; + struct lov_tgt_desc *tgt; + struct pool_desc *pool = NULL; + __u64 curspace = 0; + __u64 bhardlimit = 0; + int i, rc = 0; + ENTRY; if (oqctl->qc_cmd != Q_GETOQUOTA && - oqctl->qc_cmd != LUSTRE_Q_SETQUOTA) { - CERROR("%s: bad quota opc %x for lov obd\n", - obd->obd_name, oqctl->qc_cmd); - RETURN(-EFAULT); + oqctl->qc_cmd != LUSTRE_Q_SETQUOTA && + oqctl->qc_cmd != LUSTRE_Q_GETQUOTAPOOL) { + rc = -EFAULT; + CERROR("%s: bad quota opc %x for lov obd: rc = %d\n", + obd->obd_name, oqctl->qc_cmd, rc); + RETURN(rc); + } + + if (oqctl->qc_cmd == LUSTRE_Q_GETQUOTAPOOL) { + rcu_read_lock(); + pool = rhashtable_lookup(&lov->lov_pools_hash_body, + oqctl->qc_poolname, + pools_hash_params); + if (pool && !atomic_inc_not_zero(&pool->pool_refcount)) + pool = NULL; + rcu_read_unlock(); + if (!pool) + RETURN(-ENOENT); + /* Set Q_GETOQUOTA back as targets report it's own + * usage and doesn't care about pools */ + oqctl->qc_cmd = Q_GETOQUOTA; } /* for lov tgt */ @@ -1288,16 +1306,21 @@ static int lov_quotactl(struct obd_device *obd, struct obd_export *exp, if (!tgt) continue; - if (!tgt->ltd_active || tgt->ltd_reap) { - if (oqctl->qc_cmd == Q_GETOQUOTA && - lov->lov_tgts[i]->ltd_activate) { + if (pool && + tgt_check_index(tgt->ltd_index, &pool->pool_obds)) + continue; + + if (!tgt->ltd_active || tgt->ltd_reap) { + if (oqctl->qc_cmd == Q_GETOQUOTA && + lov->lov_tgts[i]->ltd_activate) { rc = -ENETDOWN; - CERROR("ost %d is inactive\n", i); - } else { - CDEBUG(D_HA, "ost %d is inactive\n", i); - } - continue; - } + CERROR("%s: ost %d is inactive: rc = %d\n", + obd->obd_name, i, rc); + } else { + CDEBUG(D_HA, "ost %d is inactive\n", i); + } + continue; + } err = obd_quotactl(tgt->ltd_exp, oqctl); if (err) { @@ -1312,6 +1335,8 @@ static int lov_quotactl(struct obd_device *obd, struct obd_export *exp, } } lov_tgts_putref(obd); + if (pool) + lov_pool_putref(pool); if (oqctl->qc_cmd == Q_GETOQUOTA) { oqctl->qc_dqblk.dqb_curspace = curspace; diff --git a/lustre/lov/lov_pool.c b/lustre/lov/lov_pool.c index 78bc92e..ef941ab 100644 --- a/lustre/lov/lov_pool.c +++ b/lustre/lov/lov_pool.c @@ -65,7 +65,7 @@ static int pool_cmpfn(struct rhashtable_compare_arg *arg, const void *obj) return strcmp(pool_name, pool->pool_name); } -static const struct rhashtable_params pools_hash_params = { +const struct rhashtable_params pools_hash_params = { .key_len = 1, /* actually variable */ .key_offset = offsetof(struct pool_desc, pool_name), .head_offset = offsetof(struct pool_desc, pool_hash), @@ -80,7 +80,7 @@ static void lov_pool_getref(struct pool_desc *pool) atomic_inc(&pool->pool_refcount); } -static void lov_pool_putref(struct pool_desc *pool) +void lov_pool_putref(struct pool_desc *pool) { CDEBUG(D_INFO, "pool %p\n", pool); if (atomic_dec_and_test(&pool->pool_refcount)) { diff --git a/lustre/ptlrpc/Makefile.in b/lustre/ptlrpc/Makefile.in index 31e956f..89bf4aa 100644 --- a/lustre/ptlrpc/Makefile.in +++ b/lustre/ptlrpc/Makefile.in @@ -15,7 +15,8 @@ target_objs += $(TARGET)tgt_handler.o $(TARGET)out_handler.o target_objs += $(TARGET)out_lib.o $(TARGET)update_trans.o target_objs += $(TARGET)update_records.o $(TARGET)update_recovery.o target_objs += $(TARGET)tgt_grant.o $(TARGET)tgt_fmd.o -target_objs += $(TARGET)tgt_pool.o + +target_pool_objs := $(TARGET)tgt_pool.o ptlrpc_objs := client.o recover.o connection.o niobuf.o pack_generic.o ptlrpc_objs += events.o ptlrpc_module.o service.o pinger.o @@ -29,7 +30,7 @@ nodemap_objs := nodemap_handler.o nodemap_lproc.o nodemap_range.o nodemap_objs += nodemap_idmap.o nodemap_rbtree.o nodemap_member.o nodemap_objs += nodemap_storage.o -ptlrpc-objs := $(ldlm_objs) $(ptlrpc_objs) $(TARGET)barrier.o +ptlrpc-objs := $(ldlm_objs) $(ptlrpc_objs) $(TARGET)barrier.o $(target_pool_objs) @SERVER_TRUE@ptlrpc-objs += $(target_objs) $(nodemap_objs) @GSS_TRUE@obj-m += gss/ diff --git a/lustre/tests/sanity-quota.sh b/lustre/tests/sanity-quota.sh index 36aebe0..9e440fa 100755 --- a/lustre/tests/sanity-quota.sh +++ b/lustre/tests/sanity-quota.sh @@ -4560,6 +4560,53 @@ test_71b() } run_test 71b "Check SEL with quota pools" +test_72() +{ + local limit=10 # 10M + local global_limit=50 # 50M + local testfile="$DIR/$tdir/$tfile-0" + local qpool="qpool1" + + mds_supports_qp + setup_quota_test || error "setup quota failed with $?" + stack_trap cleanup_quota_test EXIT + + # enable ost quota + set_ost_qtype $QTYPE || error "enable ost quota failed" + + # test for user + log "User quota (block hardlimit:$global_limit MB)" + $LFS setquota -u $TSTUSR -b 0 -B ${global_limit}M -i 0 -I 0 $DIR || + error "set user quota failed" + + pool_add $qpool || error "pool_add failed" + pool_add_targets $qpool 1 1 || error "pool_add_targets failed" + + $LFS setquota -u $TSTUSR -B ${limit}M -o $qpool $DIR || + error "set user quota failed" + + # make sure the system is clean + local used=$(getquota -u $TSTUSR global curspace) + echo "used $used" + [ $used -ne 0 ] && error "Used space($used) for user $TSTUSR isn't 0." + + used=$(getquota -u $TSTUSR global bhardlimit $qpool) + + $LFS setstripe $testfile -c 1 -i 1 || error "setstripe $testfile failed" + chown $TSTUSR.$TSTUSR $testfile || error "chown $testfile failed" + test_1_check_write $testfile "user" $limit + used=$(getquota -u $TSTUSR global bhardlimit $qpool) + echo "used $used" + [ $used -ge $limit ] || error "used($used) is less than limit($limit)" + # check that lfs quota -uv --pool prints only OST that + # was added in a pool + lfs quota -v -u quota_usr --pool $qpool $DIR | grep -v "OST0001" | + grep "OST\|MDT" && error "$qpool consists wrong targets" + + cleanup_quota_test +} +run_test 72 "lfs quota --pool prints only pool's OSTs" + quota_fini() { do_nodes $(comma_list $(nodes_list)) "lctl set_param debug=-quota" diff --git a/lustre/utils/lfs.c b/lustre/utils/lfs.c index 82fa2c5..ed31744 100644 --- a/lustre/utils/lfs.c +++ b/lustre/utils/lfs.c @@ -7324,16 +7324,39 @@ static void print_quota(char *mnt, struct if_quotactl *qctl, int type, } } +static int tgt_name2index(const char *tgtname, unsigned int *idx) +{ + char *dash, *endp; + + /* format is "lustre-OST0001" */ + dash = memchr(tgtname, '-', LUSTRE_MAXFSNAME + 1); + if (!dash) { + fprintf(stderr, "wrong tgtname format '%s'\n", tgtname); + return -EINVAL; + } + dash += 4; + + *idx = strtoul(dash, &endp, 16); + if (*idx > 0xffff) { + fprintf(stderr, "wrong index %s\n", tgtname); + return -ERANGE; + } + + return 0; +} + static int print_obd_quota(char *mnt, struct if_quotactl *qctl, int is_mdt, bool h, __u64 *total) { - int rc = 0, rc1 = 0, count = 0; + int rc = 0, rc1 = 0, count = 0, i = 0; + char **list = NULL, *buffer = NULL; __u32 valid = qctl->qc_valid; - /* - * TODO: for commands LUSTRE_Q_"S\|G"ETQUOTAPOOL we need - * to go only through OSTs that belong to requested pool. - */ + if (qctl->qc_cmd == LUSTRE_Q_GETQUOTAPOOL && is_mdt) + return 0; + + /* Is it correct for the case OST0000, OST0002, OST0003 - + * we will ask OST0001 that is absent and won't ask OST0003? */ rc = llapi_get_obd_count(mnt, &count, is_mdt); if (rc) { fprintf(stderr, "can not get %s count: %s\n", @@ -7341,7 +7364,39 @@ static int print_obd_quota(char *mnt, struct if_quotactl *qctl, int is_mdt, return rc; } - for (qctl->qc_idx = 0; qctl->qc_idx < count; qctl->qc_idx++) { + if (qctl->qc_cmd == LUSTRE_Q_GETQUOTAPOOL) { + char fname[PATH_MAX]; + char fsname[LUSTRE_MAXFSNAME + 1]; + int bufsize = sizeof(struct obd_uuid) * count; + + rc = llapi_search_fsname(mnt, fsname); + if (rc) { + fprintf(stderr, "cannot get fsname for mountpoint %s\n", + mnt); + goto out; + } + buffer = malloc(bufsize + sizeof(*list) * count); + if (!buffer) + return -ENOMEM; + list = (char **)(buffer + bufsize); + snprintf(fname, PATH_MAX, "%s.%s", fsname, qctl->qc_poolname); + count = llapi_get_poolmembers(fname, list, count, + buffer, bufsize); + if (count <= 0) + goto out; + } + + for (i = 0; i < count; i++) { + if (qctl->qc_cmd == LUSTRE_Q_GETQUOTAPOOL) { + unsigned int index; + + if (tgt_name2index(list[i], &index)) + continue; + qctl->qc_idx = index; + } else { + qctl->qc_idx = i; + } + qctl->qc_valid = is_mdt ? QC_MDTIDX : QC_OSTIDX; rc = llapi_quotactl(mnt, qctl); if (rc) { @@ -7364,6 +7419,8 @@ static int print_obd_quota(char *mnt, struct if_quotactl *qctl, int is_mdt, qctl->qc_dqblk.dqb_bhardlimit; } out: + if (buffer) + free(buffer); qctl->qc_valid = valid; return rc ? : rc1; }