From 18cd3e1e28afd311e4743dab1011f85fba0e1765 Mon Sep 17 00:00:00 2001 From: Wang Shilong Date: Mon, 21 Dec 2020 10:18:45 +0800 Subject: [PATCH 1/1] LU-12702 quota: wait pending write before acquiring remotely There is a window that ZFS has updated usage, but pending_write has not been reduced. This could cause us to grant more space. With soft least qunit introduced, after timer reach, local slave might still have some space to consume, it is possible that we might exceed this limit. Test-Parameters: fstype=zfs testlist=sanity-quota,sanity-quota,sanity-quota Signed-off-by: Wang Shilong Change-Id: I0e1a94fd0b637e154642de5922a0670780c56ef2 Reviewed-on: https://review.whamcloud.com/41023 Tested-by: jenkins Reviewed-by: Andreas Dilger Reviewed-by: Hongchao Zhang Tested-by: Maloo Reviewed-by: Oleg Drokin --- lustre/include/dt_object.h | 19 +++++++++++++++++++ lustre/osd-zfs/osd_handler.c | 10 ++++++++++ lustre/quota/qsd_handler.c | 15 ++++++++++++++- lustre/tests/sanity-quota.sh | 10 ++++------ 4 files changed, 47 insertions(+), 7 deletions(-) diff --git a/lustre/include/dt_object.h b/lustre/include/dt_object.h index 45b136a..e1d574a 100644 --- a/lustre/include/dt_object.h +++ b/lustre/include/dt_object.h @@ -298,6 +298,17 @@ struct dt_device_operations { struct dt_device *dev); /** + * Wait pending quota update finish + * + * There might be a window that quota usage has been updated, + * but commit callback to reduce pending write have not been + * finished, this is used to wait all pending update done. + * + * \param[in] dev dt device + */ + void (*dt_wait_quota_pending)(struct dt_device *dev); + + /** * Start transaction commit asynchronously. * @@ -2697,6 +2708,14 @@ static inline int dt_ro(const struct lu_env *env, struct dt_device *dev) return dev->dd_ops->dt_ro(env, dev); } +static inline void dt_wait_quota_pending(struct dt_device *dev) +{ + LASSERT(dev); + LASSERT(dev->dd_ops); + if (dev->dd_ops->dt_wait_quota_pending) + dev->dd_ops->dt_wait_quota_pending(dev); +} + static inline int dt_declare_insert(const struct lu_env *env, struct dt_object *dt, const struct dt_rec *rec, diff --git a/lustre/osd-zfs/osd_handler.c b/lustre/osd-zfs/osd_handler.c index 0b4c41f..7a8bc16 100644 --- a/lustre/osd-zfs/osd_handler.c +++ b/lustre/osd-zfs/osd_handler.c @@ -699,6 +699,15 @@ static int osd_ro(const struct lu_env *env, struct dt_device *d) RETURN(0); } +static void osd_wait_quota_pending(struct dt_device *d) +{ + struct osd_device *o = osd_dt_dev(d); + + if (o->od_quota_slave_md != NULL || + o->od_quota_slave_dt != NULL) + txg_wait_callbacks(spa_get_dsl(dmu_objset_spa(o->od_os))); +} + static struct dt_device_operations osd_dt_ops = { .dt_root_get = osd_root_get, .dt_statfs = osd_statfs, @@ -710,6 +719,7 @@ static struct dt_device_operations osd_dt_ops = { .dt_sync = osd_sync, .dt_commit_async = osd_commit_async, .dt_ro = osd_ro, + .dt_wait_quota_pending = osd_wait_quota_pending, }; /* diff --git a/lustre/quota/qsd_handler.c b/lustre/quota/qsd_handler.c index ed67286..ae770f5 100644 --- a/lustre/quota/qsd_handler.c +++ b/lustre/quota/qsd_handler.c @@ -631,11 +631,14 @@ static bool qsd_acquire(const struct lu_env *env, struct lquota_entry *lqe, long long space, int *ret) { int rc = 0, count; + int wait_pending = 0; + struct qsd_qtype_info *qqi = lqe2qqi(lqe); + ENTRY; for (count = 0; rc == 0; count++) { LQUOTA_DEBUG(lqe, "acquiring:%lld count=%d", space, count); - +again: if (lqe2qqi(lqe)->qqi_qsd->qsd_stopping) { rc = -EINPROGRESS; break; @@ -652,6 +655,16 @@ static bool qsd_acquire(const struct lu_env *env, struct lquota_entry *lqe, /* rc == 0, Wouhou! enough local quota space * rc < 0, something bad happened */ break; + /* + * There might be a window that commit transaction + * have updated usage but pending write doesn't change + * wait for it before acquiring remotely. + */ + if (lqe->lqe_pending_write >= space && !wait_pending) { + wait_pending = 1; + dt_wait_quota_pending(qqi->qqi_qsd->qsd_dev); + goto again; + } /* if we have gotten some quota and stil wait more quota, * it's better to give QMT some time to reclaim from clients */ diff --git a/lustre/tests/sanity-quota.sh b/lustre/tests/sanity-quota.sh index 210a579..2347d59 100755 --- a/lustre/tests/sanity-quota.sh +++ b/lustre/tests/sanity-quota.sh @@ -1565,6 +1565,8 @@ test_file_soft() { local LIMIT=$2 local grace=$3 local qtype=$4 + local SOFT_LIMIT=$(do_facet $SINGLEMDS $LCTL get_param -n \ + qmt.$FSNAME-QMT0000.md-0x0.soft_least_qunit) setup_quota_test trap cleanup_quota_test EXIT @@ -1597,12 +1599,8 @@ test_file_soft() { $SHOW_QUOTA_INFO_PROJID echo "Create file after timer goes off" - # There is a window that space is accounted in the quota usage but - # hasn't been decreased from the pending write, if we acquire quota - # in this window, we'll acquire more than we needed. - $RUNAS touch ${TESTFILE}_after_1 ${TESTFILE}_after_2 || true - sync_all_data || true - $RUNAS touch ${TESTFILE}_after_3 && + # exceed least soft limit is possible + $RUNAS createmany -m ${TESTFILE}_after_3 $((SOFT_LIMIT + 1)) && quota_error a $TSTUSR "create after timer expired," \ "but expect EDQUOT" sync_all_data || true -- 1.8.3.1