From 3074aac49bbe9401db6bd16859edd9c51cee8368 Mon Sep 17 00:00:00 2001 From: Sergey Cheremencev Date: Fri, 28 Oct 2022 18:29:03 +0800 Subject: [PATCH] LU-15031 quota: reseed glbe in qmt_lvbo_udate Reseed glbe array in qmt_lvbo_update after changing edquot. Without a fix edquot flag wasn't set in glbe array. Later, when edquot was cleared, need_update(nu) flag wasn't set in glbe array to notify OSTs with a new edquot. The patch also adds test 80 to check that OST gets correct edquot value after failover. Lustre-change: https://review.whamcloud.com/45032 Lustre-commit: 61ec1e0f2ca8dc4c9f7ed41f782960e65cab0920 HPE-bug-id: LUS-10029 Change-Id: I5b7e1a553e3351c22649431860d51b5a671c6fd9 Signed-off-by: Sergey Cheremencev Reviewed-on: https://review.whamcloud.com/c/ex/lustre-release/+/46655 Tested-by: jenkins Tested-by: Maloo Reviewed-by: Andreas Dilger --- lustre/include/obd_support.h | 1 + lustre/quota/qmt_handler.c | 4 ++ lustre/quota/qmt_lock.c | 13 ++++-- lustre/tests/sanity-quota.sh | 95 ++++++++++++++++++++++++++++++++++++++++++++ 4 files changed, 109 insertions(+), 4 deletions(-) diff --git a/lustre/include/obd_support.h b/lustre/include/obd_support.h index 2dfc2cd..09734ee 100644 --- a/lustre/include/obd_support.h +++ b/lustre/include/obd_support.h @@ -536,6 +536,7 @@ extern char obd_jobid_var[]; #define OBD_FAIL_QUOTA_DELAY_REINT 0xA03 #define OBD_FAIL_QUOTA_RECOVERABLE_ERR 0xA04 #define OBD_FAIL_QUOTA_INIT 0xA05 +#define OBD_FAIL_QUOTA_PREACQ 0xA06 #define OBD_FAIL_QUOTA_RECALC 0xA07 #define OBD_FAIL_LPROC_REMOVE 0xB00 diff --git a/lustre/quota/qmt_handler.c b/lustre/quota/qmt_handler.c index e5165b7..6ef8377 100644 --- a/lustre/quota/qmt_handler.c +++ b/lustre/quota/qmt_handler.c @@ -735,6 +735,10 @@ int qmt_dqacq0(const struct lu_env *env, struct qmt_device *qmt, if (OBD_FAIL_CHECK(OBD_FAIL_QUOTA_RECOVERABLE_ERR)) RETURN(-cfs_fail_val); + if (OBD_FAIL_CHECK(OBD_FAIL_QUOTA_PREACQ) && + (req_is_preacq(qb_flags) || req_is_rel(qb_flags))) + RETURN(-EAGAIN); + if (qti_lqes_restore_init(env)) RETURN(-ENOMEM); diff --git a/lustre/quota/qmt_lock.c b/lustre/quota/qmt_lock.c index e5534693..538df0b 100644 --- a/lustre/quota/qmt_lock.c +++ b/lustre/quota/qmt_lock.c @@ -287,11 +287,12 @@ static bool qmt_clear_lgeg_arr_nu(struct lquota_entry *lqe, int stype, int idx) return false; } -static void qmt_set_revoke(struct lu_env *env, struct lquota_entry *lqe, +static bool qmt_set_revoke(struct lu_env *env, struct lquota_entry *lqe, int stype, int idx) { unsigned long least_qunit = lqe2qpi(lqe)->qpi_least_qunit; struct lqe_glbl_data *lgd = lqe->lqe_glbl_data; + bool notify = false; if (lgd->lqeg_arr[idx].lge_qunit == least_qunit) { int i; @@ -305,12 +306,13 @@ static void qmt_set_revoke(struct lu_env *env, struct lquota_entry *lqe, if (qti_lqes(env)[i]->lqe_qunit == least_qunit) { qti_lqes(env)[i]->lqe_revoke_time = ktime_get_seconds(); - qmt_adjust_edquot(qti_lqes(env)[i], + notify |= qmt_adjust_edquot(qti_lqes(env)[i], ktime_get_real_seconds()); } } qti_lqes_write_unlock(env); } + return notify; } /* @@ -399,8 +401,11 @@ int qmt_lvbo_update(struct lu_device *ld, struct ldlm_resource *res, if (rc) GOTO(out_exp, rc); - if (need_revoke) - qmt_set_revoke(env, lqe, rc, idx); + if (need_revoke && qmt_set_revoke(env, lqe, rc, idx) && + lqe->lqe_glbl_data) { + qmt_seed_glbe_edquot(env, lqe->lqe_glbl_data); + qmt_id_lock_notify(qmt, lqe); + } if (lvb->lvb_id_rel) { LQUOTA_DEBUG(lqe, "releasing:%llu may release:%llu", diff --git a/lustre/tests/sanity-quota.sh b/lustre/tests/sanity-quota.sh index 148eba4..f2e4ed3 100755 --- a/lustre/tests/sanity-quota.sh +++ b/lustre/tests/sanity-quota.sh @@ -5085,6 +5085,7 @@ function cleanup_quota_test_75() resetquota -u $TSTUSR + rm -rf $DIR/${tdir}_dom cleanup_quota_test } @@ -5279,6 +5280,7 @@ test_78() check_set_fallocate_or_skip setup_quota_test || error "setup quota failed with $?" + stack_trap cleanup_quota_test EXIT # enable ost quota set_ost_qtype $QTYPE || error "enable ost quota failed" @@ -5313,6 +5315,99 @@ test_78() } run_test 78 "Check fallocate increase quota usage" +test_79() +{ + local qpool="qpool1" + local cmd="$LCTL get_param -n qmt.$FSNAME-QMT0000.dt-$qpool.info" + local stopf=$TMP/$tfile + + do_facet mds1 "touch $stopf" + stack_trap "do_facet mds1 'rm -f $stopf'" + do_facet mds1 "while [ -e $stopf ]; do $cmd &>/dev/null; done"& + local pid=$! + pool_add $qpool || error "pool_add failed" + do_facet mds1 "rm $stopf" + wait $pid +} +run_test 79 "access to non-existed dt-pool/info doesn't cause a panic" + +test_80() +{ + local dir1="$DIR/$tdir/dir1" + local dir2="$DIR/$tdir/dir2" + local TESTFILE0="$dir1/$tfile-0" + local TESTFILE1="$dir1/$tfile-1" + local TESTFILE2="$dir1/$tfile-2" + local TESTFILE3="$dir2/$tfile-0" + local global_limit=100 # 100M + local limit=10 # 10M + local qpool="qpool1" + + [ "$OSTCOUNT" -lt "2" ] && skip "needs >= 2 OSTs" + mds_supports_qp + [ "$ost1_FSTYPE" == zfs ] && + skip "ZFS grants some block space together with inode" + setup_quota_test || error "setup quota failed with $?" + stack_trap cleanup_quota_test EXIT + set_ost_qtype $QTYPE || error "enable ost quota failed" + + # make sure the system is clean + local used=$(getquota -u $TSTUSR global curspace) + [ $used -ne 0 ] && { + ls -laR $DIR + $LFS quota -v -u $TSTUSR $DIR + error "Used space($used) for user $TSTUSR is not 0." + } + + pool_add $qpool || error "pool_add failed" + pool_add_targets $qpool 0 1 || + error "pool_add_targets failed" + + $LFS setquota -u $TSTUSR -b 0 -B ${global_limit}M -i 0 -I 0 $DIR || + error "set user quota failed" + + $LFS setquota -u $TSTUSR -B ${global_limit}M --pool $qpool $DIR || + error "set user quota failed" + $LFS setquota -u $TSTUSR -B ${limit}M --pool $qpool $DIR || + error "set user quota failed" + + mkdir -p $dir1 || error "failed to mkdir" + chown $TSTUSR.$TSTUSR $dir1 || error "chown $dir1 failed" + mkdir -p $dir2 || error "failed to mkdir" + chown $TSTUSR.$TSTUSR $dir2 || error "chown $dir2 failed" + + $LFS setstripe $dir1 -i 1 -c 1|| error "setstripe $testfile failed" + $LFS setstripe $dir2 -i 0 -c 1|| error "setstripe $testfile failed" + lfs getstripe $dir1 + lfs getstripe $dir2 + sleep 3 + + $LFS quota -uv $TSTUSR $DIR + #define OBD_FAIL_QUOTA_PREACQ 0xA06 + do_facet mds1 $LCTL set_param fail_loc=0xa06 + $RUNAS $DD of=$TESTFILE3 count=3 || + quota_error u $TSTUSR "write failed" + $RUNAS $DD of=$TESTFILE2 count=7 || + quota_error u $TSTUSR "write failed" + $RUNAS $DD of=$TESTFILE1 count=1 oflag=direct || + quota_error u $TSTUSR "write failed" + sync + sleep 3 + $LFS quota -uv --pool $qpool $TSTUSR $DIR + + rm -f $TESTFILE2 + stop ost2 + do_facet mds1 $LCTL set_param fail_loc=0 + start ost2 $(ostdevname 2) $OST_MOUNT_OPTS || error "start ost2 failed" + $LFS quota -uv $TSTUSR --pool $qpool $DIR + # OST0 needs some time to update quota usage after removing TESTFILE2 + sleep 4 + $LFS quota -uv $TSTUSR --pool $qpool $DIR + $RUNAS $DD of=$TESTFILE0 count=2 oflag=direct || + quota_error u $TSTUSR "write failure, but expect success" +} +run_test 80 "check for EDQUOT after OST failover" + test_81() { local global_limit=20 # 100M local testfile="$DIR/$tdir/$tfile-0" -- 1.8.3.1