From c226e70007aab798c39ccd0fe13ddcba65f04f23 Mon Sep 17 00:00:00 2001 From: Alexander Zarochentsev Date: Tue, 31 Aug 2021 08:36:30 +0300 Subject: [PATCH 1/1] LU-15114 osp: changes queuing throttle Prevent queue of sync changes from growing too much by adding resends when queue size reaches some (tunable) limit. HPE-bug-id: LUS-10345 Signed-off-by: Alexander Zarochentsev Change-Id: I5efabb91d3700c58d9451f81c5fed9a22ae404fb Reviewed-on: https://review.whamcloud.com/45265 Reviewed-by: Alexander Boyko Reviewed-by: Andrew Perepechko Reviewed-by: Lai Siyao Tested-by: jenkins Tested-by: Maloo Reviewed-by: Oleg Drokin --- lustre/osp/lproc_osp.c | 32 ++++++++++++++++++++++++++++++++ lustre/osp/osp_internal.h | 2 ++ lustre/osp/osp_sync.c | 10 ++++++++++ lustre/tests/sanity.sh | 42 ++++++++++++++++++++++++++++++++++++++++++ 4 files changed, 86 insertions(+) diff --git a/lustre/osp/lproc_osp.c b/lustre/osp/lproc_osp.c index f915719..f0eb765 100644 --- a/lustre/osp/lproc_osp.c +++ b/lustre/osp/lproc_osp.c @@ -193,6 +193,37 @@ static ssize_t sync_changes_store(struct kobject *kobj, struct attribute *attr, } LUSTRE_RW_ATTR(sync_changes); +static ssize_t max_sync_changes_show(struct kobject *kobj, + struct attribute *attr, char *buf) +{ + struct dt_device *dt = container_of(kobj, struct dt_device, + dd_kobj); + struct osp_device *osp = dt2osp_dev(dt); + + return sprintf(buf, "%u\n", osp->opd_sync_max_changes); +} + + +static ssize_t max_sync_changes_store(struct kobject *kobj, + struct attribute *attr, const char *buffer, size_t count) +{ + struct dt_device *dt = container_of(kobj, struct dt_device, + dd_kobj); + struct osp_device *osp = dt2osp_dev(dt); + int val; + int rc; + + rc = kstrtoint(buffer, 0, &val); + if (rc) + return rc; + if (val <= 0) + return -ERANGE; + osp->opd_sync_max_changes = val; + return count; +} + +LUSTRE_RW_ATTR(max_sync_changes); + /** * Show maximum number of RPCs in flight allowed * @@ -1011,6 +1042,7 @@ static struct attribute *osp_obd_attrs[] = { &lustre_attr_sync_in_flight.attr, &lustre_attr_sync_in_progress.attr, &lustre_attr_sync_changes.attr, + &lustre_attr_max_sync_changes.attr, &lustre_attr_force_sync.attr, &lustre_attr_old_sync_processed.attr, &lustre_attr_create_count.attr, diff --git a/lustre/osp/osp_internal.h b/lustre/osp/osp_internal.h index 72e79f2..8930a2c 100644 --- a/lustre/osp/osp_internal.h +++ b/lustre/osp/osp_internal.h @@ -213,6 +213,8 @@ struct osp_device { struct llog_gen opd_sync_generation; /* number of changes to sync, used to wake up sync thread */ atomic_t opd_sync_changes; + /* limit of changes to sync */ + int opd_sync_max_changes; /* processing of changes from previous mount is done? */ int opd_sync_prev_done; /* found records */ diff --git a/lustre/osp/osp_sync.c b/lustre/osp/osp_sync.c index d270453..2f1db6b 100644 --- a/lustre/osp/osp_sync.c +++ b/lustre/osp/osp_sync.c @@ -83,6 +83,7 @@ #define OSP_SYNC_THRESHOLD 10 #define OSP_MAX_RPCS_IN_FLIGHT 8 #define OSP_MAX_RPCS_IN_PROGRESS 4096 +#define OSP_MAX_SYNC_CHANGES 2000000000 #define OSP_JOB_MAGIC 0x26112005 @@ -309,6 +310,14 @@ int osp_sync_declare_add(const struct lu_env *env, struct osp_object *o, /* it's a layering violation, to access internals of th, * but we can do this as a sanity check, for a while */ LASSERT(th->th_top != NULL); + + if (atomic_read(&d->opd_sync_changes) > d->opd_sync_max_changes) { + /* Attempt to slow sync changes queuing rate */ + CWARN("%s: queued changes counter exceeds limit %d > %d\n", + d->opd_obd->obd_name, atomic_read(&d->opd_sync_changes), + d->opd_sync_max_changes); + RETURN(-EINPROGRESS); + } storage_th = thandle_get_sub_by_dt(env, th->th_top, d->opd_storage); if (IS_ERR(storage_th)) RETURN(PTR_ERR(storage_th)); @@ -1520,6 +1529,7 @@ int osp_sync_init(const struct lu_env *env, struct osp_device *d) d->opd_sync_max_rpcs_in_flight = OSP_MAX_RPCS_IN_FLIGHT; d->opd_sync_max_rpcs_in_progress = OSP_MAX_RPCS_IN_PROGRESS; + d->opd_sync_max_changes = OSP_MAX_SYNC_CHANGES; spin_lock_init(&d->opd_sync_lock); init_waitqueue_head(&d->opd_sync_waitq); init_waitqueue_head(&d->opd_sync_barrier_waitq); diff --git a/lustre/tests/sanity.sh b/lustre/tests/sanity.sh index b3b11f8..ce1e1ae 100755 --- a/lustre/tests/sanity.sh +++ b/lustre/tests/sanity.sh @@ -27591,6 +27591,48 @@ test_823() { } run_test 823 "Setting create_count > OST_MAX_PRECREATE is lowered to maximum" +test_831() { + local sync_changes=$(do_facet $SINGLEMDS \ + $LCTL get_param -n osp.$FSNAME-OST0000-osc-MDT0000.sync_changes) + + [ "$sync_changes" -gt 100 ] && + skip "Sync changes $sync_changes > 100 already" + + local p="$TMP/$TESTSUITE-$TESTNAME.parameters" + + $LFS mkdir -i 0 $DIR/$tdir + $LFS setstripe -c 1 -i 0 $DIR/$tdir + + save_lustre_params mds1 \ + "osp.$FSNAME-OST*-osc-MDT0000.max_sync_changes" > $p + save_lustre_params mds1 \ + "osp.$FSNAME-OST*-osc-MDT0000.max_rpcs_in_progress" >> $p + + do_facet mds1 "$LCTL set_param -n \ + osp.$FSNAME-OST*-osc-MDT0000.max_sync_changes=100 \ + osp.$FSNAME-OST*-osc-MDT0000.max_rpcs_in_progress=128" + stack_trap "restore_lustre_params < $p" EXIT + + createmany -o $DIR/$tdir/f- 1000 + unlinkmany $DIR/$tdir/f- 1000 & + local UNLINK_PID=$! + + while sleep 1; do + sync_changes=$(do_facet mds1 \ + $LCTL get_param -n osp.$FSNAME-OST0000-osc-MDT0000.sync_changes) + # the check in the code is racy, fail the test + # if the value above the limit by 10. + [ $sync_changes -gt 110 ] && { + kill -2 $UNLINK_PID + wait + error "osp changes throttling failed, $sync_changes>110" + } + kill -0 $UNLINK_PID 2> /dev/null || break + done + wait +} +run_test 831 "throttling unlink/setattr queuing on OSP" + # # tests that do cleanup/setup should be run at the end # -- 1.8.3.1