LU-15114 osp: changes queuing throttle

Prevent queue of sync changes from growing too much
by adding resends when queue size reaches
some (tunable) limit.

HPE-bug-id: LUS-10345
Signed-off-by: Alexander Zarochentsev <alexander.zarochentsev@hpe.com>
Change-Id: I5efabb91d3700c58d9451f81c5fed9a22ae404fb
Reviewed-on: https://review.whamcloud.com/45265
Reviewed-by: Alexander Boyko <alexander.boyko@hpe.com>
Reviewed-by: Andrew Perepechko <andrew.perepechko@hpe.com>
Reviewed-by: Lai Siyao <lai.siyao@whamcloud.com>
Tested-by: jenkins <devops@whamcloud.com>
Tested-by: Maloo <maloo@whamcloud.com>
Reviewed-by: Oleg Drokin <green@whamcloud.com>

diff --git a/lustre/osp/lproc_osp.c b/lustre/osp/lproc_osp.c
index f915719..f0eb765 100644 (file)
--- a/lustre/osp/lproc_osp.c
+++ b/lustre/osp/lproc_osp.c
@@ -193,6 +193,37 @@ static ssize_t sync_changes_store(struct kobject *kobj, struct attribute *attr,
 }
 LUSTRE_RW_ATTR(sync_changes);
 
+static ssize_t max_sync_changes_show(struct kobject *kobj,
+               struct attribute *attr, char *buf)
+{
+       struct dt_device *dt = container_of(kobj, struct dt_device,
+                                           dd_kobj);
+       struct osp_device *osp = dt2osp_dev(dt);
+
+       return sprintf(buf, "%u\n", osp->opd_sync_max_changes);
+}
+
+
+static ssize_t max_sync_changes_store(struct kobject *kobj,
+               struct attribute *attr, const char *buffer, size_t count)
+{
+       struct dt_device *dt = container_of(kobj, struct dt_device,
+                                           dd_kobj);
+       struct osp_device *osp = dt2osp_dev(dt);
+       int val;
+       int rc;
+
+       rc = kstrtoint(buffer, 0, &val);
+       if (rc)
+               return rc;
+       if (val <= 0)
+               return -ERANGE;
+       osp->opd_sync_max_changes = val;
+       return count;
+}
+
+LUSTRE_RW_ATTR(max_sync_changes);
+
 /**
  * Show maximum number of RPCs in flight allowed
  *
@@ -1011,6 +1042,7 @@ static struct attribute *osp_obd_attrs[] = {
        &lustre_attr_sync_in_flight.attr,
        &lustre_attr_sync_in_progress.attr,
        &lustre_attr_sync_changes.attr,
+       &lustre_attr_max_sync_changes.attr,
        &lustre_attr_force_sync.attr,
        &lustre_attr_old_sync_processed.attr,
        &lustre_attr_create_count.attr,

diff --git a/lustre/osp/osp_internal.h b/lustre/osp/osp_internal.h
index 72e79f2..8930a2c 100644 (file)
--- a/lustre/osp/osp_internal.h
+++ b/lustre/osp/osp_internal.h
@@ -213,6 +213,8 @@ struct osp_device {
        struct llog_gen                  opd_sync_generation;
        /* number of changes to sync, used to wake up sync thread */
        atomic_t                         opd_sync_changes;
+       /* limit of changes to sync */
+       int                              opd_sync_max_changes;
        /* processing of changes from previous mount is done? */
        int                              opd_sync_prev_done;
        /* found records */

diff --git a/lustre/osp/osp_sync.c b/lustre/osp/osp_sync.c
index d270453..2f1db6b 100644 (file)
--- a/lustre/osp/osp_sync.c
+++ b/lustre/osp/osp_sync.c
@@ -83,6 +83,7 @@
 #define OSP_SYNC_THRESHOLD             10
 #define OSP_MAX_RPCS_IN_FLIGHT         8
 #define OSP_MAX_RPCS_IN_PROGRESS       4096
+#define OSP_MAX_SYNC_CHANGES           2000000000
 
 #define OSP_JOB_MAGIC          0x26112005
 
@@ -309,6 +310,14 @@ int osp_sync_declare_add(const struct lu_env *env, struct osp_object *o,
        /* it's a layering violation, to access internals of th,
         * but we can do this as a sanity check, for a while */
        LASSERT(th->th_top != NULL);
+
+       if (atomic_read(&d->opd_sync_changes) > d->opd_sync_max_changes) {
+               /* Attempt to slow sync changes queuing rate */
+               CWARN("%s: queued changes counter exceeds limit %d > %d\n",
+                     d->opd_obd->obd_name, atomic_read(&d->opd_sync_changes),
+                     d->opd_sync_max_changes);
+               RETURN(-EINPROGRESS);
+       }
        storage_th = thandle_get_sub_by_dt(env, th->th_top, d->opd_storage);
        if (IS_ERR(storage_th))
                RETURN(PTR_ERR(storage_th));
@@ -1520,6 +1529,7 @@ int osp_sync_init(const struct lu_env *env, struct osp_device *d)
 
        d->opd_sync_max_rpcs_in_flight = OSP_MAX_RPCS_IN_FLIGHT;
        d->opd_sync_max_rpcs_in_progress = OSP_MAX_RPCS_IN_PROGRESS;
+       d->opd_sync_max_changes = OSP_MAX_SYNC_CHANGES;
        spin_lock_init(&d->opd_sync_lock);
        init_waitqueue_head(&d->opd_sync_waitq);
        init_waitqueue_head(&d->opd_sync_barrier_waitq);

diff --git a/lustre/tests/sanity.sh b/lustre/tests/sanity.sh
index b3b11f8..ce1e1ae 100755 (executable)
--- a/lustre/tests/sanity.sh
+++ b/lustre/tests/sanity.sh
@@ -27591,6 +27591,48 @@ test_823() {
 }
 run_test 823 "Setting create_count > OST_MAX_PRECREATE is lowered to maximum"
 
+test_831() {
+       local sync_changes=$(do_facet $SINGLEMDS \
+               $LCTL get_param -n osp.$FSNAME-OST0000-osc-MDT0000.sync_changes)
+
+       [ "$sync_changes" -gt 100 ] &&
+               skip "Sync changes $sync_changes > 100 already"
+
+       local p="$TMP/$TESTSUITE-$TESTNAME.parameters"
+
+       $LFS mkdir -i 0 $DIR/$tdir
+       $LFS setstripe -c 1 -i 0 $DIR/$tdir
+
+       save_lustre_params mds1 \
+               "osp.$FSNAME-OST*-osc-MDT0000.max_sync_changes" > $p
+       save_lustre_params mds1 \
+               "osp.$FSNAME-OST*-osc-MDT0000.max_rpcs_in_progress" >> $p
+
+       do_facet mds1 "$LCTL set_param -n \
+               osp.$FSNAME-OST*-osc-MDT0000.max_sync_changes=100 \
+               osp.$FSNAME-OST*-osc-MDT0000.max_rpcs_in_progress=128"
+       stack_trap "restore_lustre_params < $p" EXIT
+
+       createmany -o $DIR/$tdir/f- 1000
+       unlinkmany $DIR/$tdir/f- 1000 &
+       local UNLINK_PID=$!
+
+       while sleep 1; do
+               sync_changes=$(do_facet mds1 \
+               $LCTL get_param -n osp.$FSNAME-OST0000-osc-MDT0000.sync_changes)
+               # the check in the code is racy, fail the test
+               # if the value above the limit by 10.
+               [ $sync_changes -gt 110 ] && {
+                       kill -2 $UNLINK_PID
+                       wait
+                       error "osp changes throttling failed, $sync_changes>110"
+               }
+               kill -0 $UNLINK_PID 2> /dev/null || break
+       done
+       wait
+}
+run_test 831 "throttling unlink/setattr queuing on OSP"
+
 #
 # tests that do cleanup/setup should be run at the end
 #