LU-11912 tests: fix racing in force_new_seq_all

We run force_new_seq in parallel to reduce time spent
on consuming precreated objects.

However this could be racy when multiple MDTs are on
the same MDS, a task could finish for one MDT early
and reset the fail_loc to 0 on MDS while other tasks
are still working on other MDTs.

Replace OBD_FAIL_OSP_FORCE_NEW_SEQ with a new param
prealloc_force_new_seq for osp, so we can control
the seq rollover individually for each osp device.

Lustre-change: https://review.whamcloud.com/52801
Lustre-commit: TBD (from af6dcd597d7f5134de553349c05091e51e0f3dd6)

Change-Id: I52dbd550564ca628a8a85c42951694d58b2b93a9
Fixes: 656fc937cf ("LU-11912 tests: consume precreated objects in parallel")
Signed-off-by: Li Dongyang <dongyangli@ddn.com>
Reviewed-on: https://review.whamcloud.com/c/ex/lustre-release/+/52802
Reviewed-by: Andreas Dilger <adilger@whamcloud.com>
Tested-by: jenkins <devops@whamcloud.com>
Tested-by: Maloo <maloo@whamcloud.com>

diff --git a/lustre/include/obd_support.h b/lustre/include/obd_support.h
index 78d16f2..a4a9ac1 100644 (file)
--- a/lustre/include/obd_support.h
+++ b/lustre/include/obd_support.h
@@ -752,7 +752,6 @@ extern char obd_jobid_var[];
 #define OBD_FAIL_OSP_INVALID_LOGID             0x2106
 #define OBD_FAIL_OSP_CON_EVENT_DELAY           0x2107
 #define OBD_FAIL_OSP_PRECREATE_PAUSE           0x2108
-#define OBD_FAIL_OSP_FORCE_NEW_SEQ             0x210a
 
 /* barrier */
 #define OBD_FAIL_MGS_BARRIER_READ_NET          0x2200

diff --git a/lustre/osp/lproc_osp.c b/lustre/osp/lproc_osp.c
index a356e96..ddb3d9d 100644 (file)
--- a/lustre/osp/lproc_osp.c
+++ b/lustre/osp/lproc_osp.c
@@ -654,6 +654,44 @@ static ssize_t prealloc_status_show(struct kobject *kobj,
 }
 LUSTRE_RO_ATTR(prealloc_status);
 
+static ssize_t prealloc_force_new_seq_show(struct kobject *kobj,
+                                          struct attribute *attr,
+                                          char *buf)
+{
+       struct dt_device *dt = container_of(kobj, struct dt_device,
+                                           dd_kobj);
+       struct osp_device *osp = dt2osp_dev(dt);
+
+       if (!osp->opd_pre)
+               return -EINVAL;
+
+       return scnprintf(buf, PAGE_SIZE, "%d\n", osp->opd_pre_force_new_seq);
+}
+
+static ssize_t prealloc_force_new_seq_store(struct kobject *kobj,
+                                           struct attribute *attr,
+                                           const char *buffer,
+                                           size_t count)
+{
+       struct dt_device *dt = container_of(kobj, struct dt_device,
+                                           dd_kobj);
+       struct osp_device *osp = dt2osp_dev(dt);
+       bool val;
+       int rc;
+
+       if (!osp->opd_pre)
+               return -EINVAL;
+
+       rc = kstrtobool(buffer, &val);
+       if (rc)
+               return rc;
+
+       osp->opd_pre_force_new_seq = val;
+
+       return count;
+}
+LUSTRE_RW_ATTR(prealloc_force_new_seq);
+
 /**
  * Show the number of RPCs in processing (including uncommitted by OST) plus
  * changes to sync, i.e. this is the total number of changes OST needs to apply
@@ -1047,6 +1085,7 @@ static struct attribute *osp_obd_attrs[] = {
        &lustre_attr_prealloc_next_seq.attr,
        &lustre_attr_prealloc_last_seq.attr,
        &lustre_attr_prealloc_reserved.attr,
+       &lustre_attr_prealloc_force_new_seq.attr,
        &lustre_attr_sync_in_flight.attr,
        &lustre_attr_sync_in_progress.attr,
        &lustre_attr_sync_changes.attr,

diff --git a/lustre/osp/osp_internal.h b/lustre/osp/osp_internal.h
index 7a7f5e4..fd0879b 100644 (file)
--- a/lustre/osp/osp_internal.h
+++ b/lustre/osp/osp_internal.h
@@ -84,9 +84,11 @@ struct osp_precreate {
        int                              osp_pre_min_create_count;
        int                              osp_pre_max_create_count;
        /* whether to increase precreation window next time or not */
-       int                              osp_pre_create_slow;
+       unsigned int                     osp_pre_create_slow:1,
        /* cleaning up orphans or recreating missing objects */
-       int                              osp_pre_recovering;
+                                        osp_pre_recovering:1,
+       /* force new seq rollover */
+                                        osp_pre_force_new_seq:1;
 };
 
 struct osp_update_request_sub {
@@ -296,6 +298,7 @@ struct osp_device {
 #define opd_pre_max_create_count       opd_pre->osp_pre_max_create_count
 #define opd_pre_create_slow            opd_pre->osp_pre_create_slow
 #define opd_pre_recovering             opd_pre->osp_pre_recovering
+#define opd_pre_force_new_seq          opd_pre->osp_pre_force_new_seq
 
 extern struct kmem_cache *osp_object_kmem;
 
@@ -673,7 +676,7 @@ static bool osp_fid_end_seq(const struct lu_env *env, struct lu_fid *fid,
        /* Skip IDIF sequence for MDT0000 */
        if (fid_is_idif(fid))
                return true;
-       if (OBD_FAIL_CHECK(OBD_FAIL_OSP_FORCE_NEW_SEQ))
+       if (osp->opd_pre_force_new_seq)
                return true;
        return fid_oid(fid) >= min(OBIF_MAX_OID, seq_width);
 }

diff --git a/lustre/tests/test-framework.sh b/lustre/tests/test-framework.sh
index bef0caa..25d5e89 100755 (executable)
--- a/lustre/tests/test-framework.sh
+++ b/lustre/tests/test-framework.sh
@@ -11493,20 +11493,32 @@ exhaust_all_precreations() {
        sleep_maxage
 }
 
+force_new_seq_ost() {
+       local dir=$1
+       local mfacet=$2
+       local OSTIDX=$3
+       local OST=$(ostname_from_index $OSTIDX)
+       local mdtosc_proc=$(get_mdtosc_proc_path $mfacet $OST)
+
+       do_facet $mfacet $LCTL set_param \
+               osp.$mdtosc_proc.prealloc_force_new_seq=1
+       # consume preallocated objects, to wake up precreate thread
+       consume_precreations $dir $mfacet $OSTIDX
+       do_facet $mfacet $LCTL set_param \
+               osp.$mdtosc_proc.prealloc_force_new_seq=0
+}
+
 force_new_seq() {
        local mfacet=$1
        local MDTIDX=$(facet_index $mfacet)
        local MDT=$(mdtname_from_index $MDTIDX $DIR)
        local i
 
-#define OBD_FAIL_OSP_FORCE_NEW_SEQ             0x210a
-       do_facet $mfacet $LCTL set_param fail_loc=0x210a
        mkdir_on_mdt -i $MDTIDX $DIR/${MDT}
        for (( i=0; i < OSTCOUNT; i++ )) ; do
-               # consume preallocated objects, to wake up precreate thread
-               consume_precreations $DIR/${MDT} $mfacet $i
+               force_new_seq_ost $DIR/${MDT} $mfacet $i &
        done
-       do_facet $mfacet $LCTL set_param fail_loc=0
+       wait
        rm -rf $DIR/${MDT}
 }