LU-11912 tests: fix racing in force_new_seq_all

We run force_new_seq in parallel to reduce time spent
on consuming precreated objects.

However this could be racy when multiple MDTs are on
the same MDS, a task could finish for one MDT early
and reset the fail_loc to 0 on MDS while other tasks
are still working on other MDTs.

Replace OBD_FAIL_OSP_FORCE_NEW_SEQ with a new param
prealloc_force_new_seq for osp, so we can control
the seq rollover individually for each osp device.

Change-Id: I52dbd550564ca628a8a85c42951694d58b2b93a9
Fixes: 656fc937cf ("LU-11912 tests: consume precreated objects in parallel")
Signed-off-by: Li Dongyang <dongyangli@ddn.com>
Reviewed-on: https://review.whamcloud.com/c/fs/lustre-release/+/52801
Tested-by: jenkins <devops@whamcloud.com>
Tested-by: Maloo <maloo@whamcloud.com>
Reviewed-by: Jian Yu <yujian@whamcloud.com>
Reviewed-by: Andreas Dilger <adilger@whamcloud.com>
Reviewed-by: Oleg Drokin <green@whamcloud.com>

diff --git a/lustre/include/obd_support.h b/lustre/include/obd_support.h
index c444e80..ebb4809 100644 (file)
--- a/lustre/include/obd_support.h
+++ b/lustre/include/obd_support.h
@@ -749,7 +749,6 @@ extern char obd_jobid_var[];
 #define OBD_FAIL_OSP_CANT_PROCESS_LLOG         0x2105
 #define OBD_FAIL_OSP_INVALID_LOGID             0x2106
 #define OBD_FAIL_OSP_CON_EVENT_DELAY           0x2107
-#define OBD_FAIL_OSP_FORCE_NEW_SEQ             0x210a
 
 /* barrier */
 #define OBD_FAIL_MGS_BARRIER_READ_NET          0x2200

diff --git a/lustre/osp/lproc_osp.c b/lustre/osp/lproc_osp.c
index 6d4e09b..bc7e695 100644 (file)
--- a/lustre/osp/lproc_osp.c
+++ b/lustre/osp/lproc_osp.c
@@ -732,6 +732,44 @@ static ssize_t prealloc_status_show(struct kobject *kobj,
 }
 LUSTRE_RO_ATTR(prealloc_status);
 
+static ssize_t prealloc_force_new_seq_show(struct kobject *kobj,
+                                          struct attribute *attr,
+                                          char *buf)
+{
+       struct dt_device *dt = container_of(kobj, struct dt_device,
+                                           dd_kobj);
+       struct osp_device *osp = dt2osp_dev(dt);
+
+       if (!osp->opd_pre)
+               return -EINVAL;
+
+       return scnprintf(buf, PAGE_SIZE, "%d\n", osp->opd_pre_force_new_seq);
+}
+
+static ssize_t prealloc_force_new_seq_store(struct kobject *kobj,
+                                           struct attribute *attr,
+                                           const char *buffer,
+                                           size_t count)
+{
+       struct dt_device *dt = container_of(kobj, struct dt_device,
+                                           dd_kobj);
+       struct osp_device *osp = dt2osp_dev(dt);
+       bool val;
+       int rc;
+
+       if (!osp->opd_pre)
+               return -EINVAL;
+
+       rc = kstrtobool(buffer, &val);
+       if (rc)
+               return rc;
+
+       osp->opd_pre_force_new_seq = val;
+
+       return count;
+}
+LUSTRE_RW_ATTR(prealloc_force_new_seq);
+
 /**
  * Show the number of RPCs in processing (including uncommitted by OST) plus
  * changes to sync, i.e. this is the total number of changes OST needs to apply
@@ -1147,6 +1185,7 @@ static struct attribute *osp_obd_attrs[] = {
        &lustre_attr_prealloc_next_seq.attr,
        &lustre_attr_prealloc_last_seq.attr,
        &lustre_attr_prealloc_reserved.attr,
+       &lustre_attr_prealloc_force_new_seq.attr,
        &lustre_attr_sync_in_flight.attr,
        &lustre_attr_sync_in_progress.attr,
        &lustre_attr_sync_changes.attr,

diff --git a/lustre/osp/osp_internal.h b/lustre/osp/osp_internal.h
index c1383c2..00ed7b2 100644 (file)
--- a/lustre/osp/osp_internal.h
+++ b/lustre/osp/osp_internal.h
@@ -83,9 +83,11 @@ struct osp_precreate {
        int                              osp_pre_min_create_count;
        int                              osp_pre_max_create_count;
        /* whether to increase precreation window next time or not */
-       int                              osp_pre_create_slow;
+       unsigned int                     osp_pre_create_slow:1,
        /* cleaning up orphans or recreating missing objects */
-       int                              osp_pre_recovering;
+                                        osp_pre_recovering:1,
+       /* force new seq rollover */
+                                        osp_pre_force_new_seq:1;
 };
 
 struct osp_update_request_sub {
@@ -288,6 +290,7 @@ struct osp_device {
 #define opd_pre_max_create_count       opd_pre->osp_pre_max_create_count
 #define opd_pre_create_slow            opd_pre->osp_pre_create_slow
 #define opd_pre_recovering             opd_pre->osp_pre_recovering
+#define opd_pre_force_new_seq          opd_pre->osp_pre_force_new_seq
 
 extern struct kmem_cache *osp_object_kmem;
 
@@ -589,7 +592,7 @@ static bool osp_fid_end_seq(const struct lu_env *env, struct lu_fid *fid,
        /* Skip IDIF sequence for MDT0000 */
        if (fid_is_idif(fid))
                return true;
-       if (CFS_FAIL_CHECK(OBD_FAIL_OSP_FORCE_NEW_SEQ))
+       if (osp->opd_pre_force_new_seq)
                return true;
        return fid_oid(fid) >= min(OBIF_MAX_OID, seq_width);
 }

diff --git a/lustre/tests/test-framework.sh b/lustre/tests/test-framework.sh
index 22e8615..958d5d7 100755 (executable)
--- a/lustre/tests/test-framework.sh
+++ b/lustre/tests/test-framework.sh
@@ -11477,20 +11477,32 @@ exhaust_all_precreations() {
        sleep_maxage
 }
 
+force_new_seq_ost() {
+       local dir=$1
+       local mfacet=$2
+       local OSTIDX=$3
+       local OST=$(ostname_from_index $OSTIDX)
+       local mdtosc_proc=$(get_mdtosc_proc_path $mfacet $OST)
+
+       do_facet $mfacet $LCTL set_param \
+               osp.$mdtosc_proc.prealloc_force_new_seq=1
+       # consume preallocated objects, to wake up precreate thread
+       consume_precreations $dir $mfacet $OSTIDX
+       do_facet $mfacet $LCTL set_param \
+               osp.$mdtosc_proc.prealloc_force_new_seq=0
+}
+
 force_new_seq() {
        local mfacet=$1
        local MDTIDX=$(facet_index $mfacet)
        local MDT=$(mdtname_from_index $MDTIDX $DIR)
        local i
 
-#define OBD_FAIL_OSP_FORCE_NEW_SEQ             0x210a
-       do_facet $mfacet $LCTL set_param fail_loc=0x210a
        mkdir_on_mdt -i $MDTIDX $DIR/${MDT}
        for (( i=0; i < OSTCOUNT; i++ )) ; do
-               # consume preallocated objects, to wake up precreate thread
-               consume_precreations $DIR/${MDT} $mfacet $i
+               force_new_seq_ost $DIR/${MDT} $mfacet $i &
        done
-       do_facet $mfacet $LCTL set_param fail_loc=0
+       wait
        rm -rf $DIR/${MDT}
 }