From e65102c7b41f0c6274eb937149170095ce2427e4 Mon Sep 17 00:00:00 2001 From: Li Dongyang Date: Mon, 23 Oct 2023 22:49:55 +1100 Subject: [PATCH] LU-11912 tests: fix racing in force_new_seq_all We run force_new_seq in parallel to reduce time spent on consuming precreated objects. However this could be racy when multiple MDTs are on the same MDS, a task could finish for one MDT early and reset the fail_loc to 0 on MDS while other tasks are still working on other MDTs. Replace OBD_FAIL_OSP_FORCE_NEW_SEQ with a new param prealloc_force_new_seq for osp, so we can control the seq rollover individually for each osp device. Lustre-change: https://review.whamcloud.com/52801 Lustre-commit: TBD (from af6dcd597d7f5134de553349c05091e51e0f3dd6) Change-Id: I52dbd550564ca628a8a85c42951694d58b2b93a9 Fixes: 656fc937cf ("LU-11912 tests: consume precreated objects in parallel") Signed-off-by: Li Dongyang Reviewed-on: https://review.whamcloud.com/c/ex/lustre-release/+/52802 Reviewed-by: Andreas Dilger Tested-by: jenkins Tested-by: Maloo --- lustre/include/obd_support.h | 1 - lustre/osp/lproc_osp.c | 39 +++++++++++++++++++++++++++++++++++++++ lustre/osp/osp_internal.h | 9 ++++++--- lustre/tests/test-framework.sh | 22 +++++++++++++++++----- 4 files changed, 62 insertions(+), 9 deletions(-) diff --git a/lustre/include/obd_support.h b/lustre/include/obd_support.h index 78d16f2..a4a9ac1 100644 --- a/lustre/include/obd_support.h +++ b/lustre/include/obd_support.h @@ -752,7 +752,6 @@ extern char obd_jobid_var[]; #define OBD_FAIL_OSP_INVALID_LOGID 0x2106 #define OBD_FAIL_OSP_CON_EVENT_DELAY 0x2107 #define OBD_FAIL_OSP_PRECREATE_PAUSE 0x2108 -#define OBD_FAIL_OSP_FORCE_NEW_SEQ 0x210a /* barrier */ #define OBD_FAIL_MGS_BARRIER_READ_NET 0x2200 diff --git a/lustre/osp/lproc_osp.c b/lustre/osp/lproc_osp.c index a356e96..ddb3d9d 100644 --- a/lustre/osp/lproc_osp.c +++ b/lustre/osp/lproc_osp.c @@ -654,6 +654,44 @@ static ssize_t prealloc_status_show(struct kobject *kobj, } LUSTRE_RO_ATTR(prealloc_status); +static ssize_t prealloc_force_new_seq_show(struct kobject *kobj, + struct attribute *attr, + char *buf) +{ + struct dt_device *dt = container_of(kobj, struct dt_device, + dd_kobj); + struct osp_device *osp = dt2osp_dev(dt); + + if (!osp->opd_pre) + return -EINVAL; + + return scnprintf(buf, PAGE_SIZE, "%d\n", osp->opd_pre_force_new_seq); +} + +static ssize_t prealloc_force_new_seq_store(struct kobject *kobj, + struct attribute *attr, + const char *buffer, + size_t count) +{ + struct dt_device *dt = container_of(kobj, struct dt_device, + dd_kobj); + struct osp_device *osp = dt2osp_dev(dt); + bool val; + int rc; + + if (!osp->opd_pre) + return -EINVAL; + + rc = kstrtobool(buffer, &val); + if (rc) + return rc; + + osp->opd_pre_force_new_seq = val; + + return count; +} +LUSTRE_RW_ATTR(prealloc_force_new_seq); + /** * Show the number of RPCs in processing (including uncommitted by OST) plus * changes to sync, i.e. this is the total number of changes OST needs to apply @@ -1047,6 +1085,7 @@ static struct attribute *osp_obd_attrs[] = { &lustre_attr_prealloc_next_seq.attr, &lustre_attr_prealloc_last_seq.attr, &lustre_attr_prealloc_reserved.attr, + &lustre_attr_prealloc_force_new_seq.attr, &lustre_attr_sync_in_flight.attr, &lustre_attr_sync_in_progress.attr, &lustre_attr_sync_changes.attr, diff --git a/lustre/osp/osp_internal.h b/lustre/osp/osp_internal.h index 7a7f5e4..fd0879b 100644 --- a/lustre/osp/osp_internal.h +++ b/lustre/osp/osp_internal.h @@ -84,9 +84,11 @@ struct osp_precreate { int osp_pre_min_create_count; int osp_pre_max_create_count; /* whether to increase precreation window next time or not */ - int osp_pre_create_slow; + unsigned int osp_pre_create_slow:1, /* cleaning up orphans or recreating missing objects */ - int osp_pre_recovering; + osp_pre_recovering:1, + /* force new seq rollover */ + osp_pre_force_new_seq:1; }; struct osp_update_request_sub { @@ -296,6 +298,7 @@ struct osp_device { #define opd_pre_max_create_count opd_pre->osp_pre_max_create_count #define opd_pre_create_slow opd_pre->osp_pre_create_slow #define opd_pre_recovering opd_pre->osp_pre_recovering +#define opd_pre_force_new_seq opd_pre->osp_pre_force_new_seq extern struct kmem_cache *osp_object_kmem; @@ -673,7 +676,7 @@ static bool osp_fid_end_seq(const struct lu_env *env, struct lu_fid *fid, /* Skip IDIF sequence for MDT0000 */ if (fid_is_idif(fid)) return true; - if (OBD_FAIL_CHECK(OBD_FAIL_OSP_FORCE_NEW_SEQ)) + if (osp->opd_pre_force_new_seq) return true; return fid_oid(fid) >= min(OBIF_MAX_OID, seq_width); } diff --git a/lustre/tests/test-framework.sh b/lustre/tests/test-framework.sh index bef0caa..25d5e89 100755 --- a/lustre/tests/test-framework.sh +++ b/lustre/tests/test-framework.sh @@ -11493,20 +11493,32 @@ exhaust_all_precreations() { sleep_maxage } +force_new_seq_ost() { + local dir=$1 + local mfacet=$2 + local OSTIDX=$3 + local OST=$(ostname_from_index $OSTIDX) + local mdtosc_proc=$(get_mdtosc_proc_path $mfacet $OST) + + do_facet $mfacet $LCTL set_param \ + osp.$mdtosc_proc.prealloc_force_new_seq=1 + # consume preallocated objects, to wake up precreate thread + consume_precreations $dir $mfacet $OSTIDX + do_facet $mfacet $LCTL set_param \ + osp.$mdtosc_proc.prealloc_force_new_seq=0 +} + force_new_seq() { local mfacet=$1 local MDTIDX=$(facet_index $mfacet) local MDT=$(mdtname_from_index $MDTIDX $DIR) local i -#define OBD_FAIL_OSP_FORCE_NEW_SEQ 0x210a - do_facet $mfacet $LCTL set_param fail_loc=0x210a mkdir_on_mdt -i $MDTIDX $DIR/${MDT} for (( i=0; i < OSTCOUNT; i++ )) ; do - # consume preallocated objects, to wake up precreate thread - consume_precreations $DIR/${MDT} $mfacet $i + force_new_seq_ost $DIR/${MDT} $mfacet $i & done - do_facet $mfacet $LCTL set_param fail_loc=0 + wait rm -rf $DIR/${MDT} } -- 1.8.3.1