From 887889b333aa3499277959046153d347f93bac4a Mon Sep 17 00:00:00 2001 From: Mr NeilBrown Date: Tue, 20 Feb 2024 11:57:29 +1100 Subject: [PATCH] LU-11085 tests: Add performance test for ldlm_extent code Add a new test module "ldlm_extent" which exercises the extent code by creating multiple extent locks, and discarding them. Each run is timed and a number of runs are combined to provide a mean and standard deviation. Two different tests are performed, with a ramp of locks to keep to allow seeing any scalability issues: 1/ create lots of non-overlapping extents in random order, keeping up to 8000 at a time. 2/ create both random tiny extents and whole-file extents, alternating. Keep up to 1,000,000. These are PR and so don't conflict. Each test runs for at most 5 minutes (30 loops of 10 seconds each = 300 seconds). Test-Parameters: trivial env=SLOW=yes env=ONLY=842 testlist=sanity Signed-off-by: Mr NeilBrown Change-Id: I552da3c64fb467cbefb7d25eee709dd038bd454f Reviewed-on: https://review.whamcloud.com/c/fs/lustre-release/+/54204 Reviewed-by: Andreas Dilger Reviewed-by: James Simmons Reviewed-by: Timothy Day Reviewed-by: Oleg Drokin Tested-by: jenkins Tested-by: Maloo --- lustre.spec.in | 1 + lustre/kunit/Makefile.in | 3 +- lustre/kunit/autoMakefile.am | 3 + lustre/kunit/ldlm_extent.c | 213 +++++++++++++++++++++++++++++++++++++++++++ lustre/ldlm/ldlm_internal.h | 3 + lustre/ldlm/ldlm_lock.c | 33 ++++++- lustre/ldlm/ldlm_resource.c | 5 +- lustre/tests/sanity.sh | 20 +++- rpm/kmp-lustre-tests.files | 3 + 9 files changed, 278 insertions(+), 6 deletions(-) create mode 100644 lustre/kunit/ldlm_extent.c diff --git a/lustre.spec.in b/lustre.spec.in index 8b811d2..3a99758 100644 --- a/lustre.spec.in +++ b/lustre.spec.in @@ -745,6 +745,7 @@ mkdir -p $basemodpath-tests/fs mv $basemodpath/fs/llog_test.ko $basemodpath-tests/fs/llog_test.ko mv $basemodpath/fs/obd_test.ko $basemodpath-tests/fs/obd_test.ko mv $basemodpath/fs/kinode.ko $basemodpath-tests/fs/kinode.ko +[ -f $basemodpath/fs/ldlm_extent.ko ] && mv $basemodpath/fs/ldlm_extent.ko $basemodpath-tests/fs/ldlm_extent.ko %endif %endif diff --git a/lustre/kunit/Makefile.in b/lustre/kunit/Makefile.in index 37f1339..4c938c1 100644 --- a/lustre/kunit/Makefile.in +++ b/lustre/kunit/Makefile.in @@ -9,7 +9,8 @@ # MODULES := llog_test obd_test kinode +@SERVER_TRUE@MODULES += ldlm_extent -EXTRA_DIST = llog_test.c obd_test.c kinode.c +EXTRA_DIST = llog_test.c obd_test.c kinode.c ldlm_extent.c @INCLUDE_RULES@ diff --git a/lustre/kunit/autoMakefile.am b/lustre/kunit/autoMakefile.am index a696544..466341d 100644 --- a/lustre/kunit/autoMakefile.am +++ b/lustre/kunit/autoMakefile.am @@ -12,6 +12,9 @@ if MODULES modulefs_DATA = llog_test$(KMODEXT) modulefs_DATA += obd_test$(KMODEXT) modulefs_DATA += kinode$(KMODEXT) +if SERVER +modulefs_DATA += ldlm_extent$(KMODEXT) +endif # SERVER endif # MODULES MOSTLYCLEANFILES := @MOSTLYCLEANFILES@ diff --git a/lustre/kunit/ldlm_extent.c b/lustre/kunit/ldlm_extent.c new file mode 100644 index 0000000..e13ec85 --- /dev/null +++ b/lustre/kunit/ldlm_extent.c @@ -0,0 +1,213 @@ +// SPDX-License-Identifier: GPL-2.0 + +#include +#include + +#include +#include +#include +#include +#include +#include +#include "../../ldlm/ldlm_internal.h" + +/* + * Performance tests for ldlm_extent access + */ +static int extent_setup(struct obd_device *obd, struct lustre_cfg *lcfg) +{ + return 0; +} +static int extent_cleanup(struct obd_device *obd) +{ + return 0; +} +static const struct obd_ops extent_ops = { + .o_owner = THIS_MODULE, + .o_setup = extent_setup, + .o_cleanup = extent_cleanup, +}; + +static struct ldlm_res_id RES_ID = { + .name = {1, 2, 3, 4}, +}; + +static void test_one(struct ldlm_resource *res, + u64 pos, u64 len, enum ldlm_mode mode, + int *cnt, int max, struct list_head *list) +{ + __u64 flags = 0; + enum ldlm_error err; + ldlm_processing_policy pol; + struct ldlm_lock *lock = ldlm_lock_new_testing(res); + //struct ldlm_lock *lock = ldlm_lock_create(ns, &RES_ID, + // LDLM_EXTENT, LCK_EX, + // NULL, NULL, 0, LVB_T_NONE); + + refcount_inc(&res->lr_refcount); + + lock->l_req_mode = mode; + + pol = ldlm_get_processing_policy(res); + + lock->l_policy_data.l_extent.start = pos; + lock->l_policy_data.l_extent.end = pos + len; + lock->l_policy_data.l_extent.gid = 0; + lock->l_req_extent = lock->l_policy_data.l_extent; + + lock_res(res); + + if (pol(lock, &flags, LDLM_PROCESS_ENQUEUE, &err, NULL) + == LDLM_ITER_CONTINUE) { + list_add_tail(&lock->l_lru, list); + unlock_res(res); + *cnt += 1; + } else { + unlock_res(res); + ldlm_lock_cancel(lock); + ldlm_lock_put(lock); + } + + if (*cnt > max) { + lock = list_first_entry_or_null(list, + struct ldlm_lock, l_lru); + if (lock) { + list_del_init(&lock->l_lru); + ldlm_lock_cancel(lock); + ldlm_lock_put(lock); + *cnt -= 1; + } + } +} + +enum tests { + TEST_NO_OVERLAP, + TEST_WHOLE_FILE, + + NUM_TESTS, +}; + +static int ldlm_extent_init(void) +{ + struct lustre_cfg *cfg; + struct lustre_cfg_bufs bufs; + char *name, *uuid; + struct ldlm_resource *res; + struct obd_device *obd; + struct ldlm_namespace *ns; + enum tests tnum; + struct rnd_state rstate; + + prandom_seed_state(&rstate, 42); + + class_register_type(&extent_ops, NULL, false, "ldlm_test", NULL); + + OBD_ALLOC(name, MAX_OBD_NAME); + OBD_ALLOC(uuid, MAX_OBD_NAME); + strscpy(name, "test", MAX_OBD_NAME); + lustre_cfg_bufs_reset(&bufs, name); + snprintf(uuid, MAX_OBD_NAME, "%s_UUID", name); + + lustre_cfg_bufs_set_string(&bufs, 1, "ldlm_test"); /* typename */ + lustre_cfg_bufs_set_string(&bufs, 2, uuid); + OBD_ALLOC(cfg, lustre_cfg_len(bufs.lcfg_bufcount, bufs.lcfg_buflen)); + lustre_cfg_init(cfg, LCFG_ATTACH, &bufs); + + class_attach(cfg); + obd = class_name2obd("test"); + ns = ldlm_namespace_new(obd, "extent-test", LDLM_NAMESPACE_CLIENT, + LDLM_NAMESPACE_MODEST, + LDLM_NS_TYPE_MDT); + res = ldlm_resource_get(ns, &RES_ID, LDLM_EXTENT, 1); + + pr_info("ldlm_extent: sizeof(struct ldlm_lock)=%lu\n", + sizeof(struct ldlm_lock)); + for (tnum = 0; tnum < NUM_TESTS; tnum++) { + long sum = 0, sumsq = 0, ns; + int min_iters = 10; + int loops; + + pr_info("ldlm_extent: start test %d\n", tnum); + for (loops = 0; loops < 30 ; loops++) { + struct ldlm_lock *lock; + ktime_t start, now; + LIST_HEAD(list); + int cnt = 0; + int max = 1; + int i; + + start = now = ktime_get(); + for (i = 0; i < 10000000; i++) { + switch (tnum) { + case TEST_NO_OVERLAP: + max = min(8000, min_iters); + if (i < max * 16 / 15) + max = i * 15 / 16; + test_one(res, + prandom_u32_state(&rstate), 1, + LCK_EX, &cnt, max, &list); + break; + case TEST_WHOLE_FILE: + max = min(1000000, min_iters); + if (i < max * 16 / 15) + max = i * 15 / 16; + test_one(res, 0, OBD_OBJECT_EOF, LCK_PR, + &cnt, max, &list); + test_one(res, + prandom_u32_state(&rstate), 1, + LCK_PR, &cnt, max, &list); + break; + case NUM_TESTS: + break; + } + now = ktime_get(); + if (ktime_to_ms(ktime_sub(now, start)) > 10000) + break; + cond_resched(); + } + i++; + if (i < min_iters / 3) + min_iters = i; + else if (i > min_iters * 3) + min_iters *= 10; + ns = ktime_to_ns(ktime_sub(now, start)) / i; + sum += ns; + sumsq += ns * ns; + pr_info("ldlm_extent: test %d loop=%d min_iters=%d iters=%d ns/iter=%lu\n", + tnum, loops, min_iters, i, ns); + + while ((lock = list_first_entry_or_null(&list, + struct ldlm_lock, l_lru)) + != NULL) { + list_del_init(&lock->l_lru); + ldlm_lock_cancel(lock); + ldlm_lock_put(lock); + } + } + + pr_info("ldlm_extent: test %d ended - loops=%d min_iters=%d mean=%ld stddev=%ld\n", + tnum, loops, min_iters, sum / loops, + int_sqrt((sumsq - sum*sum/loops) / loops-1)); + } + class_detach(obd, cfg); + + OBD_FREE(name, MAX_OBD_NAME); + OBD_FREE(uuid, MAX_OBD_NAME); + OBD_FREE(cfg, lustre_cfg_len(bufs.lcfg_bufcount, bufs.lcfg_buflen)); + + ldlm_resource_putref(res); + ldlm_namespace_free_post(ns); + class_unregister_type("ldlm_test"); + + return 0; +} + +static void ldlm_extent_exit(void) +{ +} + +MODULE_DESCRIPTION("Lustre ldlm_extent performance test"); +MODULE_LICENSE("GPL"); + +module_init(ldlm_extent_init); +module_exit(ldlm_extent_exit); diff --git a/lustre/ldlm/ldlm_internal.h b/lustre/ldlm/ldlm_internal.h index 1fdb119..1234f72 100644 --- a/lustre/ldlm/ldlm_internal.h +++ b/lustre/ldlm/ldlm_internal.h @@ -426,3 +426,6 @@ static inline bool ldlm_res_eq(const struct ldlm_res_id *res0, { return memcmp(res0, res1, sizeof(*res0)) == 0; } + +/* exports for testing */ +struct ldlm_lock *ldlm_lock_new_testing(struct ldlm_resource *resource); diff --git a/lustre/ldlm/ldlm_lock.c b/lustre/ldlm/ldlm_lock.c index 4d6b77a..c41a2f1 100644 --- a/lustre/ldlm/ldlm_lock.c +++ b/lustre/ldlm/ldlm_lock.c @@ -241,7 +241,11 @@ void ldlm_lock_put(struct ldlm_lock *lock) ldlm_resource_putref(res); lock->l_resource = NULL; lu_ref_fini(&lock->l_reference); - call_rcu(&lock->l_handle.h_rcu, lock_handle_free); + if (lock->l_flags & BIT(63)) + /* Performance testing - bypassing RCU removes overhead */ + lock_handle_free(&lock->l_handle.h_rcu); + else + call_rcu(&lock->l_handle.h_rcu, lock_handle_free); } EXIT; @@ -497,6 +501,33 @@ static struct ldlm_lock *ldlm_lock_new(struct ldlm_resource *resource) RETURN(lock); } +struct ldlm_lock *ldlm_lock_new_testing(struct ldlm_resource *resource) +{ + struct ldlm_lock *lock = ldlm_lock_new(resource); + int rc; + + if (!lock) + return NULL; + lock->l_flags |= BIT(63); + switch (resource->lr_type) { + case LDLM_EXTENT: + rc = ldlm_extent_alloc_lock(lock); + break; + case LDLM_IBITS: + rc = ldlm_inodebits_alloc_lock(lock); + break; + default: + rc = 0; + } + + if (!rc) + return lock; + ldlm_lock_destroy(lock); + LDLM_LOCK_RELEASE(lock); + return NULL; +} +EXPORT_SYMBOL(ldlm_lock_new_testing); + /** * Moves LDLM lock \a lock to another resource. * This is used on client when server returns some other lock than requested diff --git a/lustre/ldlm/ldlm_resource.c b/lustre/ldlm/ldlm_resource.c index 3ee5f81..26cf77d 100644 --- a/lustre/ldlm/ldlm_resource.c +++ b/lustre/ldlm/ldlm_resource.c @@ -1609,10 +1609,13 @@ static void __ldlm_resource_putref_final(struct cfs_hash_bd *bd, /* Returns 1 if the resource was freed, 0 if it remains. */ int ldlm_resource_putref(struct ldlm_resource *res) { - struct ldlm_namespace *ns = ldlm_res_to_ns(res); + struct ldlm_namespace *ns; struct cfs_hash_bd bd; int refcount; + if (refcount_dec_not_one(&res->lr_refcount)) + return 0; + ns = ldlm_res_to_ns(res); refcount = refcount_read(&res->lr_refcount); LASSERT(refcount < LI_POISON); diff --git a/lustre/tests/sanity.sh b/lustre/tests/sanity.sh index ab02e17..c9f1ee5 100755 --- a/lustre/tests/sanity.sh +++ b/lustre/tests/sanity.sh @@ -67,8 +67,8 @@ if (( $LINUX_VERSION_CODE < $(version_code 4.18.0) )); then always_except LU-13063 411b fi -# 5 12 8 12 15 (min)" -[[ "$SLOW" = "no" ]] && EXCEPT_SLOW="27m 60i 64b 68 71 135 136 230d 300o" +# minutes runtime: 5 12 8 12 15 10 +[[ "$SLOW" = "no" ]] && EXCEPT_SLOW="27m 60i 64b 68 71 135 136 230d 300o 842" if [[ "$mds1_FSTYPE" == "zfs" ]]; then # 13 (min)" @@ -32007,6 +32007,21 @@ test_833() { } run_test 833 "Mixed buffered/direct read and write should not return -EIO" +test_842() { + local oss1=$(facet_host ost1) + + # Try to insert the module. This will leave results in dmesg + now=$(date +%s) + log "STAMP $now" > /dev/kmsg + do_rpc_nodes $oss1 load_module kunit/ldlm_extent || + error "$oss1 load_module ldlm_extent failed" + + do_node $oss1 dmesg | sed -n -e "1,/STAMP $now/d" -e '/ldlm_extent:/p' + do_node $oss1 rmmod -v ldlm_extent || + error "rmmod failed (may trigger a failure in a later test)" +} +run_test 842 "Measure ldlm_extent performance" + test_850() { local dir=$DIR/$tdir local file=$dir/$tfile @@ -32296,7 +32311,6 @@ test_907() { } run_test 907 "write rpc error during unlink" - complete_test $SECONDS [ -f $EXT2_DEV ] && rm $EXT2_DEV || true check_and_cleanup_lustre diff --git a/rpm/kmp-lustre-tests.files b/rpm/kmp-lustre-tests.files index 9deabbc..a6a27d2 100644 --- a/rpm/kmp-lustre-tests.files +++ b/rpm/kmp-lustre-tests.files @@ -3,3 +3,6 @@ %{modules_fs_path}/%{lustre_name}-tests/fs/llog_test.ko %{modules_fs_path}/%{lustre_name}-tests/fs/obd_test.ko %{modules_fs_path}/%{lustre_name}-tests/fs/kinode.ko +%if %{with servers} +%{modules_fs_path}/%{lustre_name}-tests/fs/ldlm_extent.ko +%endif -- 1.8.3.1