Whamcloud - gitweb
LU-11085 tests: Add performance test for ldlm_extent code 04/54204/17
authorMr NeilBrown <neilb@suse.de>
Tue, 20 Feb 2024 00:57:29 +0000 (11:57 +1100)
committerOleg Drokin <green@whamcloud.com>
Sat, 23 Mar 2024 05:55:57 +0000 (05:55 +0000)
Add a new test module "ldlm_extent" which exercises the extent code by
creating multiple extent locks, and discarding them.
Each run is timed and a number of runs are combined to provide a
mean and standard deviation.

Two different tests are performed, with a ramp of locks to keep to
allow seeing any scalability issues:
1/ create lots of non-overlapping extents in
   random order, keeping up to 8000 at a time.
2/ create both random tiny extents and whole-file
   extents, alternating.  Keep up to 1,000,000.
   These are PR and so don't conflict.

Each test runs for at most 5 minutes
(30 loops of 10 seconds each = 300 seconds).

Test-Parameters: trivial env=SLOW=yes env=ONLY=842 testlist=sanity
Signed-off-by: Mr NeilBrown <neilb@suse.de>
Change-Id: I552da3c64fb467cbefb7d25eee709dd038bd454f
Reviewed-on: https://review.whamcloud.com/c/fs/lustre-release/+/54204
Reviewed-by: Andreas Dilger <adilger@whamcloud.com>
Reviewed-by: James Simmons <jsimmons@infradead.org>
Reviewed-by: Timothy Day <timday@amazon.com>
Reviewed-by: Oleg Drokin <green@whamcloud.com>
Tested-by: jenkins <devops@whamcloud.com>
Tested-by: Maloo <maloo@whamcloud.com>
lustre.spec.in
lustre/kunit/Makefile.in
lustre/kunit/autoMakefile.am
lustre/kunit/ldlm_extent.c [new file with mode: 0644]
lustre/ldlm/ldlm_internal.h
lustre/ldlm/ldlm_lock.c
lustre/ldlm/ldlm_resource.c
lustre/tests/sanity.sh
rpm/kmp-lustre-tests.files

index 8b811d2..3a99758 100644 (file)
@@ -745,6 +745,7 @@ mkdir -p $basemodpath-tests/fs
 mv $basemodpath/fs/llog_test.ko $basemodpath-tests/fs/llog_test.ko
 mv $basemodpath/fs/obd_test.ko $basemodpath-tests/fs/obd_test.ko
 mv $basemodpath/fs/kinode.ko $basemodpath-tests/fs/kinode.ko
+[ -f $basemodpath/fs/ldlm_extent.ko ] && mv $basemodpath/fs/ldlm_extent.ko $basemodpath-tests/fs/ldlm_extent.ko
 %endif
 %endif
 
index 37f1339..4c938c1 100644 (file)
@@ -9,7 +9,8 @@
 #
 
 MODULES := llog_test obd_test kinode
+@SERVER_TRUE@MODULES += ldlm_extent
 
-EXTRA_DIST = llog_test.c obd_test.c kinode.c
+EXTRA_DIST = llog_test.c obd_test.c kinode.c ldlm_extent.c
 
 @INCLUDE_RULES@
index a696544..466341d 100644 (file)
@@ -12,6 +12,9 @@ if MODULES
 modulefs_DATA = llog_test$(KMODEXT)
 modulefs_DATA += obd_test$(KMODEXT)
 modulefs_DATA += kinode$(KMODEXT)
+if SERVER
+modulefs_DATA += ldlm_extent$(KMODEXT)
+endif # SERVER
 endif # MODULES
 
 MOSTLYCLEANFILES := @MOSTLYCLEANFILES@
diff --git a/lustre/kunit/ldlm_extent.c b/lustre/kunit/ldlm_extent.c
new file mode 100644 (file)
index 0000000..e13ec85
--- /dev/null
@@ -0,0 +1,213 @@
+// SPDX-License-Identifier: GPL-2.0
+
+#include <linux/module.h>
+#include <linux/kernel.h>
+
+#include <libcfs/libcfs.h>
+#include <lustre_dlm.h>
+#include <obd_support.h>
+#include <obd.h>
+#include <obd_class.h>
+#include <lustre_lib.h>
+#include "../../ldlm/ldlm_internal.h"
+
+/*
+ * Performance tests for ldlm_extent access
+ */
+static int extent_setup(struct obd_device *obd, struct lustre_cfg *lcfg)
+{
+       return 0;
+}
+static int extent_cleanup(struct obd_device *obd)
+{
+       return 0;
+}
+static const struct obd_ops extent_ops = {
+       .o_owner       = THIS_MODULE,
+       .o_setup       = extent_setup,
+       .o_cleanup     = extent_cleanup,
+};
+
+static struct ldlm_res_id RES_ID = {
+       .name = {1, 2, 3, 4},
+};
+
+static void test_one(struct ldlm_resource *res,
+                    u64 pos, u64 len, enum ldlm_mode mode,
+                    int *cnt, int max, struct list_head *list)
+{
+       __u64 flags = 0;
+       enum ldlm_error err;
+       ldlm_processing_policy pol;
+       struct ldlm_lock *lock = ldlm_lock_new_testing(res);
+       //struct ldlm_lock *lock = ldlm_lock_create(ns, &RES_ID,
+       //                                        LDLM_EXTENT, LCK_EX,
+       //                                        NULL, NULL, 0, LVB_T_NONE);
+
+       refcount_inc(&res->lr_refcount);
+
+       lock->l_req_mode = mode;
+
+       pol = ldlm_get_processing_policy(res);
+
+       lock->l_policy_data.l_extent.start = pos;
+       lock->l_policy_data.l_extent.end = pos + len;
+       lock->l_policy_data.l_extent.gid = 0;
+       lock->l_req_extent = lock->l_policy_data.l_extent;
+
+       lock_res(res);
+
+       if (pol(lock, &flags, LDLM_PROCESS_ENQUEUE, &err, NULL)
+           == LDLM_ITER_CONTINUE) {
+               list_add_tail(&lock->l_lru, list);
+               unlock_res(res);
+               *cnt += 1;
+       } else {
+               unlock_res(res);
+               ldlm_lock_cancel(lock);
+               ldlm_lock_put(lock);
+       }
+
+       if (*cnt > max) {
+               lock = list_first_entry_or_null(list,
+                                               struct ldlm_lock, l_lru);
+               if (lock) {
+                       list_del_init(&lock->l_lru);
+                       ldlm_lock_cancel(lock);
+                       ldlm_lock_put(lock);
+                       *cnt -= 1;
+               }
+       }
+}
+
+enum tests {
+       TEST_NO_OVERLAP,
+       TEST_WHOLE_FILE,
+
+       NUM_TESTS,
+};
+
+static int ldlm_extent_init(void)
+{
+       struct lustre_cfg *cfg;
+       struct lustre_cfg_bufs bufs;
+       char *name, *uuid;
+       struct ldlm_resource *res;
+       struct obd_device *obd;
+       struct ldlm_namespace *ns;
+       enum tests tnum;
+       struct rnd_state rstate;
+
+       prandom_seed_state(&rstate, 42);
+
+       class_register_type(&extent_ops, NULL, false, "ldlm_test", NULL);
+
+       OBD_ALLOC(name, MAX_OBD_NAME);
+       OBD_ALLOC(uuid, MAX_OBD_NAME);
+       strscpy(name, "test", MAX_OBD_NAME);
+       lustre_cfg_bufs_reset(&bufs, name);
+       snprintf(uuid, MAX_OBD_NAME, "%s_UUID", name);
+
+       lustre_cfg_bufs_set_string(&bufs, 1, "ldlm_test"); /* typename */
+       lustre_cfg_bufs_set_string(&bufs, 2, uuid);
+       OBD_ALLOC(cfg, lustre_cfg_len(bufs.lcfg_bufcount, bufs.lcfg_buflen));
+       lustre_cfg_init(cfg, LCFG_ATTACH, &bufs);
+
+       class_attach(cfg);
+       obd = class_name2obd("test");
+       ns = ldlm_namespace_new(obd, "extent-test", LDLM_NAMESPACE_CLIENT,
+                               LDLM_NAMESPACE_MODEST,
+                               LDLM_NS_TYPE_MDT);
+       res = ldlm_resource_get(ns, &RES_ID, LDLM_EXTENT, 1);
+
+       pr_info("ldlm_extent: sizeof(struct ldlm_lock)=%lu\n",
+              sizeof(struct ldlm_lock));
+       for (tnum = 0; tnum < NUM_TESTS; tnum++) {
+               long sum = 0, sumsq = 0, ns;
+               int min_iters = 10;
+               int loops;
+
+               pr_info("ldlm_extent: start test %d\n", tnum);
+               for (loops = 0; loops < 30 ; loops++) {
+                       struct ldlm_lock *lock;
+                       ktime_t start, now;
+                       LIST_HEAD(list);
+                       int cnt = 0;
+                       int max = 1;
+                       int i;
+
+                       start = now = ktime_get();
+                       for (i = 0; i < 10000000; i++) {
+                               switch (tnum) {
+                               case TEST_NO_OVERLAP:
+                                       max = min(8000, min_iters);
+                                       if (i < max * 16 / 15)
+                                               max = i * 15 / 16;
+                                       test_one(res,
+                                                prandom_u32_state(&rstate), 1,
+                                                LCK_EX, &cnt, max, &list);
+                                       break;
+                               case TEST_WHOLE_FILE:
+                                       max = min(1000000, min_iters);
+                                       if (i < max * 16 / 15)
+                                               max = i * 15 / 16;
+                                       test_one(res, 0, OBD_OBJECT_EOF, LCK_PR,
+                                                &cnt, max, &list);
+                                       test_one(res,
+                                                prandom_u32_state(&rstate), 1,
+                                                LCK_PR, &cnt, max, &list);
+                                       break;
+                               case NUM_TESTS:
+                                       break;
+                               }
+                               now = ktime_get();
+                               if (ktime_to_ms(ktime_sub(now, start)) > 10000)
+                                       break;
+                               cond_resched();
+                       }
+                       i++;
+                       if (i < min_iters / 3)
+                               min_iters = i;
+                       else if (i > min_iters * 3)
+                               min_iters *= 10;
+                       ns = ktime_to_ns(ktime_sub(now, start)) / i;
+                       sum += ns;
+                       sumsq += ns * ns;
+                       pr_info("ldlm_extent: test %d loop=%d min_iters=%d iters=%d ns/iter=%lu\n",
+                               tnum, loops, min_iters, i, ns);
+
+                       while ((lock = list_first_entry_or_null(&list,
+                                               struct ldlm_lock, l_lru))
+                              != NULL) {
+                               list_del_init(&lock->l_lru);
+                               ldlm_lock_cancel(lock);
+                               ldlm_lock_put(lock);
+                       }
+               }
+
+               pr_info("ldlm_extent: test %d ended - loops=%d min_iters=%d mean=%ld stddev=%ld\n",
+                      tnum, loops, min_iters, sum / loops,
+                      int_sqrt((sumsq - sum*sum/loops) / loops-1));
+       }
+       class_detach(obd, cfg);
+
+       OBD_FREE(name, MAX_OBD_NAME);
+       OBD_FREE(uuid, MAX_OBD_NAME);
+       OBD_FREE(cfg, lustre_cfg_len(bufs.lcfg_bufcount, bufs.lcfg_buflen));
+
+       ldlm_resource_putref(res);
+       ldlm_namespace_free_post(ns);
+       class_unregister_type("ldlm_test");
+
+       return 0;
+}
+
+static void ldlm_extent_exit(void)
+{
+}
+
+MODULE_DESCRIPTION("Lustre ldlm_extent performance test");
+MODULE_LICENSE("GPL");
+
+module_init(ldlm_extent_init);
+module_exit(ldlm_extent_exit);
index 1fdb119..1234f72 100644 (file)
@@ -426,3 +426,6 @@ static inline bool ldlm_res_eq(const struct ldlm_res_id *res0,
 {
        return memcmp(res0, res1, sizeof(*res0)) == 0;
 }
+
+/* exports for testing */
+struct ldlm_lock *ldlm_lock_new_testing(struct ldlm_resource *resource);
index 4d6b77a..c41a2f1 100644 (file)
@@ -241,7 +241,11 @@ void ldlm_lock_put(struct ldlm_lock *lock)
                ldlm_resource_putref(res);
                lock->l_resource = NULL;
                lu_ref_fini(&lock->l_reference);
-               call_rcu(&lock->l_handle.h_rcu, lock_handle_free);
+               if (lock->l_flags & BIT(63))
+                       /* Performance testing - bypassing RCU removes overhead */
+                       lock_handle_free(&lock->l_handle.h_rcu);
+               else
+                       call_rcu(&lock->l_handle.h_rcu, lock_handle_free);
        }
 
        EXIT;
@@ -497,6 +501,33 @@ static struct ldlm_lock *ldlm_lock_new(struct ldlm_resource *resource)
        RETURN(lock);
 }
 
+struct ldlm_lock *ldlm_lock_new_testing(struct ldlm_resource *resource)
+{
+       struct ldlm_lock *lock = ldlm_lock_new(resource);
+       int rc;
+
+       if (!lock)
+               return NULL;
+       lock->l_flags |= BIT(63);
+       switch (resource->lr_type) {
+       case LDLM_EXTENT:
+               rc = ldlm_extent_alloc_lock(lock);
+               break;
+       case LDLM_IBITS:
+               rc = ldlm_inodebits_alloc_lock(lock);
+               break;
+       default:
+               rc = 0;
+       }
+
+       if (!rc)
+               return lock;
+       ldlm_lock_destroy(lock);
+       LDLM_LOCK_RELEASE(lock);
+       return NULL;
+}
+EXPORT_SYMBOL(ldlm_lock_new_testing);
+
 /**
  * Moves LDLM lock \a lock to another resource.
  * This is used on client when server returns some other lock than requested
index 3ee5f81..26cf77d 100644 (file)
@@ -1609,10 +1609,13 @@ static void __ldlm_resource_putref_final(struct cfs_hash_bd *bd,
 /* Returns 1 if the resource was freed, 0 if it remains. */
 int ldlm_resource_putref(struct ldlm_resource *res)
 {
-       struct ldlm_namespace *ns = ldlm_res_to_ns(res);
+       struct ldlm_namespace *ns;
        struct cfs_hash_bd bd;
        int refcount;
 
+       if (refcount_dec_not_one(&res->lr_refcount))
+               return 0;
+       ns = ldlm_res_to_ns(res);
        refcount = refcount_read(&res->lr_refcount);
        LASSERT(refcount < LI_POISON);
 
index ab02e17..c9f1ee5 100755 (executable)
@@ -67,8 +67,8 @@ if (( $LINUX_VERSION_CODE < $(version_code 4.18.0) )); then
        always_except LU-13063 411b
 fi
 
-#                                  5              12     8   12  15   (min)"
-[[ "$SLOW" = "no" ]] && EXCEPT_SLOW="27m 60i 64b 68 71 135 136 230d 300o"
+# minutes runtime:                   5              12     8   12   15   10
+[[ "$SLOW" = "no" ]] && EXCEPT_SLOW="27m 60i 64b 68 71 135 136 230d 300o 842"
 
 if [[ "$mds1_FSTYPE" == "zfs" ]]; then
        #                                               13    (min)"
@@ -32007,6 +32007,21 @@ test_833() {
 }
 run_test 833 "Mixed buffered/direct read and write should not return -EIO"
 
+test_842() {
+       local oss1=$(facet_host ost1)
+
+       # Try to insert the module.  This will leave results in dmesg
+       now=$(date +%s)
+       log "STAMP $now" > /dev/kmsg
+       do_rpc_nodes $oss1 load_module kunit/ldlm_extent ||
+               error "$oss1 load_module ldlm_extent failed"
+
+       do_node $oss1 dmesg | sed -n -e "1,/STAMP $now/d" -e '/ldlm_extent:/p'
+       do_node $oss1 rmmod -v ldlm_extent ||
+               error "rmmod failed (may trigger a failure in a later test)"
+}
+run_test 842 "Measure ldlm_extent performance"
+
 test_850() {
        local dir=$DIR/$tdir
        local file=$dir/$tfile
@@ -32296,7 +32311,6 @@ test_907() {
 }
 run_test 907 "write rpc error during unlink"
 
-
 complete_test $SECONDS
 [ -f $EXT2_DEV ] && rm $EXT2_DEV || true
 check_and_cleanup_lustre
index 9deabbc..a6a27d2 100644 (file)
@@ -3,3 +3,6 @@
 %{modules_fs_path}/%{lustre_name}-tests/fs/llog_test.ko
 %{modules_fs_path}/%{lustre_name}-tests/fs/obd_test.ko
 %{modules_fs_path}/%{lustre_name}-tests/fs/kinode.ko
+%if %{with servers}
+%{modules_fs_path}/%{lustre_name}-tests/fs/ldlm_extent.ko
+%endif