From f9217c074f1099510dc0fa11d0cbf8dcab198776 Mon Sep 17 00:00:00 2001 From: Mr NeilBrown Date: Fri, 25 Oct 2024 11:57:43 -0700 Subject: [PATCH] LU-9859 libcfs: move percpt_lock into lnet lnet is the only users of percpt_lock - and there are only two such locks! So move the code into lnet, as part of deprecating libcfs. Lustre-change: https://review.whamcloud.com/50832 Lustre-commit: c4e2563ff3bfa84ab7558c2aced32445da543ef6 Test-Parameters: trivial Signed-off-by: Mr NeilBrown Change-Id: Id7091e88cf61228aa031921747fb9c7b08214931 Reviewed-on: https://review.whamcloud.com/c/fs/lustre-release/+/56793 Tested-by: jenkins Tested-by: Maloo Reviewed-by: Yang Sheng Reviewed-by: James Simmons Reviewed-by: Oleg Drokin --- libcfs/include/libcfs/libcfs_cpu.h | 62 -------------------- libcfs/libcfs/Makefile.in | 2 +- libcfs/libcfs/autoMakefile.am | 3 +- lnet/include/lnet/Makefile.am | 1 + lnet/include/lnet/lib-lnet.h | 3 +- lnet/include/lnet/lock.h | 77 +++++++++++++++++++++++++ lnet/lnet/Makefile.in | 2 +- libcfs/libcfs/libcfs_lock.c => lnet/lnet/lock.c | 16 ++--- 8 files changed, 88 insertions(+), 78 deletions(-) create mode 100644 lnet/include/lnet/lock.h rename libcfs/libcfs/libcfs_lock.c => lnet/lnet/lock.c (87%) diff --git a/libcfs/include/libcfs/libcfs_cpu.h b/libcfs/include/libcfs/libcfs_cpu.h index cb2539e..52e3b07 100644 --- a/libcfs/include/libcfs/libcfs_cpu.h +++ b/libcfs/include/libcfs/libcfs_cpu.h @@ -337,68 +337,6 @@ int cfs_percpt_number(void *vars); for (i = 0; i < cfs_percpt_number(vars) && \ ((var) = (vars)[i]) != NULL; i++) -/* - * percpu partition lock - * - * There are some use-cases like this in Lustre: - * . each CPU partition has it's own private data which is frequently changed, - * and mostly by the local CPU partition. - * . all CPU partitions share some global data, these data are rarely changed. - * - * LNet is typical example. - * CPU partition lock is designed for this kind of use-cases: - * . each CPU partition has it's own private lock - * . change on private data just needs to take the private lock - * . read on shared data just needs to take _any_ of private locks - * . change on shared data needs to take _all_ private locks, - * which is slow and should be really rare. - */ -enum { - CFS_PERCPT_LOCK_EX = -1, /* negative */ -}; - -struct cfs_percpt_lock { - /* cpu-partition-table for this lock */ - struct cfs_cpt_table *pcl_cptab; - /* exclusively locked */ - unsigned int pcl_locked; - /* private lock table */ - spinlock_t **pcl_locks; -}; - -/* return number of private locks */ -#define cfs_percpt_lock_num(pcl) cfs_cpt_number(pcl->pcl_cptab) - -/* - * create a cpu-partition lock based on CPU partition table \a cptab, - * each private lock has extra \a psize bytes padding data - */ -struct cfs_percpt_lock *cfs_percpt_lock_create(struct cfs_cpt_table *cptab, - struct lock_class_key *keys); -/* destroy a cpu-partition lock */ -void cfs_percpt_lock_free(struct cfs_percpt_lock *pcl); - -/* lock private lock \a index of \a pcl */ -void cfs_percpt_lock(struct cfs_percpt_lock *pcl, int index); - -/* unlock private lock \a index of \a pcl */ -void cfs_percpt_unlock(struct cfs_percpt_lock *pcl, int index); - -#define CFS_PERCPT_LOCK_KEYS 256 - -/* NB: don't allocate keys dynamically, lockdep needs them to be in ".data" */ -#define cfs_percpt_lock_alloc(cptab) \ -({ \ - static struct lock_class_key ___keys[CFS_PERCPT_LOCK_KEYS]; \ - struct cfs_percpt_lock *___lk; \ - \ - if (cfs_cpt_number(cptab) > CFS_PERCPT_LOCK_KEYS) \ - ___lk = cfs_percpt_lock_create(cptab, NULL); \ - else \ - ___lk = cfs_percpt_lock_create(cptab, ___keys); \ - ___lk; \ -}) - /** * allocate \a nr_bytes of physical memory from a contiguous region with the * properties of \a flags which are bound to the partition id \a cpt. This diff --git a/libcfs/libcfs/Makefile.in b/libcfs/libcfs/Makefile.in index 46ee69c..e99cef2 100644 --- a/libcfs/libcfs/Makefile.in +++ b/libcfs/libcfs/Makefile.in @@ -18,7 +18,7 @@ libcfs-objs-$(CONFIG_SMP) = libcfs_cpu.o libcfs-all-objs := debug.o fail.o module.o tracefile.o \ libcfs_string.o hash.o \ workitem.o \ - libcfs_mem.o libcfs_lock.o \ + libcfs_mem.o \ linux-crypto.o linux-crypto-adler.o \ $(libcfs-objs-y) diff --git a/libcfs/libcfs/autoMakefile.am b/libcfs/libcfs/autoMakefile.am index 75ee515..c35b24e 100644 --- a/libcfs/libcfs/autoMakefile.am +++ b/libcfs/libcfs/autoMakefile.am @@ -59,5 +59,4 @@ endif # MODULES MOSTLYCLEANFILES := @MOSTLYCLEANFILES@ linux/*.o libcfs crypto/*.o EXTRA_DIST := $(libcfs-all-objs:%.o=%.c) tracefile.h \ workitem.c fail.c libcfs_cpu.c \ - libcfs_mem.c libcfs_lock.c \ - linux-crypto.h + libcfs_mem.c linux-crypto.h diff --git a/lnet/include/lnet/Makefile.am b/lnet/include/lnet/Makefile.am index 3f2294f..189144a 100644 --- a/lnet/include/lnet/Makefile.am +++ b/lnet/include/lnet/Makefile.am @@ -4,4 +4,5 @@ EXTRA_DIST = \ lib-types.h \ udsp.h \ lnet_rdma.h \ + lock.h \ socklnd.h diff --git a/lnet/include/lnet/lib-lnet.h b/lnet/include/lnet/lib-lnet.h index fb7fffd..904f841 100644 --- a/lnet/include/lnet/lib-lnet.h +++ b/lnet/include/lnet/lib-lnet.h @@ -50,6 +50,8 @@ #include #include +#include "lock.h" + extern struct lnet the_lnet; /* THE network */ #if (BITS_PER_LONG == 32) @@ -1230,7 +1232,6 @@ __u32 lnet_sum_stats(struct lnet_element_stats *stats, void lnet_usr_translate_stats(struct lnet_ioctl_element_msg_stats *msg_stats, struct lnet_element_stats *stats); - static inline void lnet_set_route_aliveness(struct lnet_route *route, bool alive) { diff --git a/lnet/include/lnet/lock.h b/lnet/include/lnet/lock.h new file mode 100644 index 0000000..5364d2f --- /dev/null +++ b/lnet/include/lnet/lock.h @@ -0,0 +1,77 @@ +/* + * GPL HEADER START + * + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License version 2 only, + * as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but + * WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * General Public License version 2 for more details (a copy is included + * in the LICENSE file that accompanied this code). + * + * GPL HEADER END + */ + +/* percpu partition lock + * + * There are some use-cases like this in Lustre: + * . each CPU partition has it's own private data which is frequently changed, + * and mostly by the local CPU partition. + * . all CPU partitions share some global data, these data are rarely changed. + * + * LNet is typical example. + * CPU partition lock is designed for this kind of use-cases: + * . each CPU partition has it's own private lock + * . change on private data just needs to take the private lock + * . read on shared data just needs to take _any_ of private locks + * . change on shared data needs to take _all_ private locks, + * which is slow and should be really rare. + */ +enum { + CFS_PERCPT_LOCK_EX = -1, /* negative */ +}; + +struct cfs_percpt_lock { + /* cpu-partition-table for this lock */ + struct cfs_cpt_table *pcl_cptab; + /* exclusively locked */ + unsigned int pcl_locked; + /* private lock table */ + spinlock_t **pcl_locks; +}; + +/* return number of private locks */ +#define cfs_percpt_lock_num(pcl) cfs_cpt_number(pcl->pcl_cptab) + +/* create a cpu-partition lock based on CPU partition table \a cptab, + * each private lock has extra \a psize bytes padding data + */ +struct cfs_percpt_lock *cfs_percpt_lock_create(struct cfs_cpt_table *cptab, + struct lock_class_key *keys); +/* destroy a cpu-partition lock */ +void cfs_percpt_lock_free(struct cfs_percpt_lock *pcl); + +/* lock private lock \a index of \a pcl */ +void cfs_percpt_lock(struct cfs_percpt_lock *pcl, int index); + +/* unlock private lock \a index of \a pcl */ +void cfs_percpt_unlock(struct cfs_percpt_lock *pcl, int index); + +#define CFS_PERCPT_LOCK_KEYS 256 + +/* NB: don't allocate keys dynamically, lockdep needs them to be in ".data" */ +#define cfs_percpt_lock_alloc(cptab) \ +({ \ + static struct lock_class_key ___keys[CFS_PERCPT_LOCK_KEYS]; \ + struct cfs_percpt_lock *___lk; \ + \ + if (cfs_cpt_number(cptab) > CFS_PERCPT_LOCK_KEYS) \ + ___lk = cfs_percpt_lock_create(cptab, NULL); \ + else \ + ___lk = cfs_percpt_lock_create(cptab, ___keys); \ + ___lk; \ +}) diff --git a/lnet/lnet/Makefile.in b/lnet/lnet/Makefile.in index a85d6c1..d564a21 100644 --- a/lnet/lnet/Makefile.in +++ b/lnet/lnet/Makefile.in @@ -1,6 +1,6 @@ MODULES := lnet -lnet-objs := api-ni.o config.o nidstrings.o lnet_rdma.o +lnet-objs := api-ni.o config.o nidstrings.o lnet_rdma.o lock.o lnet-objs += lib-me.o lib-msg.o lib-md.o lib-ptl.o lnet-objs += lib-socket.o lib-move.o module.o lo.o lnet-objs += router.o router_proc.o acceptor.o peer.o net_fault.o udsp.o diff --git a/libcfs/libcfs/libcfs_lock.c b/lnet/lnet/lock.c similarity index 87% rename from libcfs/libcfs/libcfs_lock.c rename to lnet/lnet/lock.c index c4ad568..721d56d 100644 --- a/libcfs/libcfs/libcfs_lock.c +++ b/lnet/lnet/lock.c @@ -13,25 +13,19 @@ * General Public License version 2 for more details (a copy is included * in the LICENSE file that accompanied this code). * - * You should have received a copy of the GNU General Public License - * version 2 along with this program; if not, write to the - * Free Software Foundation, Inc., 59 Temple Place - Suite 330, - * Boston, MA 021110-1307, USA - * * GPL HEADER END */ /* Copyright (c) 2010, Oracle and/or its affiliates. All rights reserved. * Copyright (c) 2012, 2015, Intel Corporation. */ -/* - * This file is part of Lustre, http://www.lustre.org/ +/* This file is part of Lustre, http://www.lustre.org/ * * Author: liang@whamcloud.com */ #define DEBUG_SUBSYSTEM S_LNET -#include +#include /** destroy cpu-partition lock, see libcfs_private.h for more detail */ void @@ -73,8 +67,7 @@ cfs_percpt_lock_create(struct cfs_cpt_table *cptab, } if (keys == NULL) { - CWARN("Cannot setup class key for percpt lock, you may see " - "recursive locking warnings which are actually fake.\n"); + CWARN("Cannot setup class key for percpt lock, you may see recursive locking warnings which are actually fake.\n"); } cfs_percpt_for_each(lock, i, pcl->pcl_locks) { @@ -123,7 +116,8 @@ __acquires(pcl->pcl_locks) if (i == 0) { LASSERT(!pcl->pcl_locked); /* nobody should take private lock after this - * so I wouldn't starve for too long time */ + * so I wouldn't starve for too long time + */ pcl->pcl_locked = 1; } } -- 1.8.3.1