4 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
6 * This program is free software; you can redistribute it and/or modify
7 * it under the terms of the GNU General Public License version 2 only,
8 * as published by the Free Software Foundation.
10 * This program is distributed in the hope that it will be useful, but
11 * WITHOUT ANY WARRANTY; without even the implied warranty of
12 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
13 * General Public License version 2 for more details (a copy is included
14 * in the LICENSE file that accompanied this code).
19 * Copyright (c) 2010, Oracle and/or its affiliates. All rights reserved.
21 * Copyright (c) 2012, 2017, Intel Corporation.
24 * This file is part of Lustre, http://www.lustre.org/
25 * Lustre is a trademark of Sun Microsystems, Inc.
27 * libcfs/include/libcfs/libcfs_cpu.h
30 * . CPU partition is virtual processing unit
32 * . CPU partition can present 1-N cores, or 1-N NUMA nodes,
33 * in other words, CPU partition is a processors pool.
35 * CPU Partition Table (CPT)
36 * . a set of CPU partitions
38 * . There are two modes for CPT: CFS_CPU_MODE_NUMA and CFS_CPU_MODE_SMP
40 * . User can specify total number of CPU partitions while creating a
41 * CPT, ID of CPU partition is always start from 0.
43 * Example: if there are 8 cores on the system, while creating a CPT
44 * with cpu_npartitions=4:
45 * core[0, 1] = partition[0], core[2, 3] = partition[1]
46 * core[4, 5] = partition[2], core[6, 7] = partition[3]
49 * core[0, 1, ... 7] = partition[0]
51 * . User can also specify CPU partitions by string pattern
53 * Examples: cpu_partitions="0[0,1], 1[2,3]"
54 * cpu_partitions="N 0[0-3], 1[4-8]"
56 * The first character "N" means following numbers are numa ID
58 * . NUMA allocators, CPU affinity threads are built over CPU partitions,
59 * instead of HW CPUs or HW nodes.
61 * . By default, Lustre modules should refer to the global cfs_cpt_tab,
62 * instead of accessing HW CPUs directly, so concurrency of Lustre can be
63 * configured by cpu_npartitions of the global cfs_cpt_tab
65 * . If cpu_npartitions=1(all CPUs in one pool), lustre should work the
66 * same way as 2.2 or earlier versions
68 * Author: liang@whamcloud.com
71 #ifndef __LIBCFS_CPU_H__
72 #define __LIBCFS_CPU_H__
74 #include <linux/cpu.h>
75 #include <linux/cpuset.h>
76 #include <linux/slab.h>
77 #include <linux/topology.h>
78 #include <linux/version.h>
79 #include <linux/vmalloc.h>
81 #include <libcfs/linux/linux-cpu.h>
83 /* any CPU partition */
84 #define CFS_CPT_ANY (-1)
89 extern struct cfs_cpt_table *cfs_cpt_tab;
91 /** virtual processing unit */
92 struct cfs_cpu_partition {
93 /* CPUs mask for this partition */
94 cpumask_var_t cpt_cpumask;
95 /* nodes mask for this partition */
96 nodemask_t *cpt_nodemask;
97 /* NUMA distance between CPTs */
98 unsigned int *cpt_distance;
99 /* spread rotor for NUMA allocator */
100 unsigned int cpt_spread_rotor;
101 /* NUMA node if cpt_nodemask is empty */
105 /** descriptor for CPU partitions */
106 struct cfs_cpt_table {
107 /* spread rotor for NUMA allocator */
108 unsigned int ctb_spread_rotor;
109 /* maximum NUMA distance between all nodes in table */
110 unsigned int ctb_distance;
111 /* # of CPU partitions */
113 /* partitions tables */
114 struct cfs_cpu_partition *ctb_parts;
115 /* shadow HW CPU to CPU partition ID */
117 /* all cpus in this partition table */
118 cpumask_var_t ctb_cpumask;
119 /* shadow HW node to CPU partition ID */
121 /* all nodes in this partition table */
122 nodemask_t *ctb_nodemask;
126 * destroy a CPU partition table
128 void cfs_cpt_table_free(struct cfs_cpt_table *cptab);
130 * create a cfs_cpt_table with \a ncpt number of partitions
132 struct cfs_cpt_table *cfs_cpt_table_alloc(int ncpt);
134 * print string information of cpt-table
136 int cfs_cpt_table_print(struct cfs_cpt_table *cptab, char *buf, int len);
138 * print distance information of cpt-table
140 int cfs_cpt_distance_print(struct cfs_cpt_table *cptab, char *buf, int len);
142 * return total number of CPU partitions in \a cptab
144 int cfs_cpt_number(struct cfs_cpt_table *cptab);
146 * return number of HW cores or hyper-threadings in a CPU partition \a cpt
148 int cfs_cpt_weight(struct cfs_cpt_table *cptab, int cpt);
150 * is there any online CPU in CPU partition \a cpt
152 int cfs_cpt_online(struct cfs_cpt_table *cptab, int cpt);
154 * return cpumask of CPU partition \a cpt
156 cpumask_var_t *cfs_cpt_cpumask(struct cfs_cpt_table *cptab, int cpt);
158 * return nodemask of CPU partition \a cpt
160 nodemask_t *cfs_cpt_nodemask(struct cfs_cpt_table *cptab, int cpt);
162 * shadow current HW processor ID to CPU-partition ID of \a cptab
164 int cfs_cpt_current(struct cfs_cpt_table *cptab, int remap);
166 * shadow HW processor ID \a CPU to CPU-partition ID by \a cptab
168 int cfs_cpt_of_cpu(struct cfs_cpt_table *cptab, int cpu);
170 * shadow HW node ID \a NODE to CPU-partition ID by \a cptab
172 int cfs_cpt_of_node(struct cfs_cpt_table *cptab, int node);
174 * NUMA distance between \a cpt1 and \a cpt2 in \a cptab
176 unsigned int cfs_cpt_distance(struct cfs_cpt_table *cptab, int cpt1, int cpt2);
178 * bind current thread on a CPU-partition \a cpt of \a cptab
180 int cfs_cpt_bind(struct cfs_cpt_table *cptab, int cpt);
182 * add \a cpu to CPU partition @cpt of \a cptab, return 1 for success,
183 * otherwise 0 is returned
185 int cfs_cpt_set_cpu(struct cfs_cpt_table *cptab, int cpt, int cpu);
187 * remove \a cpu from CPU partition \a cpt of \a cptab
189 void cfs_cpt_unset_cpu(struct cfs_cpt_table *cptab, int cpt, int cpu);
191 * add all cpus in \a mask to CPU partition \a cpt
192 * return 1 if successfully set all CPUs, otherwise return 0
194 int cfs_cpt_set_cpumask(struct cfs_cpt_table *cptab, int cpt,
195 const cpumask_t *mask);
197 * remove all cpus in \a mask from CPU partition \a cpt
199 void cfs_cpt_unset_cpumask(struct cfs_cpt_table *cptab, int cpt,
200 const cpumask_t *mask);
202 * add all cpus in NUMA node \a node to CPU partition \a cpt
203 * return 1 if successfully set all CPUs, otherwise return 0
205 int cfs_cpt_set_node(struct cfs_cpt_table *cptab, int cpt, int node);
207 * remove all cpus in NUMA node \a node from CPU partition \a cpt
209 void cfs_cpt_unset_node(struct cfs_cpt_table *cptab, int cpt, int node);
211 * add all cpus in node mask \a mask to CPU partition \a cpt
212 * return 1 if successfully set all CPUs, otherwise return 0
214 int cfs_cpt_set_nodemask(struct cfs_cpt_table *cptab, int cpt,
215 const nodemask_t *mask);
217 * remove all cpus in node mask \a mask from CPU partition \a cpt
219 void cfs_cpt_unset_nodemask(struct cfs_cpt_table *cptab, int cpt,
220 const nodemask_t *mask);
222 * convert partition id \a cpt to numa node id, if there are more than one
223 * nodes in this partition, it might return a different node id each time.
225 int cfs_cpt_spread_node(struct cfs_cpt_table *cptab, int cpt);
227 int cfs_cpu_init(void);
228 void cfs_cpu_fini(void);
230 #else /* !CONFIG_SMP */
232 #define cfs_cpt_tab ((struct cfs_cpt_table *)NULL)
234 static inline void cfs_cpt_table_free(struct cfs_cpt_table *cptab)
238 static inline struct cfs_cpt_table *cfs_cpt_table_alloc(int ncpt)
243 static inline int cfs_cpt_table_print(struct cfs_cpt_table *cptab,
248 rc = snprintf(buf, len, "0\t: 0\n");
256 static inline int cfs_cpt_distance_print(struct cfs_cpt_table *cptab,
261 rc = snprintf(buf, len, "0\t: 0:1\n");
269 static inline cpumask_var_t *cfs_cpt_cpumask(struct cfs_cpt_table *cptab,
272 return (cpumask_var_t *) cpu_online_mask;
275 static inline int cfs_cpt_number(struct cfs_cpt_table *cptab)
280 static inline int cfs_cpt_weight(struct cfs_cpt_table *cptab, int cpt)
285 static inline nodemask_t *cfs_cpt_nodemask(struct cfs_cpt_table *cptab,
288 return &node_online_map;
291 static inline unsigned int cfs_cpt_distance(struct cfs_cpt_table *cptab,
297 static inline int cfs_cpt_set_node(struct cfs_cpt_table *cptab, int cpt,
303 static inline int cfs_cpt_spread_node(struct cfs_cpt_table *cptab, int cpt)
308 static inline int cfs_cpt_current(struct cfs_cpt_table *cptab, int remap)
313 static inline int cfs_cpt_of_node(struct cfs_cpt_table *cptab, int node)
318 static inline int cfs_cpt_bind(struct cfs_cpt_table *cptab, int cpt)
323 static inline int cfs_cpu_init(void)
328 static inline void cfs_cpu_fini(void)
332 #endif /* CONFIG_SMP */
335 * allocate per-cpu-partition data, returned value is an array of pointers,
336 * variable can be indexed by CPU ID.
337 * cptab != NULL: size of array is number of CPU partitions
338 * cptab == NULL: size of array is number of HW cores
340 void *cfs_percpt_alloc(struct cfs_cpt_table *cptab, unsigned int size);
342 * destroy per-cpu-partition variable
344 void cfs_percpt_free(void *vars);
345 int cfs_percpt_number(void *vars);
347 #define cfs_percpt_for_each(var, i, vars) \
348 for (i = 0; i < cfs_percpt_number(vars) && \
349 ((var) = (vars)[i]) != NULL; i++)
352 * percpu partition lock
354 * There are some use-cases like this in Lustre:
355 * . each CPU partition has it's own private data which is frequently changed,
356 * and mostly by the local CPU partition.
357 * . all CPU partitions share some global data, these data are rarely changed.
359 * LNet is typical example.
360 * CPU partition lock is designed for this kind of use-cases:
361 * . each CPU partition has it's own private lock
362 * . change on private data just needs to take the private lock
363 * . read on shared data just needs to take _any_ of private locks
364 * . change on shared data needs to take _all_ private locks,
365 * which is slow and should be really rare.
368 CFS_PERCPT_LOCK_EX = -1, /* negative */
371 struct cfs_percpt_lock {
372 /* cpu-partition-table for this lock */
373 struct cfs_cpt_table *pcl_cptab;
374 /* exclusively locked */
375 unsigned int pcl_locked;
376 /* private lock table */
377 spinlock_t **pcl_locks;
380 /* return number of private locks */
381 #define cfs_percpt_lock_num(pcl) cfs_cpt_number(pcl->pcl_cptab)
384 * create a cpu-partition lock based on CPU partition table \a cptab,
385 * each private lock has extra \a psize bytes padding data
387 struct cfs_percpt_lock *cfs_percpt_lock_create(struct cfs_cpt_table *cptab,
388 struct lock_class_key *keys);
389 /* destroy a cpu-partition lock */
390 void cfs_percpt_lock_free(struct cfs_percpt_lock *pcl);
392 /* lock private lock \a index of \a pcl */
393 void cfs_percpt_lock(struct cfs_percpt_lock *pcl, int index);
395 /* unlock private lock \a index of \a pcl */
396 void cfs_percpt_unlock(struct cfs_percpt_lock *pcl, int index);
398 #define CFS_PERCPT_LOCK_KEYS 256
400 /* NB: don't allocate keys dynamically, lockdep needs them to be in ".data" */
401 #define cfs_percpt_lock_alloc(cptab) \
403 static struct lock_class_key ___keys[CFS_PERCPT_LOCK_KEYS]; \
404 struct cfs_percpt_lock *___lk; \
406 if (cfs_cpt_number(cptab) > CFS_PERCPT_LOCK_KEYS) \
407 ___lk = cfs_percpt_lock_create(cptab, NULL); \
409 ___lk = cfs_percpt_lock_create(cptab, ___keys); \
414 * allocate \a nr_bytes of physical memory from a contiguous region with the
415 * properties of \a flags which are bound to the partition id \a cpt. This
416 * function should only be used for the case when only a few pages of memory
420 cfs_cpt_malloc(struct cfs_cpt_table *cptab, int cpt, size_t nr_bytes,
423 return kmalloc_node(nr_bytes, flags,
424 cfs_cpt_spread_node(cptab, cpt));
428 * allocate \a nr_bytes of virtually contiguous memory that is bound to the
429 * partition id \a cpt.
432 cfs_cpt_vzalloc(struct cfs_cpt_table *cptab, int cpt, size_t nr_bytes)
434 /* vzalloc_node() sets __GFP_FS by default but no current Kernel
435 * exported entry-point allows for both a NUMA node specification
436 * and a custom allocation flags mask. This may be an issue since
437 * __GFP_FS usage can cause some deadlock situations in our code,
438 * like when memory reclaim started, within the same context of a
439 * thread doing FS operations, that can also attempt conflicting FS
442 return vzalloc_node(nr_bytes, cfs_cpt_spread_node(cptab, cpt));
446 * allocate a single page of memory with the properties of \a flags were
447 * that page is bound to the partition id \a cpt.
449 static inline struct page *
450 cfs_page_cpt_alloc(struct cfs_cpt_table *cptab, int cpt, gfp_t flags)
452 return alloc_pages_node(cfs_cpt_spread_node(cptab, cpt), flags, 0);
456 * allocate a chunck of memory from a memory pool that is bound to the
457 * partition id \a cpt with the properites of \a flags.
460 cfs_mem_cache_cpt_alloc(struct kmem_cache *cachep, struct cfs_cpt_table *cptab,
461 int cpt, gfp_t flags)
463 return kmem_cache_alloc_node(cachep, flags,
464 cfs_cpt_spread_node(cptab, cpt));
468 * iterate over all CPU partitions in \a cptab
470 #define cfs_cpt_for_each(i, cptab) \
471 for (i = 0; i < cfs_cpt_number(cptab); i++)
473 #endif /* __LIBCFS_CPU_H__ */