4 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
6 * This program is free software; you can redistribute it and/or modify
7 * it under the terms of the GNU General Public License version 2 only,
8 * as published by the Free Software Foundation.
10 * This program is distributed in the hope that it will be useful, but
11 * WITHOUT ANY WARRANTY; without even the implied warranty of
12 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
13 * General Public License version 2 for more details (a copy is included
14 * in the LICENSE file that accompanied this code).
18 /* Copyright (c) 2010, Oracle and/or its affiliates. All rights reserved.
20 * Copyright (c) 2012, 2017, Intel Corporation.
22 /* This file is part of Lustre, http://www.lustre.org/
25 * . CPU partition is virtual processing unit
27 * . CPU partition can present 1-N cores, or 1-N NUMA nodes,
28 * in other words, CPU partition is a processors pool.
30 * CPU Partition Table (CPT)
31 * . a set of CPU partitions
33 * . There are two modes for CPT: CFS_CPU_MODE_NUMA and CFS_CPU_MODE_SMP
35 * . User can specify total number of CPU partitions while creating a
36 * CPT, ID of CPU partition is always start from 0.
38 * Example: if there are 8 cores on the system, while creating a CPT
39 * with cpu_npartitions=4:
40 * core[0, 1] = partition[0], core[2, 3] = partition[1]
41 * core[4, 5] = partition[2], core[6, 7] = partition[3]
44 * core[0, 1, ... 7] = partition[0]
46 * . User can also specify CPU partitions by string pattern
48 * Examples: cpu_partitions="0[0,1], 1[2,3]"
49 * cpu_partitions="N 0[0-3], 1[4-8]"
51 * The first character "N" means following numbers are numa ID
53 * . NUMA allocators, CPU affinity threads are built over CPU partitions,
54 * instead of HW CPUs or HW nodes.
56 * . By default, Lustre modules should refer to the global cfs_cpt_tab,
57 * instead of accessing HW CPUs directly, so concurrency of Lustre can be
58 * configured by cpu_npartitions of the global cfs_cpt_tab
60 * . If cpu_npartitions=1(all CPUs in one pool), lustre should work the
61 * same way as 2.2 or earlier versions
63 * Author: liang@whamcloud.com
66 #ifndef __LIBCFS_CPU_H__
67 #define __LIBCFS_CPU_H__
69 #include <linux/cpu.h>
70 #include <linux/cpuset.h>
71 #include <linux/slab.h>
72 #include <linux/topology.h>
73 #include <linux/version.h>
74 #include <linux/vmalloc.h>
76 #include <libcfs/linux/linux-cpu.h>
78 /* any CPU partition */
79 #define CFS_CPT_ANY (-1)
84 extern struct cfs_cpt_table *cfs_cpt_tab;
87 * destroy a CPU partition table
89 void cfs_cpt_table_free(struct cfs_cpt_table *cptab);
91 * create a cfs_cpt_table with \a ncpt number of partitions
93 struct cfs_cpt_table *cfs_cpt_table_alloc(int ncpt);
95 * print string information of cpt-table
97 int cfs_cpt_table_print(struct cfs_cpt_table *cptab, char *buf, int len);
99 * print distance information of cpt-table
101 int cfs_cpt_distance_print(struct cfs_cpt_table *cptab, char *buf, int len);
103 * return total number of CPU partitions in \a cptab
105 int cfs_cpt_number(struct cfs_cpt_table *cptab);
107 * return number of HW cores or hyper-threadings in a CPU partition \a cpt
109 int cfs_cpt_weight(struct cfs_cpt_table *cptab, int cpt);
111 * is there any online CPU in CPU partition \a cpt
113 int cfs_cpt_online(struct cfs_cpt_table *cptab, int cpt);
115 * return cpumask of CPU partition \a cpt
117 cpumask_var_t *cfs_cpt_cpumask(struct cfs_cpt_table *cptab, int cpt);
119 * return nodemask of CPU partition \a cpt
121 nodemask_t *cfs_cpt_nodemask(struct cfs_cpt_table *cptab, int cpt);
123 * shadow current HW processor ID to CPU-partition ID of \a cptab
125 int cfs_cpt_current(struct cfs_cpt_table *cptab, int remap);
127 * shadow HW processor ID \a CPU to CPU-partition ID by \a cptab
129 int cfs_cpt_of_cpu(struct cfs_cpt_table *cptab, int cpu);
131 * shadow HW node ID \a NODE to CPU-partition ID by \a cptab
133 int cfs_cpt_of_node(struct cfs_cpt_table *cptab, int node);
135 * NUMA distance between \a cpt1 and \a cpt2 in \a cptab
137 unsigned int cfs_cpt_distance(struct cfs_cpt_table *cptab, int cpt1, int cpt2);
139 * bind current thread on a CPU-partition \a cpt of \a cptab
141 int cfs_cpt_bind(struct cfs_cpt_table *cptab, int cpt);
143 * add \a cpu to CPU partition @cpt of \a cptab, return 1 for success,
144 * otherwise 0 is returned
146 int cfs_cpt_set_cpu(struct cfs_cpt_table *cptab, int cpt, int cpu);
148 * remove \a cpu from CPU partition \a cpt of \a cptab
150 void cfs_cpt_unset_cpu(struct cfs_cpt_table *cptab, int cpt, int cpu);
152 * add all cpus in \a mask to CPU partition \a cpt
153 * return 1 if successfully set all CPUs, otherwise return 0
155 int cfs_cpt_set_cpumask(struct cfs_cpt_table *cptab, int cpt,
156 const cpumask_t *mask);
158 * remove all cpus in \a mask from CPU partition \a cpt
160 void cfs_cpt_unset_cpumask(struct cfs_cpt_table *cptab, int cpt,
161 const cpumask_t *mask);
163 * add all cpus in NUMA node \a node to CPU partition \a cpt
164 * return 1 if successfully set all CPUs, otherwise return 0
166 int cfs_cpt_set_node(struct cfs_cpt_table *cptab, int cpt, int node);
168 * remove all cpus in NUMA node \a node from CPU partition \a cpt
170 void cfs_cpt_unset_node(struct cfs_cpt_table *cptab, int cpt, int node);
172 * add all cpus in node mask \a mask to CPU partition \a cpt
173 * return 1 if successfully set all CPUs, otherwise return 0
175 int cfs_cpt_set_nodemask(struct cfs_cpt_table *cptab, int cpt,
176 const nodemask_t *mask);
178 * remove all cpus in node mask \a mask from CPU partition \a cpt
180 void cfs_cpt_unset_nodemask(struct cfs_cpt_table *cptab, int cpt,
181 const nodemask_t *mask);
183 * convert partition id \a cpt to numa node id, if there are more than one
184 * nodes in this partition, it might return a different node id each time.
186 int cfs_cpt_spread_node(struct cfs_cpt_table *cptab, int cpt);
188 int cfs_cpu_init(void);
189 void cfs_cpu_fini(void);
191 #else /* !CONFIG_SMP */
193 #define cfs_cpt_tab ((struct cfs_cpt_table *)NULL)
195 static inline void cfs_cpt_table_free(struct cfs_cpt_table *cptab)
199 static inline struct cfs_cpt_table *cfs_cpt_table_alloc(int ncpt)
204 static inline int cfs_cpt_table_print(struct cfs_cpt_table *cptab,
209 rc = snprintf(buf, len, "0\t: 0\n");
217 static inline int cfs_cpt_distance_print(struct cfs_cpt_table *cptab,
222 rc = snprintf(buf, len, "0\t: 0:1\n");
230 static inline cpumask_var_t *cfs_cpt_cpumask(struct cfs_cpt_table *cptab,
233 return (cpumask_var_t *) cpu_online_mask;
236 static inline int cfs_cpt_number(struct cfs_cpt_table *cptab)
241 static inline int cfs_cpt_weight(struct cfs_cpt_table *cptab, int cpt)
246 static inline nodemask_t *cfs_cpt_nodemask(struct cfs_cpt_table *cptab,
249 return &node_online_map;
252 static inline unsigned int cfs_cpt_distance(struct cfs_cpt_table *cptab,
258 static inline int cfs_cpt_set_node(struct cfs_cpt_table *cptab, int cpt,
264 static inline int cfs_cpt_spread_node(struct cfs_cpt_table *cptab, int cpt)
269 static inline int cfs_cpt_current(struct cfs_cpt_table *cptab, int remap)
274 static inline int cfs_cpt_of_node(struct cfs_cpt_table *cptab, int node)
279 static inline int cfs_cpt_bind(struct cfs_cpt_table *cptab, int cpt)
284 static inline int cfs_cpu_init(void)
289 static inline void cfs_cpu_fini(void)
293 #endif /* CONFIG_SMP */
295 /* Module parameters */
296 extern int cpu_npartitions;
297 extern char *cpu_pattern;
300 struct workqueue_struct *cfs_cpt_bind_workqueue(const char *wq_name,
301 struct cfs_cpt_table *tbl,
302 int flags, int cpt, int nthrs)
304 cpumask_var_t *mask = cfs_cpt_cpumask(tbl, cpt);
305 struct workqueue_attrs attrs = { };
306 struct workqueue_struct *wq;
308 wq = alloc_workqueue("%s", WQ_UNBOUND | flags, nthrs, wq_name);
310 return ERR_PTR(-ENOMEM);
312 if (mask && alloc_cpumask_var(&attrs.cpumask, GFP_KERNEL)) {
313 cpumask_copy(attrs.cpumask, *mask);
315 cfs_apply_workqueue_attrs(wq, &attrs);
317 free_cpumask_var(attrs.cpumask);
323 /* allocate per-cpu-partition data, returned value is an array of pointers,
324 * variable can be indexed by CPU ID.
325 * cptab != NULL: size of array is number of CPU partitions
326 * cptab == NULL: size of array is number of HW cores
328 void *cfs_percpt_alloc(struct cfs_cpt_table *cptab, unsigned int size);
329 /* destroy per-cpu-partition variable */
330 void cfs_percpt_free(void *vars);
331 int cfs_percpt_number(void *vars);
333 #define cfs_percpt_for_each(var, i, vars) \
334 for (i = 0; i < cfs_percpt_number(vars) && \
335 ((var) = (vars)[i]) != NULL; i++)
338 * allocate \a nr_bytes of physical memory from a contiguous region with the
339 * properties of \a flags which are bound to the partition id \a cpt. This
340 * function should only be used for the case when only a few pages of memory
344 cfs_cpt_malloc(struct cfs_cpt_table *cptab, int cpt, size_t nr_bytes,
347 return kmalloc_node(nr_bytes, flags,
348 cfs_cpt_spread_node(cptab, cpt));
352 * allocate \a nr_bytes of virtually contiguous memory that is bound to the
353 * partition id \a cpt.
356 cfs_cpt_vzalloc(struct cfs_cpt_table *cptab, int cpt, size_t nr_bytes)
358 /* vzalloc_node() sets __GFP_FS by default but no current Kernel
359 * exported entry-point allows for both a NUMA node specification
360 * and a custom allocation flags mask. This may be an issue since
361 * __GFP_FS usage can cause some deadlock situations in our code,
362 * like when memory reclaim started, within the same context of a
363 * thread doing FS operations, that can also attempt conflicting FS
366 return vzalloc_node(nr_bytes, cfs_cpt_spread_node(cptab, cpt));
370 * allocate a single page of memory with the properties of \a flags were
371 * that page is bound to the partition id \a cpt.
373 static inline struct page *
374 cfs_page_cpt_alloc(struct cfs_cpt_table *cptab, int cpt, gfp_t flags)
376 return alloc_pages_node(cfs_cpt_spread_node(cptab, cpt), flags, 0);
380 * allocate a chunck of memory from a memory pool that is bound to the
381 * partition id \a cpt with the properites of \a flags.
384 cfs_mem_cache_cpt_alloc(struct kmem_cache *cachep, struct cfs_cpt_table *cptab,
385 int cpt, gfp_t flags)
387 return kmem_cache_alloc_node(cachep, flags,
388 cfs_cpt_spread_node(cptab, cpt));
392 * iterate over all CPU partitions in \a cptab
394 #define cfs_cpt_for_each(i, cptab) \
395 for (i = 0; i < cfs_cpt_number(cptab); i++)
397 #endif /* __LIBCFS_CPU_H__ */