+/**
+ * Choose max to \a number CPUs from \a node and set them in \a cpt.
+ * We always prefer to choose CPU in the same core/socket.
+ */
+static int cfs_cpt_choose_ncpus(struct cfs_cpt_table *cptab, int cpt,
+ cpumask_t *node_mask, int number)
+{
+ cpumask_t *socket_mask = NULL;
+ cpumask_t *core_mask = NULL;
+ int rc = 0;
+ int cpu;
+ int i;
+
+ LASSERT(number > 0);
+
+ if (number >= cpumask_weight(node_mask)) {
+ while (!cpumask_empty(node_mask)) {
+ cpu = cpumask_first(node_mask);
+ cpumask_clear_cpu(cpu, node_mask);
+
+ if (!cpu_online(cpu))
+ continue;
+
+ rc = cfs_cpt_set_cpu(cptab, cpt, cpu);
+ if (!rc)
+ return -EINVAL;
+ }
+ return 0;
+ }
+
+ /* allocate scratch buffer */
+ LIBCFS_ALLOC(socket_mask, cpumask_size());
+ LIBCFS_ALLOC(core_mask, cpumask_size());
+ if (!socket_mask || !core_mask) {
+ rc = -ENOMEM;
+ goto out;
+ }
+
+ while (!cpumask_empty(node_mask)) {
+ cpu = cpumask_first(node_mask);
+
+ /* get cpumask for cores in the same socket */
+ cpumask_and(socket_mask, topology_core_cpumask(cpu), node_mask);
+ while (!cpumask_empty(socket_mask)) {
+ /* get cpumask for hts in the same core */
+ cpumask_and(core_mask, topology_sibling_cpumask(cpu),
+ node_mask);
+
+ for_each_cpu(i, core_mask) {
+ cpumask_clear_cpu(i, socket_mask);
+ cpumask_clear_cpu(i, node_mask);
+
+ if (!cpu_online(i))
+ continue;
+
+ rc = cfs_cpt_set_cpu(cptab, cpt, i);
+ if (!rc) {
+ rc = -EINVAL;
+ goto out;
+ }
+
+ if (!--number)
+ goto out;
+ }
+ cpu = cpumask_first(socket_mask);
+ }
+ }
+
+out:
+ if (core_mask)
+ LIBCFS_FREE(core_mask, cpumask_size());
+ if (socket_mask)
+ LIBCFS_FREE(socket_mask, cpumask_size());
+ return rc;
+}
+
+#define CPT_WEIGHT_MIN 4
+
+static int cfs_cpt_num_estimate(void)
+{
+ int nthr = cpumask_weight(topology_sibling_cpumask(smp_processor_id()));
+ int ncpu = num_online_cpus();
+ int ncpt = 1;
+
+ if (ncpu > CPT_WEIGHT_MIN)
+ for (ncpt = 2; ncpu > 2 * nthr * ncpt; ncpt++)
+ ; /* nothing */
+
+#if (BITS_PER_LONG == 32)
+ /* config many CPU partitions on 32-bit system could consume
+ * too much memory
+ */
+ ncpt = min(2, ncpt);
+#endif
+ while (ncpu % ncpt)
+ ncpt--; /* worst case is 1 */
+
+ return ncpt;
+}
+
+static struct cfs_cpt_table *cfs_cpt_table_create(int ncpt)
+{
+ struct cfs_cpt_table *cptab = NULL;
+ cpumask_t *node_mask = NULL;
+ int cpt = 0;
+ int node;
+ int num;
+ int rem;
+ int rc = 0;
+
+ num = cfs_cpt_num_estimate();
+ if (ncpt <= 0)
+ ncpt = num;
+
+ if (ncpt > num_online_cpus() || ncpt > 4 * num) {
+ CWARN("CPU partition number %d is larger than suggested value (%d), your system may have performance issue or run out of memory while under pressure\n",
+ ncpt, num);
+ }
+
+ cptab = cfs_cpt_table_alloc(ncpt);
+ if (!cptab) {
+ CERROR("Failed to allocate CPU map(%d)\n", ncpt);
+ rc = -ENOMEM;
+ goto failed;
+ }
+
+ LIBCFS_ALLOC(node_mask, cpumask_size());
+ if (!node_mask) {
+ CERROR("Failed to allocate scratch cpumask\n");
+ rc = -ENOMEM;
+ goto failed;
+ }
+
+ num = num_online_cpus() / ncpt;
+ rem = num_online_cpus() % ncpt;
+ for_each_online_node(node) {
+ cpumask_copy(node_mask, cpumask_of_node(node));
+
+ while (cpt < ncpt && !cpumask_empty(node_mask)) {
+ struct cfs_cpu_partition *part = &cptab->ctb_parts[cpt];
+ int ncpu = cpumask_weight(part->cpt_cpumask);
+
+ rc = cfs_cpt_choose_ncpus(cptab, cpt, node_mask,
+ num - ncpu);
+ if (rc < 0) {
+ rc = -EINVAL;
+ goto failed_mask;
+ }
+
+ ncpu = cpumask_weight(part->cpt_cpumask);
+ if (ncpu == num + !!(rem > 0)) {
+ cpt++;
+ rem--;
+ }
+ }
+ }
+
+ LIBCFS_FREE(node_mask, cpumask_size());
+
+ return cptab;
+
+failed_mask:
+ if (node_mask)
+ LIBCFS_FREE(node_mask, cpumask_size());
+failed:
+ CERROR("Failed (rc = %d) to setup CPU partition table with %d partitions, online HW NUMA nodes: %d, HW CPU cores: %d.\n",
+ rc, ncpt, num_online_nodes(), num_online_cpus());
+
+ if (cptab)
+ cfs_cpt_table_free(cptab);
+
+ return ERR_PTR(rc);
+}
+
+static struct cfs_cpt_table *cfs_cpt_table_create_pattern(const char *pattern)
+{
+ struct cfs_cpt_table *cptab;
+ char *pattern_dup;
+ char *bracket;
+ char *str;
+ int node = 0;
+ int ncpt = 0;
+ int cpt = 0;
+ int high;
+ int rc;
+ int c;
+ int i;
+
+ pattern_dup = kstrdup(pattern, GFP_KERNEL);
+ if (!pattern_dup) {
+ CERROR("Failed to duplicate pattern '%s'\n", pattern);
+ return ERR_PTR(-ENOMEM);
+ }
+
+ str = cfs_trimwhite(pattern_dup);
+ if (*str == 'n' || *str == 'N') {
+ str++; /* skip 'N' char */
+ node = 1; /* NUMA pattern */
+ if (*str == '\0') {
+ node = -1;
+ for_each_online_node(i) {
+ if (!cpumask_empty(cpumask_of_node(i)))
+ ncpt++;
+ }
+ if (ncpt == 1) { /* single NUMA node */
+ kfree(pattern_dup);
+ return cfs_cpt_table_create(cpu_npartitions);
+ }
+ }
+ }
+
+ if (!ncpt) { /* scanning bracket which is mark of partition */
+ bracket = str;
+ while ((bracket = strchr(bracket, '['))) {
+ bracket++;
+ ncpt++;
+ }
+ }
+
+ if (!ncpt ||
+ (node && ncpt > num_online_nodes()) ||
+ (!node && ncpt > num_online_cpus())) {
+ CERROR("Invalid pattern '%s', or too many partitions %d\n",
+ pattern_dup, ncpt);
+ rc = -EINVAL;
+ goto err_free_str;
+ }
+
+ cptab = cfs_cpt_table_alloc(ncpt);
+ if (!cptab) {
+ CERROR("Failed to allocate CPU partition table\n");
+ rc = -ENOMEM;
+ goto err_free_str;
+ }
+
+ if (node < 0) { /* shortcut to create CPT from NUMA & CPU topology */
+ for_each_online_node(i) {
+ if (cpumask_empty(cpumask_of_node(i)))
+ continue;
+
+ rc = cfs_cpt_set_node(cptab, cpt++, i);
+ if (!rc) {
+ rc = -EINVAL;
+ goto err_free_table;
+ }
+ }
+ kfree(pattern_dup);
+ return cptab;
+ }
+
+ high = node ? nr_node_ids - 1 : nr_cpu_ids - 1;
+
+ for (str = cfs_trimwhite(str), c = 0; /* until break */; c++) {
+ struct cfs_range_expr *range;
+ struct cfs_expr_list *el;
+ int n;
+
+ bracket = strchr(str, '[');
+ if (!bracket) {
+ if (*str) {
+ CERROR("Invalid pattern '%s'\n", str);
+ rc = -EINVAL;
+ goto err_free_table;
+ } else if (c != ncpt) {
+ CERROR("Expect %d partitions but found %d\n",
+ ncpt, c);
+ rc = -EINVAL;
+ goto err_free_table;
+ }
+ break;
+ }
+
+ if (sscanf(str, "%d%n", &cpt, &n) < 1) {
+ CERROR("Invalid CPU pattern '%s'\n", str);
+ rc = -EINVAL;
+ goto err_free_table;
+ }
+
+ if (cpt < 0 || cpt >= ncpt) {
+ CERROR("Invalid partition id %d, total partitions %d\n",
+ cpt, ncpt);
+ rc = -EINVAL;
+ goto err_free_table;
+ }
+
+ if (cfs_cpt_weight(cptab, cpt)) {
+ CERROR("Partition %d has already been set.\n", cpt);
+ rc = -EPERM;
+ goto err_free_table;
+ }
+
+ str = cfs_trimwhite(str + n);
+ if (str != bracket) {
+ CERROR("Invalid pattern '%s'\n", str);
+ rc = -EINVAL;
+ goto err_free_table;
+ }
+
+ bracket = strchr(str, ']');
+ if (!bracket) {
+ CERROR("Missing right bracket for partition %d in '%s'\n",
+ cpt, str);
+ rc = -EINVAL;
+ goto err_free_table;
+ }
+
+ rc = cfs_expr_list_parse(str, (bracket - str) + 1, 0, high,
+ &el);
+ if (rc) {
+ CERROR("Can't parse number range in '%s'\n", str);
+ rc = -ERANGE;
+ goto err_free_table;
+ }
+
+ list_for_each_entry(range, &el->el_exprs, re_link) {
+ for (i = range->re_lo; i <= range->re_hi; i++) {
+ if ((i - range->re_lo) % range->re_stride)
+ continue;
+
+ rc = node ? cfs_cpt_set_node(cptab, cpt, i)
+ : cfs_cpt_set_cpu(cptab, cpt, i);
+ if (!rc) {
+ cfs_expr_list_free(el);
+ rc = -EINVAL;
+ goto err_free_table;
+ }
+ }
+ }
+
+ cfs_expr_list_free(el);
+
+ if (!cfs_cpt_online(cptab, cpt)) {
+ CERROR("No online CPU is found on partition %d\n", cpt);
+ rc = -ENODEV;
+ goto err_free_table;
+ }
+
+ str = cfs_trimwhite(bracket + 1);
+ }
+
+ kfree(pattern_dup);
+ return cptab;
+
+err_free_table:
+ cfs_cpt_table_free(cptab);
+err_free_str:
+ kfree(pattern_dup);
+ return ERR_PTR(rc);
+}
+
+#ifdef CONFIG_HOTPLUG_CPU
+#ifdef HAVE_HOTPLUG_STATE_MACHINE
+static enum cpuhp_state lustre_cpu_online;
+
+static int cfs_cpu_online(unsigned int cpu)
+{
+ return 0;
+}
+#endif
+
+static int cfs_cpu_dead(unsigned int cpu)
+{
+ bool warn;
+
+ /* if all HTs in a core are offline, it may break affinity */
+ warn = cpumask_any_and(topology_sibling_cpumask(cpu),
+ cpu_online_mask) >= nr_cpu_ids;
+ CDEBUG(warn ? D_WARNING : D_INFO,
+ "Lustre: can't support CPU plug-out well now, performance and stability could be impacted [CPU %u]\n",
+ cpu);
+ return 0;
+}
+
+#ifndef HAVE_HOTPLUG_STATE_MACHINE
+static int cfs_cpu_notify(struct notifier_block *self, unsigned long action,
+ void *hcpu)
+{
+ int cpu = (unsigned long)hcpu;
+
+ switch (action) {
+ case CPU_DEAD:
+ case CPU_DEAD_FROZEN:
+ case CPU_ONLINE:
+ case CPU_ONLINE_FROZEN:
+ default:
+ if (action != CPU_DEAD && action != CPU_DEAD_FROZEN) {
+ CDEBUG(D_INFO, "CPU changed [cpu %u action %lx]\n",
+ cpu, action);
+ break;
+ }
+
+ cfs_cpu_dead(cpu);
+ }
+
+ return NOTIFY_OK;
+}
+
+static struct notifier_block cfs_cpu_notifier = {
+ .notifier_call = cfs_cpu_notify,
+ .priority = 0
+};
+#endif /* !HAVE_HOTPLUG_STATE_MACHINE */
+#endif /* CONFIG_HOTPLUG_CPU */
+
+void cfs_cpu_fini(void)
+{
+ if (!IS_ERR_OR_NULL(cfs_cpt_table))
+ cfs_cpt_table_free(cfs_cpt_table);
+
+#ifdef CONFIG_HOTPLUG_CPU
+#ifdef HAVE_HOTPLUG_STATE_MACHINE
+ if (lustre_cpu_online > 0)
+ cpuhp_remove_state_nocalls(lustre_cpu_online);
+ cpuhp_remove_state_nocalls(CPUHP_LUSTRE_CFS_DEAD);
+#else
+ unregister_hotcpu_notifier(&cfs_cpu_notifier);
+#endif /* !HAVE_HOTPLUG_STATE_MACHINE */
+#endif /* CONFIG_HOTPLUG_CPU */
+}
+
+int cfs_cpu_init(void)
+{
+ int ret;
+
+ LASSERT(!cfs_cpt_table);
+
+#ifdef CONFIG_HOTPLUG_CPU
+#ifdef HAVE_HOTPLUG_STATE_MACHINE
+ ret = cpuhp_setup_state_nocalls(CPUHP_LUSTRE_CFS_DEAD,
+ "fs/lustre/cfe:dead", NULL,
+ cfs_cpu_dead);
+ if (ret < 0)
+ goto failed_cpu_dead;
+
+ ret = cpuhp_setup_state_nocalls(CPUHP_AP_ONLINE_DYN,
+ "fs/lustre/cfe:online",
+ cfs_cpu_online, NULL);
+ if (ret < 0)
+ goto failed_cpu_online;
+
+ lustre_cpu_online = ret;
+#else
+ register_hotcpu_notifier(&cfs_cpu_notifier);
+#endif /* !HAVE_HOTPLUG_STATE_MACHINE */
+#endif /* CONFIG_HOTPLUG_CPU */
+
+ get_online_cpus();
+ if (*cpu_pattern) {
+ cfs_cpt_table = cfs_cpt_table_create_pattern(cpu_pattern);
+ if (IS_ERR(cfs_cpt_table)) {
+ CERROR("Failed to create cptab from pattern '%s'\n",
+ cpu_pattern);
+ ret = PTR_ERR(cfs_cpt_table);
+ goto failed_alloc_table;
+ }
+
+ } else {
+ cfs_cpt_table = cfs_cpt_table_create(cpu_npartitions);
+ if (IS_ERR(cfs_cpt_table)) {
+ CERROR("Failed to create cptab with npartitions %d\n",
+ cpu_npartitions);
+ ret = PTR_ERR(cfs_cpt_table);
+ goto failed_alloc_table;
+ }
+ }
+
+ put_online_cpus();
+
+ LCONSOLE(0, "HW NUMA nodes: %d, HW CPU cores: %d, npartitions: %d\n",
+ num_online_nodes(), num_online_cpus(),
+ cfs_cpt_number(cfs_cpt_table));
+ return 0;
+
+failed_alloc_table:
+ put_online_cpus();
+
+ if (cfs_cpt_table)
+ cfs_cpt_table_free(cfs_cpt_table);
+
+#ifdef CONFIG_HOTPLUG_CPU
+#ifdef HAVE_HOTPLUG_STATE_MACHINE
+ if (lustre_cpu_online > 0)
+ cpuhp_remove_state_nocalls(lustre_cpu_online);
+failed_cpu_online:
+ cpuhp_remove_state_nocalls(CPUHP_LUSTRE_CFS_DEAD);
+failed_cpu_dead:
+#else
+ unregister_hotcpu_notifier(&cfs_cpu_notifier);
+#endif /* !HAVE_HOTPLUG_STATE_MACHINE */
+#endif /* CONFIG_HOTPLUG_CPU */
+ return ret;
+}
+
+#else /* ! CONFIG_SMP */
+
+struct cfs_cpt_table *cfs_cpt_table_alloc(int ncpt)
+{
+ struct cfs_cpt_table *cptab;
+
+ if (ncpt != 1) {
+ CERROR("Can't support cpu partition number %d\n", ncpt);
+ return NULL;
+ }
+
+ LIBCFS_ALLOC(cptab, sizeof(*cptab));
+ if (!cptab)
+ return NULL;
+
+ cpumask_set_cpu(0, cptab->ctb_cpumask);
+ node_set(0, cptab->ctb_nodemask);
+
+ return cptab;
+}
+EXPORT_SYMBOL(cfs_cpt_table_alloc);
+
+int cfs_cpt_table_print(struct cfs_cpt_table *cptab, char *buf, int len)
+{
+ int rc;
+
+ rc = snprintf(buf, len, "0\t: 0\n");
+ len -= rc;
+ if (len <= 0)
+ return -EFBIG;
+
+ return rc;
+}
+EXPORT_SYMBOL(cfs_cpt_table_print);
+
+int cfs_cpt_distance_print(struct cfs_cpt_table *cptab, char *buf, int len)
+{
+ int rc;
+
+ rc = snprintf(buf, len, "0\t: 0:1\n");
+ len -= rc;
+ if (len <= 0)
+ return -EFBIG;
+
+ return rc;
+}
+EXPORT_SYMBOL(cfs_cpt_distance_print);
+