]) # LC_HAVE_FSMAP_HEADER
#
+# LC_HAVE_PERCPU_COUNTER_ADD_BATCH
+#
+# Linux commit v4.11-12447-g104b4e5139fe
+# percpu_counter: Rename __percpu_counter_add to percpu_counter_add_batch
+#
+AC_DEFUN([LC_SRC_HAVE_PERCPU_COUNTER_ADD_BATCH], [
+ LB2_LINUX_TEST_SRC([percpu_counter_add_batch_exists], [
+ #include <linux/percpu_counter.h>
+ ],[
+ (void)percpu_counter_add_batch(NULL, 0, 0);
+ ],[-Werror])
+])
+AC_DEFUN([LC_HAVE_PERCPU_COUNTER_ADD_BATCH], [
+ LB2_MSG_LINUX_TEST_RESULT([if 'percpu_counter_add_batch()' exists],
+ [percpu_counter_add_batch_exists], [
+ AC_DEFINE(HAVE_PERCPU_COUNTER_ADD_BATCH, 1,
+ ['percpu_counter_add_batch()' exists])
+ ])
+]) # LC_HAVE_PERCPU_COUNTER_ADD_BATCH
+
+#
# Kernel version 4.12 commit 47f38c539e9a42344ff5a664942075bd4df93876
# CURRENT_TIME is not 64 bit time safe so it was replaced with
# current_time()
LC_SRC_HAVE_KEY_USAGE_REFCOUNT
LC_SRC_HAVE_CRYPTO_MAX_ALG_NAME_128
LC_SRC_HAVE_FSMAP_HEADER
+ LC_SRC_HAVE_PERCPU_COUNTER_ADD_BATCH
# 4.12
LC_SRC_CURRENT_TIME
LC_HAVE_KEY_USAGE_REFCOUNT
LC_HAVE_CRYPTO_MAX_ALG_NAME_128
LC_HAVE_FSMAP_HEADER
+ LC_HAVE_PERCPU_COUNTER_ADD_BATCH
# 4.12
LC_CURRENT_TIME
__s64 lc_count;
__s64 lc_min;
__s64 lc_max;
+ __s64 lc_sum;
__s64 lc_sumsquare;
- /*
- * Every counter has lc_array_sum[0], while lc_array_sum[1] is only
- * for irq context counter, i.e. stats with
- * LPROCFS_STATS_FLAG_IRQ_SAFE flag, its counter need
- * lc_array_sum[1]
- */
- __s64 lc_array_sum[1];
};
-#define lc_sum lc_array_sum[0]
-#define lc_sum_irq lc_array_sum[1]
struct lprocfs_percpu {
struct lprocfs_counter lp_cntr[0];
enum lprocfs_stats_flags {
LPROCFS_STATS_FLAG_NONE = 0x0000, /* per cpu counter */
LPROCFS_STATS_FLAG_NOPERCPU = 0x0001, /* need locking(no percpu area) */
- LPROCFS_STATS_FLAG_IRQ_SAFE = 0x0002, /* alloc need irq safe */
};
enum lprocfs_fields_flags {
percpusize = offsetof(struct lprocfs_percpu, lp_cntr[stats->ls_num]);
- /* irq safe stats need lc_array_sum[1] */
- if ((stats->ls_flags & LPROCFS_STATS_FLAG_IRQ_SAFE) != 0)
- percpusize += stats->ls_num * sizeof(__s64);
-
if ((stats->ls_flags & LPROCFS_STATS_FLAG_NOPERCPU) == 0)
percpusize = L1_CACHE_ALIGN(percpusize);
cntr = &stats->ls_percpu[cpuid]->lp_cntr[index];
- if ((stats->ls_flags & LPROCFS_STATS_FLAG_IRQ_SAFE) != 0)
- cntr = (void *)cntr + index * sizeof(__s64);
-
return cntr;
}
#include <linux/highmem.h>
#include <linux/slab.h>
#include <linux/types.h>
+#include <linux/percpu_counter.h>
#include <libcfs/libcfs.h>
#include <lnet/lib-cpt.h>
#include <lustre_handles.h>
/* global variables */
-extern struct lprocfs_stats *obd_memory;
-enum {
- OBD_MEMORY_STAT = 0,
- OBD_STATS_NUM,
-};
+extern struct percpu_counter obd_memory;
extern unsigned int obd_debug_peer_on_timeout;
extern unsigned int obd_dump_on_timeout;
extern atomic64_t libcfs_kmem;
-#ifdef CONFIG_PROC_FS
-#define obd_memory_add(size) \
- lprocfs_counter_add(obd_memory, OBD_MEMORY_STAT, (long)(size))
-#define obd_memory_sub(size) \
- lprocfs_counter_sub(obd_memory, OBD_MEMORY_STAT, (long)(size))
-#define obd_memory_sum() \
- lprocfs_stats_collector(obd_memory, OBD_MEMORY_STAT, \
- LPROCFS_FIELDS_FLAGS_SUM)
-
-extern void obd_update_maxusage(void);
-extern __u64 obd_memory_max(void);
-
-#else /* CONFIG_PROC_FS */
-
-extern __u64 obd_alloc;
+/* OBD_MEMORY_BATCH is the maximum error allowed per CPU core. Since
+ * obd_memory_sum() is calling percpu_counter_sum_positive(), it adds
+ * up the per-core local delta anyway, so the per-core batch size is
+ * can be large. This could be percpu_counter_add_local(), but that
+ * only exists in kernel 6.0 and later, and just uses a larger batch.
+ */
+#define OBD_MEMORY_BATCH (16 * 1024 * 1024)
-extern __u64 obd_max_alloc;
+#ifndef HAVE_PERCPU_COUNTER_ADD_BATCH
+#define percpu_counter_add_batch(fbc, amount, batch) \
+ __percpu_counter_add(fbc, amount, batch)
+#endif
-static inline void obd_memory_add(long size)
+static inline void obd_memory_add(size_t size)
{
- obd_alloc += size;
- if (obd_alloc > obd_max_alloc)
- obd_max_alloc = obd_alloc;
+ percpu_counter_add_batch(&obd_memory, size, OBD_MEMORY_BATCH);
}
-static inline void obd_memory_sub(long size)
+static inline void obd_memory_sub(size_t size)
{
- obd_alloc -= size;
+ percpu_counter_add_batch(&obd_memory, -size, OBD_MEMORY_BATCH);
}
-#define obd_memory_sum() (obd_alloc)
-
-#define obd_memory_max() (obd_max_alloc)
+static inline s64 obd_memory_sum(void)
+{
+ return percpu_counter_sum_positive(&obd_memory);
+}
-#endif /* !CONFIG_PROC_FS */
+extern void obd_update_maxusage(void);
+extern __u64 obd_memory_max(void);
#define OBD_DEBUG_MEMUSAGE (1)
#include "llog_internal.h"
#include <lustre_ioctl_old.h>
-#ifdef CONFIG_PROC_FS
static __u64 obd_max_alloc;
-#else
-__u64 obd_max_alloc;
-#endif
static DEFINE_SPINLOCK(obd_updatemax_lock);
int at_extra = 30;
EXPORT_SYMBOL(at_extra);
-#ifdef CONFIG_PROC_FS
-struct lprocfs_stats *obd_memory = NULL;
+struct percpu_counter obd_memory;
EXPORT_SYMBOL(obd_memory);
-#endif
static int obdclass_oom_handler(struct notifier_block *self,
unsigned long notused, void *nfreed)
{
-#ifdef CONFIG_PROC_FS
/* in bytes */
pr_info("obd_memory max: %llu, obd_memory current: %llu\n",
obd_memory_max(), obd_memory_sum());
-#endif /* CONFIG_PROC_FS */
return NOTIFY_OK;
}
if (err)
return err;
-#ifdef CONFIG_PROC_FS
- obd_memory = lprocfs_stats_alloc(OBD_STATS_NUM,
- LPROCFS_STATS_FLAG_NONE |
- LPROCFS_STATS_FLAG_IRQ_SAFE);
- if (obd_memory == NULL) {
- CERROR("kmalloc of 'obd_memory' failed\n");
- return -ENOMEM;
+ err = percpu_counter_init(&obd_memory, 0, GFP_KERNEL);
+ if (err < 0) {
+ CERROR("obdclass: initializing 'obd_memory' failed: rc = %d\n",
+ err);
+ return err;
}
- lprocfs_counter_init(obd_memory, OBD_MEMORY_STAT,
- LPROCFS_CNTR_AVGMINMAX | LPROCFS_TYPE_BYTES,
- "memused");
-#endif
err = libcfs_kkuc_init();
if (err)
goto cleanup_obd_memory;
err = misc_register(&obd_psdev);
if (err) {
- CERROR("cannot register OBD miscdevice: err = %d\n", err);
+ CERROR("cannot register OBD miscdevice: rc = %d\n", err);
goto cleanup_class_handle;
}
libcfs_kkuc_fini();
cleanup_obd_memory:
-#ifdef CONFIG_PROC_FS
- lprocfs_stats_free(&obd_memory);
-#endif
+ percpu_counter_destroy(&obd_memory);
unregister_oom_notifier(&obdclass_oom);
return err;
}
EXPORT_SYMBOL(obd_update_maxusage);
-#ifdef CONFIG_PROC_FS
__u64 obd_memory_max(void)
{
__u64 ret;
return ret;
}
-#endif /* CONFIG_PROC_FS */
+EXPORT_SYMBOL(obd_memory_max);
static void __exit obdclass_exit(void)
{
-#ifdef CONFIG_PROC_FS
__u64 memory_leaked;
__u64 memory_max;
-#endif /* CONFIG_PROC_FS */
ENTRY;
misc_deregister(&obd_psdev);
obd_zombie_impexp_stop();
libcfs_kkuc_fini();
-#ifdef CONFIG_PROC_FS
memory_leaked = obd_memory_sum();
memory_max = obd_memory_max();
- lprocfs_stats_free(&obd_memory);
+ percpu_counter_destroy(&obd_memory);
/* the below message is checked in test-framework.sh check_mem_leak() */
CDEBUG((memory_leaked) ? D_ERROR : D_INFO,
"obd_memory max: %llu, leaked: %llu\n",
memory_max, memory_leaked);
-#endif /* CONFIG_PROC_FS */
unregister_oom_notifier(&obdclass_oom);
* as memory allocation could trigger memory shrinker call
* ldlm_pool_shrink(), which calls lprocfs_counter_add().
* LU-1727.
- *
- * Only obd_memory uses LPROCFS_STATS_FLAG_IRQ_SAFE
- * flag, because it needs accurate counting lest memory leak
- * check reports error.
*/
- if (in_interrupt() &&
- (stats->ls_flags & LPROCFS_STATS_FLAG_IRQ_SAFE) != 0)
- percpu_cntr->lc_sum_irq += amount;
- else
- percpu_cntr->lc_sum += amount;
+ percpu_cntr->lc_sum += amount;
if (header->lc_config & LPROCFS_CNTR_STDDEV)
percpu_cntr->lc_sumsquare += (__s64)amount * amount;
* softirq context - right now that's the only case we're in
* softirq context here, use separate counter for that.
* bz20650.
- *
- * Only obd_memory uses LPROCFS_STATS_FLAG_IRQ_SAFE
- * flag, because it needs accurate counting lest memory leak
- * check reports error.
*/
- if (in_interrupt() &&
- (stats->ls_flags & LPROCFS_STATS_FLAG_IRQ_SAFE) != 0)
- percpu_cntr->lc_sum_irq -= amount;
- else
- percpu_cntr->lc_sum -= amount;
+ percpu_cntr->lc_sum -= amount;
}
lprocfs_stats_unlock(stats, LPROCFS_GET_SMP_ID, &flags);
}
unsigned long *flags)
{
if (stats->ls_flags & LPROCFS_STATS_FLAG_NOPERCPU) {
- if (stats->ls_flags & LPROCFS_STATS_FLAG_IRQ_SAFE)
- spin_lock_irqsave(&stats->ls_lock, *flags);
- else
- spin_lock(&stats->ls_lock);
+ spin_lock(&stats->ls_lock);
return opc == LPROCFS_GET_NUM_CPU ? 1 : 0;
}
unsigned long *flags)
{
if (stats->ls_flags & LPROCFS_STATS_FLAG_NOPERCPU) {
- if (stats->ls_flags & LPROCFS_STATS_FLAG_IRQ_SAFE)
- spin_unlock_irqrestore(&stats->ls_lock, *flags);
- else
- spin_unlock(&stats->ls_lock);
+ spin_unlock(&stats->ls_lock);
} else if (opc == LPROCFS_GET_SMP_ID) {
put_cpu();
}
struct lprocfs_counter *cntr;
unsigned int percpusize;
int rc = -ENOMEM;
- unsigned long flags = 0;
int i;
LASSERT(stats->ls_percpu[cpuid] == NULL);
if (stats->ls_percpu[cpuid]) {
rc = 0;
if (unlikely(stats->ls_biggest_alloc_num <= cpuid)) {
- if (stats->ls_flags & LPROCFS_STATS_FLAG_IRQ_SAFE)
- spin_lock_irqsave(&stats->ls_lock, flags);
- else
- spin_lock(&stats->ls_lock);
+ spin_lock(&stats->ls_lock);
if (stats->ls_biggest_alloc_num <= cpuid)
stats->ls_biggest_alloc_num = cpuid + 1;
- if (stats->ls_flags & LPROCFS_STATS_FLAG_IRQ_SAFE) {
- spin_unlock_irqrestore(&stats->ls_lock, flags);
- } else {
- spin_unlock(&stats->ls_lock);
- }
+ spin_unlock(&stats->ls_lock);
}
/* initialize the ls_percpu[cpuid] non-zero counter */
for (i = 0; i < stats->ls_num; ++i) {
struct lprocfs_stats *stats;
unsigned int num_entry;
unsigned int percpusize = 0;
- int i;
if (num == 0)
return NULL;
if (!stats->ls_percpu[0])
goto fail;
stats->ls_biggest_alloc_num = 1;
- } else if ((flags & LPROCFS_STATS_FLAG_IRQ_SAFE) != 0) {
- /* alloc all percpu data, currently only obd_memory use this */
- for (i = 0; i < num_entry; ++i)
- if (lprocfs_stats_alloc_one(stats, i) < 0)
- goto fail;
}
return stats;
percpu_cntr->lc_max = 0;
percpu_cntr->lc_sumsquare = 0;
percpu_cntr->lc_sum = 0;
- if (stats->ls_flags & LPROCFS_STATS_FLAG_IRQ_SAFE)
- percpu_cntr->lc_sum_irq = 0;
}
}
stats->ls_init = ktime_get_real();
percpu_cntr->lc_max = 0;
percpu_cntr->lc_sumsquare = 0;
percpu_cntr->lc_sum = 0;
- if ((stats->ls_flags & LPROCFS_STATS_FLAG_IRQ_SAFE) != 0)
- percpu_cntr->lc_sum_irq = 0;
}
lprocfs_stats_unlock(stats, LPROCFS_GET_NUM_CPU, &flags);
}
break;
case LPROCFS_FIELDS_FLAGS_SUM:
ret = lc->lc_sum;
- if ((flags & LPROCFS_STATS_FLAG_IRQ_SAFE) != 0)
- ret += lc->lc_sum_irq;
break;
case LPROCFS_FIELDS_FLAGS_MIN:
ret = lc->lc_min;
ret = lc->lc_max;
break;
case LPROCFS_FIELDS_FLAGS_AVG:
- ret = div64_u64((flags & LPROCFS_STATS_FLAG_IRQ_SAFE ?
- lc->lc_sum_irq : 0) + lc->lc_sum,
- lc->lc_count);
+ ret = div64_u64(lc->lc_sum, lc->lc_count);
break;
case LPROCFS_FIELDS_FLAGS_SUMSQUARE:
ret = lc->lc_sumsquare;
git://git.whamcloud.com / fs / lustre-release.git / commitdiff