From 7a9c0ca690eb00a6e314322b62fb1fd3e9b31f0e Mon Sep 17 00:00:00 2001 From: Oleg Drokin Date: Thu, 1 Oct 2020 01:51:55 -0400 Subject: [PATCH] LU-12564 libcfs: Use vfree_atomic instead of vfree Since vfree is unsafe to use in atomic context, implement our own libcfs_vfree_atomic heavily based on code from linux 4.10 commit bf22e37a641327e34681b7b6959d9646e3886770 We can't use the one in the kernel because it's not exported. unconditionally use it in *_FREE_LARGE() macros since in_atomic() is not recommended to be used outside of core kernel code. Not everything is present on 3.10 (rhel7) so we also add llist primitive and a replacement for raw_cpu_ptr there. Change-Id: I50892f231e54a284f4d8a14d910ea9ab2fbe6a16 Signed-off-by: Oleg Drokin Reviewed-on: https://review.whamcloud.com/40136 Reviewed-by: Andreas Dilger Reviewed-by: Aurelien Degremont Tested-by: jenkins Reviewed-by: Neil Brown Tested-by: Maloo --- libcfs/include/libcfs/libcfs.h | 3 ++ libcfs/include/libcfs/libcfs_private.h | 5 ++- libcfs/libcfs/libcfs_mem.c | 60 ++++++++++++++++++++++++++++++++++ libcfs/libcfs/module.c | 4 +++ lustre/include/obd_support.h | 2 +- 5 files changed, 72 insertions(+), 2 deletions(-) diff --git a/libcfs/include/libcfs/libcfs.h b/libcfs/include/libcfs/libcfs.h index a172dda..2bbc8206 100644 --- a/libcfs/include/libcfs/libcfs.h +++ b/libcfs/include/libcfs/libcfs.h @@ -129,4 +129,7 @@ do { \ ); \ } while (0) +/* atomic-context safe vfree */ +void libcfs_vfree_atomic(const void *addr); + #endif /* _LIBCFS_LIBCFS_H_ */ diff --git a/libcfs/include/libcfs/libcfs_private.h b/libcfs/include/libcfs/libcfs_private.h index cbc003d..9c48a13 100644 --- a/libcfs/include/libcfs/libcfs_private.h +++ b/libcfs/include/libcfs/libcfs_private.h @@ -214,6 +214,9 @@ do { \ #define LIBCFS_CPT_ALLOC(ptr, cptab, cpt, size) \ LIBCFS_CPT_ALLOC_GFP(ptr, cptab, cpt, size, GFP_NOFS) +void init_libcfs_vfree_atomic(void); +void exit_libcfs_vfree_atomic(void); + #define LIBCFS_FREE(ptr, size) \ do { \ int s = (size); \ @@ -226,7 +229,7 @@ do { \ CDEBUG(D_MALLOC, "kfreed '" #ptr "': %d at %p (tot %lld).\n", \ s, (ptr), libcfs_kmem_read()); \ if (unlikely(s > LIBCFS_VMALLOC_SIZE)) \ - vfree(ptr); \ + libcfs_vfree_atomic(ptr); \ else \ kfree(ptr); \ } while (0) diff --git a/libcfs/libcfs/libcfs_mem.c b/libcfs/libcfs/libcfs_mem.c index 3e83f50..757cc19 100644 --- a/libcfs/libcfs/libcfs_mem.c +++ b/libcfs/libcfs/libcfs_mem.c @@ -33,6 +33,7 @@ #define DEBUG_SUBSYSTEM S_LNET +#include #include struct cfs_var_array { @@ -170,3 +171,62 @@ cfs_array_alloc(int count, unsigned int size) return (void *)&arr->va_ptrs[0]; } EXPORT_SYMBOL(cfs_array_alloc); + +/* + * This is opencoding of vfree_atomic from Linux kernel added in 4.10 with + * minimum changes needed to work on older kernels too. + */ + +#ifndef raw_cpu_ptr +#define raw_cpu_ptr(p) __this_cpu_ptr(p) +#endif + +#ifndef llist_for_each_safe +#define llist_for_each_safe(pos, n, node) \ + for ((pos) = (node); (pos) && ((n) = (pos)->next, true); (pos) = (n)) +#endif + +struct vfree_deferred { + struct llist_head list; + struct work_struct wq; +}; +static DEFINE_PER_CPU(struct vfree_deferred, vfree_deferred); + +static void free_work(struct work_struct *w) +{ + struct vfree_deferred *p = container_of(w, struct vfree_deferred, wq); + struct llist_node *t, *llnode; + + llist_for_each_safe(llnode, t, llist_del_all(&p->list)) + vfree((void *)llnode); +} + +void libcfs_vfree_atomic(const void *addr) +{ + struct vfree_deferred *p = raw_cpu_ptr(&vfree_deferred); + + if (!addr) + return; + + if (llist_add((struct llist_node *)addr, &p->list)) + schedule_work(&p->wq); +} +EXPORT_SYMBOL(libcfs_vfree_atomic); + +void __init init_libcfs_vfree_atomic(void) +{ + int i; + + for_each_possible_cpu(i) { + struct vfree_deferred *p; + + p = &per_cpu(vfree_deferred, i); + init_llist_head(&p->list); + INIT_WORK(&p->wq, free_work); + } +} + +void __exit exit_libcfs_vfree_atomic(void) +{ + flush_scheduled_work(); +} diff --git a/libcfs/libcfs/module.c b/libcfs/libcfs/module.c index ca2a797..51a3ce4 100644 --- a/libcfs/libcfs/module.c +++ b/libcfs/libcfs/module.c @@ -721,6 +721,8 @@ static int __init libcfs_init(void) cfs_arch_init(); + init_libcfs_vfree_atomic(); + rc = libcfs_debug_init(5 * 1024 * 1024); if (rc < 0) { pr_err("LustreError: libcfs_debug_init: rc = %d\n", rc); @@ -813,6 +815,8 @@ static void __exit libcfs_exit(void) rc = libcfs_debug_cleanup(); if (rc) pr_err("LustreError: libcfs_debug_cleanup: rc = %d\n", rc); + + exit_libcfs_vfree_atomic(); } MODULE_AUTHOR("OpenSFS, Inc. "); diff --git a/lustre/include/obd_support.h b/lustre/include/obd_support.h index 462ead5..847c443 100644 --- a/lustre/include/obd_support.h +++ b/lustre/include/obd_support.h @@ -892,7 +892,7 @@ do { \ if (is_vmalloc_addr(ptr)) { \ OBD_FREE_PRE(ptr, size, "vfreed"); \ POISON(ptr, 0x5a, size); \ - vfree(ptr); \ + libcfs_vfree_atomic(ptr); \ POISON_PTR(ptr); \ } else { \ OBD_FREE(ptr, size); \ -- 1.8.3.1