From: Oleg Drokin Date: Sun, 6 Dec 2020 05:40:24 +0000 (-0500) Subject: LU-12564 libcfs: Use vfree_atomic instead of vfree X-Git-Tag: 2.12.7-RC1~61 X-Git-Url: https://git.whamcloud.com/?a=commitdiff_plain;ds=sidebyside;h=1dc31de15ff270befceb64257bd84f15ffe75000;p=fs%2Flustre-release.git LU-12564 libcfs: Use vfree_atomic instead of vfree Since vfree is unsafe to use in atomic context, implement our own libcfs_vfree_atomic heavily based on code from linux 4.10 commit bf22e37a641327e34681b7b6959d9646e3886770 We can't use the one in the kernel because it's not exported. Unconditionally use it in *_FREE_LARGE() macros since in_atomic() is not recommended to be used outside of core kernel code. Not everything is present on 3.10 (RHEL7) so we also add llist primitive and a replacement for raw_cpu_ptr there. For RHEL6 the llist support us also missing, so just stick with the old vfree() call that was always used. Lustre-change: https://review.whamcloud.com/40136 Lustre-commit: 7a9c0ca690eb00a6e314322b62fb1fd3e9b31f0e Change-Id: I50892f231e54a284f4d8a14d910ea9ab2fbe6a16 Signed-off-by: Oleg Drokin Reviewed-on: https://review.whamcloud.com/40110 Tested-by: jenkins Reviewed-by: Andreas Dilger Tested-by: Maloo Reviewed-by: Neil Brown Reviewed-by: Aurelien Degremont --- diff --git a/libcfs/include/libcfs/libcfs.h b/libcfs/include/libcfs/libcfs.h index c09b779..2747f97 100644 --- a/libcfs/include/libcfs/libcfs.h +++ b/libcfs/include/libcfs/libcfs.h @@ -149,4 +149,11 @@ int lprocfs_call_handler(void *data, int write, loff_t *ppos, int (*handler)(void *data, int write, loff_t pos, void __user *buffer, int len)); +/* atomic-context safe vfree */ +#ifdef HAVE_LIBCFS_VFREE_ATOMIC +void libcfs_vfree_atomic(const void *addr); +#else +#define libcfs_vfree_atomic(ptr) vfree(ptr) +#endif + #endif /* _LIBCFS_LIBCFS_H_ */ diff --git a/libcfs/include/libcfs/libcfs_private.h b/libcfs/include/libcfs/libcfs_private.h index 4931e70..9a24283 100644 --- a/libcfs/include/libcfs/libcfs_private.h +++ b/libcfs/include/libcfs/libcfs_private.h @@ -216,6 +216,15 @@ do { \ #define LIBCFS_CPT_ALLOC(ptr, cptab, cpt, size) \ LIBCFS_CPT_ALLOC_GFP(ptr, cptab, cpt, size, GFP_NOFS) +#ifdef LLIST_HEAD +void init_libcfs_vfree_atomic(void); +void exit_libcfs_vfree_atomic(void); +#define HAVE_LIBCFS_VFREE_ATOMIC +#else +#define init_libcfs_vfree_atomic() do {} while(0) +#define exit_libcfs_vfree_atomic() do {} while(0) +#endif + #define LIBCFS_FREE(ptr, size) \ do { \ int s = (size); \ @@ -228,7 +237,7 @@ do { \ CDEBUG(D_MALLOC, "kfreed '" #ptr "': %d at %p (tot %d).\n", \ s, (ptr), libcfs_kmem_read()); \ if (unlikely(s > LIBCFS_VMALLOC_SIZE)) \ - vfree(ptr); \ + libcfs_vfree_atomic(ptr); \ else \ kfree(ptr); \ } while (0) diff --git a/libcfs/libcfs/libcfs_mem.c b/libcfs/libcfs/libcfs_mem.c index 3e83f50..5f85219 100644 --- a/libcfs/libcfs/libcfs_mem.c +++ b/libcfs/libcfs/libcfs_mem.c @@ -170,3 +170,66 @@ cfs_array_alloc(int count, unsigned int size) return (void *)&arr->va_ptrs[0]; } EXPORT_SYMBOL(cfs_array_alloc); + +#ifdef HAVE_LIBCFS_VFREE_ATOMIC +#include +/* + * This is opencoding of vfree_atomic from Linux kernel added in 4.10 with + * minimum changes needed to work on some older kernels too. + * For RHEL6, just use vfree() directly since it is missing too much code. + */ + +#ifndef raw_cpu_ptr +#define raw_cpu_ptr(p) __this_cpu_ptr(p) +#endif + +#ifndef llist_for_each_safe +#define llist_for_each_safe(pos, n, node) \ + for ((pos) = (node); (pos) && ((n) = (pos)->next, true); (pos) = (n)) +#endif + +struct vfree_deferred { + struct llist_head list; + struct work_struct wq; +}; +static DEFINE_PER_CPU(struct vfree_deferred, vfree_deferred); + +static void free_work(struct work_struct *w) +{ + struct vfree_deferred *p = container_of(w, struct vfree_deferred, wq); + struct llist_node *t, *llnode; + + llist_for_each_safe(llnode, t, llist_del_all(&p->list)) + vfree((void *)llnode); +} + +void libcfs_vfree_atomic(const void *addr) +{ + struct vfree_deferred *p = raw_cpu_ptr(&vfree_deferred); + + if (!addr) + return; + + if (llist_add((struct llist_node *)addr, &p->list)) + schedule_work(&p->wq); +} +EXPORT_SYMBOL(libcfs_vfree_atomic); + +void __init init_libcfs_vfree_atomic(void) +{ + int i; + + for_each_possible_cpu(i) { + struct vfree_deferred *p; + + p = &per_cpu(vfree_deferred, i); + init_llist_head(&p->list); + INIT_WORK(&p->wq, free_work); + } +} + +void __exit exit_libcfs_vfree_atomic(void) +{ + flush_scheduled_work(); +} +#endif /* HAVE_LIBCFS_VFREE_ATOMIC */ diff --git a/libcfs/libcfs/module.c b/libcfs/libcfs/module.c index 92d410b..7f8eb24 100644 --- a/libcfs/libcfs/module.c +++ b/libcfs/libcfs/module.c @@ -601,6 +601,8 @@ static int __init libcfs_init(void) #ifndef HAVE_WAIT_VAR_EVENT wait_bit_init(); #endif + init_libcfs_vfree_atomic(); + rc = libcfs_debug_init(5 * 1024 * 1024); if (rc < 0) { printk(KERN_ERR "LustreError: libcfs_debug_init: %d\n", rc); @@ -688,6 +690,8 @@ static void __exit libcfs_exit(void) if (rc) printk(KERN_ERR "LustreError: libcfs_debug_cleanup: %d\n", rc); + + exit_libcfs_vfree_atomic(); } MODULE_AUTHOR("OpenSFS, Inc. "); diff --git a/lustre/include/obd_support.h b/lustre/include/obd_support.h index 84e5be7..56dfc54 100644 --- a/lustre/include/obd_support.h +++ b/lustre/include/obd_support.h @@ -872,7 +872,7 @@ do { \ do { \ if (is_vmalloc_addr(ptr)) { \ OBD_FREE_PRE(ptr, size, "vfreed"); \ - vfree(ptr); \ + libcfs_vfree_atomic(ptr); \ POISON_PTR(ptr); \ } else { \ OBD_FREE(ptr, size); \