LU-12564 libcfs: Use vfree_atomic instead of vfree

Since vfree is unsafe to use in atomic context, implement our own
libcfs_vfree_atomic heavily based on code from linux 4.10 commit
bf22e37a641327e34681b7b6959d9646e3886770

We can't use the one in the kernel because it's not exported.

Unconditionally use it in *_FREE_LARGE() macros since in_atomic()
is not recommended to be used outside of core kernel code.

Not everything is present on 3.10 (RHEL7) so we also add
llist primitive and a replacement for raw_cpu_ptr there.
For RHEL6 the llist support us also missing, so just
stick with the old vfree() call that was always used.

Lustre-change: https://review.whamcloud.com/40136
Lustre-commit: 7a9c0ca690eb00a6e314322b62fb1fd3e9b31f0e

Change-Id: I50892f231e54a284f4d8a14d910ea9ab2fbe6a16
Signed-off-by: Oleg Drokin <green@whamcloud.com>
Reviewed-on: https://review.whamcloud.com/40110
Tested-by: jenkins <devops@whamcloud.com>
Reviewed-by: Andreas Dilger <adilger@whamcloud.com>
Tested-by: Maloo <maloo@whamcloud.com>
Reviewed-by: Neil Brown <neilb@suse.de>
Reviewed-by: Aurelien Degremont <degremoa@amazon.com>

diff --git a/libcfs/include/libcfs/libcfs.h b/libcfs/include/libcfs/libcfs.h
index c09b779..2747f97 100644 (file)
--- a/libcfs/include/libcfs/libcfs.h
+++ b/libcfs/include/libcfs/libcfs.h
@@ -149,4 +149,11 @@ int lprocfs_call_handler(void *data, int write, loff_t *ppos,
                         int (*handler)(void *data, int write, loff_t pos,
                                        void __user *buffer, int len));
 
+/* atomic-context safe vfree */
+#ifdef HAVE_LIBCFS_VFREE_ATOMIC
+void libcfs_vfree_atomic(const void *addr);
+#else
+#define libcfs_vfree_atomic(ptr) vfree(ptr)
+#endif
+
 #endif /* _LIBCFS_LIBCFS_H_ */

diff --git a/libcfs/include/libcfs/libcfs_private.h b/libcfs/include/libcfs/libcfs_private.h
index 4931e70..9a24283 100644 (file)
--- a/libcfs/include/libcfs/libcfs_private.h
+++ b/libcfs/include/libcfs/libcfs_private.h
@@ -216,6 +216,15 @@ do {                                                                           \
 #define LIBCFS_CPT_ALLOC(ptr, cptab, cpt, size)                                    \
        LIBCFS_CPT_ALLOC_GFP(ptr, cptab, cpt, size, GFP_NOFS)
 
+#ifdef LLIST_HEAD
+void init_libcfs_vfree_atomic(void);
+void exit_libcfs_vfree_atomic(void);
+#define HAVE_LIBCFS_VFREE_ATOMIC
+#else
+#define init_libcfs_vfree_atomic() do {} while(0)
+#define exit_libcfs_vfree_atomic() do {} while(0)
+#endif
+
 #define LIBCFS_FREE(ptr, size)                                         \
 do {                                                                   \
        int s = (size);                                                 \
@@ -228,7 +237,7 @@ do {                                                                        \
        CDEBUG(D_MALLOC, "kfreed '" #ptr "': %d at %p (tot %d).\n",     \
               s, (ptr), libcfs_kmem_read());                           \
        if (unlikely(s > LIBCFS_VMALLOC_SIZE))                          \
-               vfree(ptr);                                             \
+               libcfs_vfree_atomic(ptr);                               \
        else                                                            \
                kfree(ptr);                                             \
 } while (0)

diff --git a/libcfs/libcfs/libcfs_mem.c b/libcfs/libcfs/libcfs_mem.c
index 3e83f50..5f85219 100644 (file)
--- a/libcfs/libcfs/libcfs_mem.c
+++ b/libcfs/libcfs/libcfs_mem.c
@@ -170,3 +170,66 @@ cfs_array_alloc(int count, unsigned int size)
        return (void *)&arr->va_ptrs[0];
 }
 EXPORT_SYMBOL(cfs_array_alloc);
+
+#ifdef HAVE_LIBCFS_VFREE_ATOMIC
+#include <linux/workqueue.h>
+/*
+ * This is opencoding of vfree_atomic from Linux kernel added in 4.10 with
+ * minimum changes needed to work on some older kernels too.
+ * For RHEL6, just use vfree() directly since it is missing too much code.
+ */
+
+#ifndef raw_cpu_ptr
+#define raw_cpu_ptr(p) __this_cpu_ptr(p)
+#endif
+
+#ifndef llist_for_each_safe
+#define llist_for_each_safe(pos, n, node)                       \
+       for ((pos) = (node); (pos) && ((n) = (pos)->next, true); (pos) = (n))
+#endif
+
+struct vfree_deferred {
+       struct llist_head list;
+       struct work_struct wq;
+};
+static DEFINE_PER_CPU(struct vfree_deferred, vfree_deferred);
+
+static void free_work(struct work_struct *w)
+{
+       struct vfree_deferred *p = container_of(w, struct vfree_deferred, wq);
+       struct llist_node *t, *llnode;
+
+       llist_for_each_safe(llnode, t, llist_del_all(&p->list))
+               vfree((void *)llnode);
+}
+
+void libcfs_vfree_atomic(const void *addr)
+{
+       struct vfree_deferred *p = raw_cpu_ptr(&vfree_deferred);
+
+       if (!addr)
+               return;
+
+       if (llist_add((struct llist_node *)addr, &p->list))
+               schedule_work(&p->wq);
+}
+EXPORT_SYMBOL(libcfs_vfree_atomic);
+
+void __init init_libcfs_vfree_atomic(void)
+{
+       int i;
+
+       for_each_possible_cpu(i) {
+               struct vfree_deferred *p;
+
+               p = &per_cpu(vfree_deferred, i);
+               init_llist_head(&p->list);
+               INIT_WORK(&p->wq, free_work);
+       }
+}
+
+void __exit exit_libcfs_vfree_atomic(void)
+{
+       flush_scheduled_work();
+}
+#endif /* HAVE_LIBCFS_VFREE_ATOMIC */

diff --git a/libcfs/libcfs/module.c b/libcfs/libcfs/module.c
index 92d410b..7f8eb24 100644 (file)
--- a/libcfs/libcfs/module.c
+++ b/libcfs/libcfs/module.c
@@ -601,6 +601,8 @@ static int __init libcfs_init(void)
 #ifndef HAVE_WAIT_VAR_EVENT
        wait_bit_init();
 #endif
+       init_libcfs_vfree_atomic();
+
        rc = libcfs_debug_init(5 * 1024 * 1024);
        if (rc < 0) {
                printk(KERN_ERR "LustreError: libcfs_debug_init: %d\n", rc);
@@ -688,6 +690,8 @@ static void __exit libcfs_exit(void)
        if (rc)
                printk(KERN_ERR "LustreError: libcfs_debug_cleanup: %d\n",
                       rc);
+
+       exit_libcfs_vfree_atomic();
 }
 
 MODULE_AUTHOR("OpenSFS, Inc. <http://www.lustre.org/>");

diff --git a/lustre/include/obd_support.h b/lustre/include/obd_support.h
index 84e5be7..56dfc54 100644 (file)
--- a/lustre/include/obd_support.h
+++ b/lustre/include/obd_support.h
@@ -872,7 +872,7 @@ do {                                                                              \
 do {                                                                         \
        if (is_vmalloc_addr(ptr)) {                                           \
                OBD_FREE_PRE(ptr, size, "vfreed");                            \
-               vfree(ptr);                                                   \
+               libcfs_vfree_atomic(ptr);                                     \
                POISON_PTR(ptr);                                              \
        } else {                                                              \
                OBD_FREE(ptr, size);                                          \