LU-13134 obdclass: use slab allocation for cl_dio_aio

cl_dio_aio is used frequently for dio/aio, try to use
a private slab pool for it.

This could help improve aio performance.

Change-Id: Ic06523ae59eed04e55c17ac03af9187af8f695c5
Signed-off-by: Wang Shilong <wshilong@ddn.com>
Reviewed-on: https://review.whamcloud.com/37227
Reviewed-by: Patrick Farrell <farr0186@gmail.com>
Tested-by: jenkins <devops@whamcloud.com>
Tested-by: Maloo <maloo@whamcloud.com>
Reviewed-by: Andreas Dilger <adilger@whamcloud.com>
Reviewed-by: Oleg Drokin <green@whamcloud.com>

diff --git a/lustre/include/cl_object.h b/lustre/include/cl_object.h
index 7979b14..3e31227 100644 (file)
--- a/lustre/include/cl_object.h
+++ b/lustre/include/cl_object.h
@@ -2495,6 +2495,7 @@ int  cl_sync_io_wait(const struct lu_env *env, struct cl_sync_io *anchor,
                     long timeout);
 void cl_sync_io_note(const struct lu_env *env, struct cl_sync_io *anchor,
                     int ioret);
+struct cl_dio_aio *cl_aio_alloc(struct kiocb *iocb);
 static inline void cl_sync_io_init(struct cl_sync_io *anchor, int nr)
 {
        cl_sync_io_init_notify(anchor, nr, NULL, NULL);

diff --git a/lustre/llite/rw26.c b/lustre/llite/rw26.c
index 8ea28c1..c719dae 100644 (file)
--- a/lustre/llite/rw26.c
+++ b/lustre/llite/rw26.c
@@ -287,14 +287,6 @@ static unsigned long ll_iov_iter_alignment(const struct iov_iter *i)
 }
 #endif
 
-#ifndef HAVE_AIO_COMPLETE
-static inline void aio_complete(struct kiocb *iocb, ssize_t res, ssize_t res2)
-{
-       if (iocb->ki_complete)
-               iocb->ki_complete(iocb, res, res2);
-}
-#endif
-
 /** direct IO pages */
 struct ll_dio_pages {
        struct cl_dio_aio       *ldp_aio;
@@ -309,47 +301,6 @@ struct ll_dio_pages {
        loff_t                  ldp_file_offset;
 };
 
-static void ll_aio_end(const struct lu_env *env, struct cl_sync_io *anchor)
-{
-       struct cl_dio_aio *aio = container_of(anchor, typeof(*aio), cda_sync);
-       ssize_t ret = anchor->csi_sync_rc;
-
-       ENTRY;
-
-       /* release pages */
-       while (aio->cda_pages.pl_nr > 0) {
-               struct cl_page *page = cl_page_list_first(&aio->cda_pages);
-
-               cl_page_get(page);
-               cl_page_list_del(env, &aio->cda_pages, page);
-               cl_page_delete(env, page);
-               cl_page_put(env, page);
-       }
-
-       if (!is_sync_kiocb(aio->cda_iocb))
-               aio_complete(aio->cda_iocb, ret ?: aio->cda_bytes, 0);
-
-       EXIT;
-}
-
-static struct cl_dio_aio *ll_aio_alloc(struct kiocb *iocb)
-{
-       struct cl_dio_aio *aio;
-
-       OBD_ALLOC_PTR(aio);
-       if (aio != NULL) {
-               /*
-                * Hold one ref so that it won't be released until
-                * every pages is added.
-                */
-               cl_sync_io_init_notify(&aio->cda_sync, 1, is_sync_kiocb(iocb) ?
-                                      NULL : aio, ll_aio_end);
-               cl_page_list_init(&aio->cda_pages);
-               aio->cda_iocb = iocb;
-       }
-       return aio;
-}
-
 static int
 ll_direct_rw_pages(const struct lu_env *env, struct cl_io *io, size_t size,
                   int rw, struct inode *inode, struct ll_dio_pages *pv)
@@ -480,7 +431,7 @@ ll_direct_IO_impl(struct kiocb *iocb, struct iov_iter *iter, int rw)
        io = lcc->lcc_io;
        LASSERT(io != NULL);
 
-       aio = ll_aio_alloc(iocb);
+       aio = cl_aio_alloc(iocb);
        if (!aio)
                RETURN(-ENOMEM);
 

diff --git a/lustre/obdclass/cl_internal.h b/lustre/obdclass/cl_internal.h
index 0c1276d..8ef48ef 100644 (file)
--- a/lustre/obdclass/cl_internal.h
+++ b/lustre/obdclass/cl_internal.h
@@ -46,6 +46,8 @@ struct cl_thread_info {
        struct cl_sync_io clt_anchor;
 };
 
+extern struct kmem_cache *cl_dio_aio_kmem;
+
 struct cl_thread_info *cl_env_info(const struct lu_env *env);
 void cl_page_disown0(const struct lu_env *env,
                     struct cl_io *io, struct cl_page *pg);

diff --git a/lustre/obdclass/cl_io.c b/lustre/obdclass/cl_io.c
index 93a2657..45d9307 100644 (file)
--- a/lustre/obdclass/cl_io.c
+++ b/lustre/obdclass/cl_io.c
@@ -1172,6 +1172,57 @@ int cl_sync_io_wait(const struct lu_env *env, struct cl_sync_io *anchor,
 }
 EXPORT_SYMBOL(cl_sync_io_wait);
 
+#ifndef HAVE_AIO_COMPLETE
+static inline void aio_complete(struct kiocb *iocb, ssize_t res, ssize_t res2)
+{
+       if (iocb->ki_complete)
+               iocb->ki_complete(iocb, res, res2);
+}
+#endif
+
+static void cl_aio_end(const struct lu_env *env, struct cl_sync_io *anchor)
+{
+       struct cl_dio_aio *aio = container_of(anchor, typeof(*aio), cda_sync);
+       ssize_t ret = anchor->csi_sync_rc;
+
+       ENTRY;
+
+       /* release pages */
+       while (aio->cda_pages.pl_nr > 0) {
+               struct cl_page *page = cl_page_list_first(&aio->cda_pages);
+
+               cl_page_get(page);
+               cl_page_list_del(env, &aio->cda_pages, page);
+               cl_page_delete(env, page);
+               cl_page_put(env, page);
+       }
+
+       if (!is_sync_kiocb(aio->cda_iocb))
+               aio_complete(aio->cda_iocb, ret ?: aio->cda_bytes, 0);
+
+       EXIT;
+}
+
+struct cl_dio_aio *cl_aio_alloc(struct kiocb *iocb)
+{
+       struct cl_dio_aio *aio;
+
+       OBD_SLAB_ALLOC_PTR_GFP(aio, cl_dio_aio_kmem, GFP_NOFS);
+       if (aio != NULL) {
+               /*
+                * Hold one ref so that it won't be released until
+                * every pages is added.
+                */
+               cl_sync_io_init_notify(&aio->cda_sync, 1, is_sync_kiocb(iocb) ?
+                                      NULL : aio, cl_aio_end);
+               cl_page_list_init(&aio->cda_pages);
+               aio->cda_iocb = iocb;
+       }
+       return aio;
+}
+EXPORT_SYMBOL(cl_aio_alloc);
+
+
 /**
  * Indicate that transfer of a single page completed.
  */
@@ -1214,7 +1265,7 @@ void cl_sync_io_note(const struct lu_env *env, struct cl_sync_io *anchor,
                 * memory here rather than when cl_sync_io_wait() completes.
                 */
                if (aio)
-                       OBD_FREE_PTR(aio);
+                       OBD_SLAB_FREE_PTR(aio, cl_dio_aio_kmem);
        }
        EXIT;
 }

diff --git a/lustre/obdclass/cl_object.c b/lustre/obdclass/cl_object.c
index ebfc7f2..1e39da3 100644 (file)
--- a/lustre/obdclass/cl_object.c
+++ b/lustre/obdclass/cl_object.c
@@ -57,6 +57,7 @@
 #include "cl_internal.h"
 
 static struct kmem_cache *cl_env_kmem;
+struct kmem_cache *cl_dio_aio_kmem;
 
 /** Lock class of cl_object_header::coh_attr_guard */
 static struct lock_class_key cl_attr_guard_class;
@@ -1046,14 +1047,19 @@ static struct lu_context_key cl_key = {
 };
 
 static struct lu_kmem_descr cl_object_caches[] = {
-        {
-                .ckd_cache = &cl_env_kmem,
-                .ckd_name  = "cl_env_kmem",
-                .ckd_size  = sizeof (struct cl_env)
-        },
-        {
-                .ckd_cache = NULL
-        }
+       {
+               .ckd_cache = &cl_env_kmem,
+               .ckd_name  = "cl_env_kmem",
+               .ckd_size  = sizeof(struct cl_env)
+       },
+       {
+               .ckd_cache = &cl_dio_aio_kmem,
+               .ckd_name  = "cl_dio_aio_kmem",
+               .ckd_size  = sizeof(struct cl_dio_aio)
+       },
+       {
+               .ckd_cache = NULL
+       }
 };
 
 /**