From 3db1d3322eb0d4271496c6fea169fb10fa500ebb Mon Sep 17 00:00:00 2001 From: Wang Shilong Date: Tue, 14 Jan 2020 23:00:03 +0800 Subject: [PATCH 1/1] LU-13134 obdclass: use slab allocation for cl_dio_aio cl_dio_aio is used frequently for dio/aio, try to use a private slab pool for it. This could help improve aio performance. Change-Id: Ic06523ae59eed04e55c17ac03af9187af8f695c5 Signed-off-by: Wang Shilong Reviewed-on: https://review.whamcloud.com/37227 Reviewed-by: Patrick Farrell Tested-by: jenkins Tested-by: Maloo Reviewed-by: Andreas Dilger Reviewed-by: Oleg Drokin --- lustre/include/cl_object.h | 1 + lustre/llite/rw26.c | 51 +---------------------------------------- lustre/obdclass/cl_internal.h | 2 ++ lustre/obdclass/cl_io.c | 53 ++++++++++++++++++++++++++++++++++++++++++- lustre/obdclass/cl_object.c | 22 +++++++++++------- 5 files changed, 70 insertions(+), 59 deletions(-) diff --git a/lustre/include/cl_object.h b/lustre/include/cl_object.h index 7979b14..3e31227 100644 --- a/lustre/include/cl_object.h +++ b/lustre/include/cl_object.h @@ -2495,6 +2495,7 @@ int cl_sync_io_wait(const struct lu_env *env, struct cl_sync_io *anchor, long timeout); void cl_sync_io_note(const struct lu_env *env, struct cl_sync_io *anchor, int ioret); +struct cl_dio_aio *cl_aio_alloc(struct kiocb *iocb); static inline void cl_sync_io_init(struct cl_sync_io *anchor, int nr) { cl_sync_io_init_notify(anchor, nr, NULL, NULL); diff --git a/lustre/llite/rw26.c b/lustre/llite/rw26.c index 8ea28c1..c719dae 100644 --- a/lustre/llite/rw26.c +++ b/lustre/llite/rw26.c @@ -287,14 +287,6 @@ static unsigned long ll_iov_iter_alignment(const struct iov_iter *i) } #endif -#ifndef HAVE_AIO_COMPLETE -static inline void aio_complete(struct kiocb *iocb, ssize_t res, ssize_t res2) -{ - if (iocb->ki_complete) - iocb->ki_complete(iocb, res, res2); -} -#endif - /** direct IO pages */ struct ll_dio_pages { struct cl_dio_aio *ldp_aio; @@ -309,47 +301,6 @@ struct ll_dio_pages { loff_t ldp_file_offset; }; -static void ll_aio_end(const struct lu_env *env, struct cl_sync_io *anchor) -{ - struct cl_dio_aio *aio = container_of(anchor, typeof(*aio), cda_sync); - ssize_t ret = anchor->csi_sync_rc; - - ENTRY; - - /* release pages */ - while (aio->cda_pages.pl_nr > 0) { - struct cl_page *page = cl_page_list_first(&aio->cda_pages); - - cl_page_get(page); - cl_page_list_del(env, &aio->cda_pages, page); - cl_page_delete(env, page); - cl_page_put(env, page); - } - - if (!is_sync_kiocb(aio->cda_iocb)) - aio_complete(aio->cda_iocb, ret ?: aio->cda_bytes, 0); - - EXIT; -} - -static struct cl_dio_aio *ll_aio_alloc(struct kiocb *iocb) -{ - struct cl_dio_aio *aio; - - OBD_ALLOC_PTR(aio); - if (aio != NULL) { - /* - * Hold one ref so that it won't be released until - * every pages is added. - */ - cl_sync_io_init_notify(&aio->cda_sync, 1, is_sync_kiocb(iocb) ? - NULL : aio, ll_aio_end); - cl_page_list_init(&aio->cda_pages); - aio->cda_iocb = iocb; - } - return aio; -} - static int ll_direct_rw_pages(const struct lu_env *env, struct cl_io *io, size_t size, int rw, struct inode *inode, struct ll_dio_pages *pv) @@ -480,7 +431,7 @@ ll_direct_IO_impl(struct kiocb *iocb, struct iov_iter *iter, int rw) io = lcc->lcc_io; LASSERT(io != NULL); - aio = ll_aio_alloc(iocb); + aio = cl_aio_alloc(iocb); if (!aio) RETURN(-ENOMEM); diff --git a/lustre/obdclass/cl_internal.h b/lustre/obdclass/cl_internal.h index 0c1276d..8ef48ef 100644 --- a/lustre/obdclass/cl_internal.h +++ b/lustre/obdclass/cl_internal.h @@ -46,6 +46,8 @@ struct cl_thread_info { struct cl_sync_io clt_anchor; }; +extern struct kmem_cache *cl_dio_aio_kmem; + struct cl_thread_info *cl_env_info(const struct lu_env *env); void cl_page_disown0(const struct lu_env *env, struct cl_io *io, struct cl_page *pg); diff --git a/lustre/obdclass/cl_io.c b/lustre/obdclass/cl_io.c index 93a2657..45d9307 100644 --- a/lustre/obdclass/cl_io.c +++ b/lustre/obdclass/cl_io.c @@ -1172,6 +1172,57 @@ int cl_sync_io_wait(const struct lu_env *env, struct cl_sync_io *anchor, } EXPORT_SYMBOL(cl_sync_io_wait); +#ifndef HAVE_AIO_COMPLETE +static inline void aio_complete(struct kiocb *iocb, ssize_t res, ssize_t res2) +{ + if (iocb->ki_complete) + iocb->ki_complete(iocb, res, res2); +} +#endif + +static void cl_aio_end(const struct lu_env *env, struct cl_sync_io *anchor) +{ + struct cl_dio_aio *aio = container_of(anchor, typeof(*aio), cda_sync); + ssize_t ret = anchor->csi_sync_rc; + + ENTRY; + + /* release pages */ + while (aio->cda_pages.pl_nr > 0) { + struct cl_page *page = cl_page_list_first(&aio->cda_pages); + + cl_page_get(page); + cl_page_list_del(env, &aio->cda_pages, page); + cl_page_delete(env, page); + cl_page_put(env, page); + } + + if (!is_sync_kiocb(aio->cda_iocb)) + aio_complete(aio->cda_iocb, ret ?: aio->cda_bytes, 0); + + EXIT; +} + +struct cl_dio_aio *cl_aio_alloc(struct kiocb *iocb) +{ + struct cl_dio_aio *aio; + + OBD_SLAB_ALLOC_PTR_GFP(aio, cl_dio_aio_kmem, GFP_NOFS); + if (aio != NULL) { + /* + * Hold one ref so that it won't be released until + * every pages is added. + */ + cl_sync_io_init_notify(&aio->cda_sync, 1, is_sync_kiocb(iocb) ? + NULL : aio, cl_aio_end); + cl_page_list_init(&aio->cda_pages); + aio->cda_iocb = iocb; + } + return aio; +} +EXPORT_SYMBOL(cl_aio_alloc); + + /** * Indicate that transfer of a single page completed. */ @@ -1214,7 +1265,7 @@ void cl_sync_io_note(const struct lu_env *env, struct cl_sync_io *anchor, * memory here rather than when cl_sync_io_wait() completes. */ if (aio) - OBD_FREE_PTR(aio); + OBD_SLAB_FREE_PTR(aio, cl_dio_aio_kmem); } EXIT; } diff --git a/lustre/obdclass/cl_object.c b/lustre/obdclass/cl_object.c index ebfc7f2..1e39da3 100644 --- a/lustre/obdclass/cl_object.c +++ b/lustre/obdclass/cl_object.c @@ -57,6 +57,7 @@ #include "cl_internal.h" static struct kmem_cache *cl_env_kmem; +struct kmem_cache *cl_dio_aio_kmem; /** Lock class of cl_object_header::coh_attr_guard */ static struct lock_class_key cl_attr_guard_class; @@ -1046,14 +1047,19 @@ static struct lu_context_key cl_key = { }; static struct lu_kmem_descr cl_object_caches[] = { - { - .ckd_cache = &cl_env_kmem, - .ckd_name = "cl_env_kmem", - .ckd_size = sizeof (struct cl_env) - }, - { - .ckd_cache = NULL - } + { + .ckd_cache = &cl_env_kmem, + .ckd_name = "cl_env_kmem", + .ckd_size = sizeof(struct cl_env) + }, + { + .ckd_cache = &cl_dio_aio_kmem, + .ckd_name = "cl_dio_aio_kmem", + .ckd_size = sizeof(struct cl_dio_aio) + }, + { + .ckd_cache = NULL + } }; /** -- 1.8.3.1