LU-13799 llite: Modify AIO/DIO reference counting

For DIO pages, it's enough to have a reference on the
cl_object associated with the AIO.  This saves taking a
reference on the cl_object for each page, which saves about
5% of the time when doing DIO/AIO.

This is possible because the lifecycle of the aio struct is
always greater than that of the associated pages.

This patch reduces i/o time in ms/GiB by:
Write: 6 ms/GiB
Read: 1 ms/GiB

Totals:
Write: 198 ms/GiB
Read: 197 ms/GiB

mpirun -np 1  $IOR -w -r -t 64M -b 64G -o ./iorfile --posix.odirect

With previous patches in series:
write     5030 MiB/s
read      5174 MiB/s

Plus this patch:
write     5183 MiB/s
read      5200 MiB/s

Signed-off-by: Patrick Farrell <farr0186@gmail.com>
Change-Id: I970cda20417265b4b66a8eed6e74440e5d3373b8
Reviewed-on: https://review.whamcloud.com/39442
Reviewed-by: Wang Shilong <wshilong@whamcloud.com>
Reviewed-by: Andreas Dilger <adilger@whamcloud.com>
Reviewed-by: Shaun Tancheff <shaun.tancheff@hpe.com>
Tested-by: jenkins <devops@whamcloud.com>
Tested-by: Maloo <maloo@whamcloud.com>
Reviewed-by: Oleg Drokin <green@whamcloud.com>

diff --git a/lustre/include/cl_object.h b/lustre/include/cl_object.h
index 26fbd28..637015b 100644 (file)
--- a/lustre/include/cl_object.h
+++ b/lustre/include/cl_object.h
@@ -2581,8 +2581,8 @@ void cl_sync_io_note(const struct lu_env *env, struct cl_sync_io *anchor,
                     int ioret);
 int cl_sync_io_wait_recycle(const struct lu_env *env, struct cl_sync_io *anchor,
                            long timeout, int ioret);
-struct cl_dio_aio *cl_aio_alloc(struct kiocb *iocb);
-void cl_aio_free(struct cl_dio_aio *aio);
+struct cl_dio_aio *cl_aio_alloc(struct kiocb *iocb, struct cl_object *obj);
+void cl_aio_free(const struct lu_env *env, struct cl_dio_aio *aio);
 static inline void cl_sync_io_init(struct cl_sync_io *anchor, int nr)
 {
        cl_sync_io_init_notify(anchor, nr, NULL, NULL);
@@ -2611,6 +2611,7 @@ struct cl_sync_io {
 struct cl_dio_aio {
        struct cl_sync_io       cda_sync;
        struct cl_page_list     cda_pages;
+       struct cl_object        *cda_obj;
        struct kiocb            *cda_iocb;
        ssize_t                 cda_bytes;
        unsigned                cda_no_aio_complete:1;

diff --git a/lustre/llite/file.c b/lustre/llite/file.c
index a96286f..9a63398 100644 (file)
--- a/lustre/llite/file.c
+++ b/lustre/llite/file.c
@@ -1635,7 +1635,8 @@ ll_file_io_generic(const struct lu_env *env, struct vvp_io_args *args,
                if (!ll_sbi_has_parallel_dio(sbi))
                        is_parallel_dio = false;
 
-               ci_aio = cl_aio_alloc(args->u.normal.via_iocb);
+               ci_aio = cl_aio_alloc(args->u.normal.via_iocb,
+                                     ll_i2info(inode)->lli_clob);
                if (!ci_aio)
                        GOTO(out, rc = -ENOMEM);
        }
@@ -1790,7 +1791,7 @@ out:
                cl_sync_io_note(env, &io->ci_aio->cda_sync,
                                rc == -EIOCBQUEUED ? 0 : rc);
                if (!is_aio) {
-                       cl_aio_free(io->ci_aio);
+                       cl_aio_free(env, io->ci_aio);
                        io->ci_aio = NULL;
                }
        }

diff --git a/lustre/obdclass/cl_io.c b/lustre/obdclass/cl_io.c
index d11c5a6..4ddf192 100644 (file)
--- a/lustre/obdclass/cl_io.c
+++ b/lustre/obdclass/cl_io.c
@@ -1232,7 +1232,7 @@ static void cl_aio_end(const struct lu_env *env, struct cl_sync_io *anchor)
        EXIT;
 }
 
-struct cl_dio_aio *cl_aio_alloc(struct kiocb *iocb)
+struct cl_dio_aio *cl_aio_alloc(struct kiocb *iocb, struct cl_object *obj)
 {
        struct cl_dio_aio *aio;
 
@@ -1247,15 +1247,19 @@ struct cl_dio_aio *cl_aio_alloc(struct kiocb *iocb)
                cl_page_list_init(&aio->cda_pages);
                aio->cda_iocb = iocb;
                aio->cda_no_aio_complete = 0;
+               cl_object_get(obj);
+               aio->cda_obj = obj;
        }
        return aio;
 }
 EXPORT_SYMBOL(cl_aio_alloc);
 
-void cl_aio_free(struct cl_dio_aio *aio)
+void cl_aio_free(const struct lu_env *env, struct cl_dio_aio *aio)
 {
-       if (aio)
+       if (aio) {
+               cl_object_put(env, aio->cda_obj);
                OBD_SLAB_FREE_PTR(aio, cl_dio_aio_kmem);
+       }
 }
 EXPORT_SYMBOL(cl_aio_free);
 
@@ -1301,7 +1305,7 @@ void cl_sync_io_note(const struct lu_env *env, struct cl_sync_io *anchor,
                 * If anchor->csi_aio is set, we are responsible for freeing
                 * memory here rather than when cl_sync_io_wait() completes.
                 */
-               cl_aio_free(aio);
+               cl_aio_free(env, aio);
        }
        EXIT;
 }

diff --git a/lustre/obdclass/cl_page.c b/lustre/obdclass/cl_page.c
index e345958..5525ef5 100644 (file)
--- a/lustre/obdclass/cl_page.c
+++ b/lustre/obdclass/cl_page.c
@@ -203,7 +203,8 @@ static void cl_page_free(const struct lu_env *env, struct cl_page *cl_page,
        cs_pagestate_dec(obj, cl_page->cp_state);
        lu_object_ref_del_at(&obj->co_lu, &cl_page->cp_obj_ref,
                             "cl_page", cl_page);
-       cl_object_put(env, obj);
+       if (cl_page->cp_type != CPT_TRANSIENT)
+               cl_object_put(env, obj);
        lu_ref_fini(&cl_page->cp_reference);
        __cl_page_free(cl_page, bufsize);
        EXIT;
@@ -286,7 +287,8 @@ struct cl_page *cl_page_alloc(const struct lu_env *env, struct cl_object *o,
                BUILD_BUG_ON((1 << CP_TYPE_BITS) < CPT_NR); /* cp_type */
                atomic_set(&cl_page->cp_ref, 1);
                cl_page->cp_obj = o;
-               cl_object_get(o);
+               if (type != CPT_TRANSIENT)
+                       cl_object_get(o);
                lu_object_ref_add_at(&o->co_lu, &cl_page->cp_obj_ref,
                                     "cl_page", cl_page);
                cl_page->cp_vmpage = vmpage;