From 1e4d10af3909452b0eee1f99010d80aeb01d42a7 Mon Sep 17 00:00:00 2001 From: Patrick Farrell Date: Fri, 7 May 2021 15:50:15 -0400 Subject: [PATCH] LU-13799 llite: Adjust dio refcounting We get a page reference in cl_page_find, then immediately add another for cl_2queue_add and remove the first reference. This is pretty silly, since the life cycle is the same on these. This improves DIO/AIO page submission by around 2%. This patch reduces i/o time in ms/GiB by: Write: 2 ms/GiB Read: 2 ms/GiB Totals: Write: 170 ms/GiB Read: 162 ms/GiB mpirun -np 1 $IOR -w -r -t 64M -b 64G -o ./iorfile --posix.odirect With previous pa5ches in series: write 5955 MiB/s read 6218 MiB/s Plus this patch: write 6028 MiB/s read 6305 MiB/s Signed-off-by: Patrick Farrell Change-Id: I228eca6d48c6007bbf2c8caae5e477b7d40521d1 Reviewed-on: https://review.whamcloud.com/39447 Reviewed-by: Wang Shilong Reviewed-by: Andreas Dilger Tested-by: jenkins Tested-by: Maloo Reviewed-by: Oleg Drokin --- lustre/include/cl_object.h | 52 +++++++++++++++++++++++--------------------- lustre/llite/llite_lib.c | 2 +- lustre/llite/rw.c | 4 ++-- lustre/llite/rw26.c | 9 ++++---- lustre/llite/vvp_io.c | 4 ++-- lustre/llite/vvp_page.c | 12 ++++++---- lustre/obdclass/cl_io.c | 20 ++++++++--------- lustre/obdecho/echo_client.c | 4 ++-- 8 files changed, 57 insertions(+), 50 deletions(-) diff --git a/lustre/include/cl_object.h b/lustre/include/cl_object.h index 637015b..e180f68 100644 --- a/lustre/include/cl_object.h +++ b/lustre/include/cl_object.h @@ -2530,33 +2530,35 @@ static inline struct cl_page *cl_page_list_first(struct cl_page_list *plist) #define cl_page_list_for_each_safe(page, temp, list) \ list_for_each_entry_safe((page), (temp), &(list)->pl_pages, cp_batch) -void cl_page_list_init (struct cl_page_list *plist); -void cl_page_list_add (struct cl_page_list *plist, struct cl_page *page); -void cl_page_list_move (struct cl_page_list *dst, struct cl_page_list *src, - struct cl_page *page); +void cl_page_list_init(struct cl_page_list *plist); +void cl_page_list_add(struct cl_page_list *plist, struct cl_page *page, + bool get_ref); +void cl_page_list_move(struct cl_page_list *dst, struct cl_page_list *src, + struct cl_page *page); void cl_page_list_move_head(struct cl_page_list *dst, struct cl_page_list *src, - struct cl_page *page); -void cl_page_list_splice (struct cl_page_list *list, - struct cl_page_list *head); -void cl_page_list_del (const struct lu_env *env, - struct cl_page_list *plist, struct cl_page *page); -void cl_page_list_disown (const struct lu_env *env, - struct cl_io *io, struct cl_page_list *plist); -void cl_page_list_assume (const struct lu_env *env, - struct cl_io *io, struct cl_page_list *plist); + struct cl_page *page); +void cl_page_list_splice(struct cl_page_list *list, + struct cl_page_list *head); +void cl_page_list_del(const struct lu_env *env, + struct cl_page_list *plist, struct cl_page *page); +void cl_page_list_disown(const struct lu_env *env, + struct cl_io *io, struct cl_page_list *plist); +void cl_page_list_assume(const struct lu_env *env, + struct cl_io *io, struct cl_page_list *plist); void cl_page_list_discard(const struct lu_env *env, - struct cl_io *io, struct cl_page_list *plist); -void cl_page_list_fini (const struct lu_env *env, struct cl_page_list *plist); - -void cl_2queue_init (struct cl_2queue *queue); -void cl_2queue_add (struct cl_2queue *queue, struct cl_page *page); -void cl_2queue_disown (const struct lu_env *env, - struct cl_io *io, struct cl_2queue *queue); -void cl_2queue_assume (const struct lu_env *env, - struct cl_io *io, struct cl_2queue *queue); -void cl_2queue_discard (const struct lu_env *env, - struct cl_io *io, struct cl_2queue *queue); -void cl_2queue_fini (const struct lu_env *env, struct cl_2queue *queue); + struct cl_io *io, struct cl_page_list *plist); +void cl_page_list_fini(const struct lu_env *env, struct cl_page_list *plist); + +void cl_2queue_init(struct cl_2queue *queue); +void cl_2queue_add(struct cl_2queue *queue, struct cl_page *page, + bool get_ref); +void cl_2queue_disown(const struct lu_env *env, struct cl_io *io, + struct cl_2queue *queue); +void cl_2queue_assume(const struct lu_env *env, struct cl_io *io, + struct cl_2queue *queue); +void cl_2queue_discard(const struct lu_env *env, struct cl_io *io, + struct cl_2queue *queue); +void cl_2queue_fini(const struct lu_env *env, struct cl_2queue *queue); void cl_2queue_init_page(struct cl_2queue *queue, struct cl_page *page); /** @} cl_page_list */ diff --git a/lustre/llite/llite_lib.c b/lustre/llite/llite_lib.c index d0597f2..e20a0c7 100644 --- a/lustre/llite/llite_lib.c +++ b/lustre/llite/llite_lib.c @@ -1970,7 +1970,7 @@ int ll_io_zero_page(struct inode *inode, pgoff_t index, pgoff_t offset, anchor = &vvp_env_info(env)->vti_anchor; cl_sync_io_init(anchor, 1); clpage->cp_sync_io = anchor; - cl_2queue_add(queue, clpage); + cl_2queue_add(queue, clpage, true); rc = cl_io_submit_rw(env, io, CRT_WRITE, queue); if (rc) GOTO(queuefini1, rc); diff --git a/lustre/llite/rw.c b/lustre/llite/rw.c index 04cf0e7..3d69fec 100644 --- a/lustre/llite/rw.c +++ b/lustre/llite/rw.c @@ -247,7 +247,7 @@ static int ll_read_ahead_page(const struct lu_env *env, struct cl_io *io, vpg->vpg_defer_uptodate = 1; vpg->vpg_ra_used = 0; } - cl_page_list_add(queue, page); + cl_page_list_add(queue, page, true); } else { /* skip completed pages */ cl_page_unassume(env, io, page); @@ -1657,7 +1657,7 @@ int ll_io_read_page(const struct lu_env *env, struct cl_io *io, cl_sync_io_init(anchor, 1); page->cp_sync_io = anchor; - cl_2queue_add(queue, page); + cl_2queue_add(queue, page, true); } io_start_index = cl_index(io->ci_obj, io->u.ci_rw.crw_pos); diff --git a/lustre/llite/rw26.c b/lustre/llite/rw26.c index d4122613..6fb6d53 100644 --- a/lustre/llite/rw26.c +++ b/lustre/llite/rw26.c @@ -367,7 +367,10 @@ ll_direct_rw_pages(const struct lu_env *env, struct cl_io *io, size_t size, */ page->cp_inode = inode; } - cl_2queue_add(queue, page); + /* We keep the refcount from cl_page_find, so we don't need + * another one here + */ + cl_2queue_add(queue, page, false); /* * Set page clip to tell transfer formation engine * that page has to be sent even if it is beyond KMS. @@ -376,8 +379,6 @@ ll_direct_rw_pages(const struct lu_env *env, struct cl_io *io, size_t size, cl_page_clip(env, page, 0, size); ++io_pages; - /* drop the reference count for cl_page_find */ - cl_page_put(env, page); offset += page_size; size -= page_size; } @@ -873,7 +874,7 @@ static int ll_write_end(struct file *file, struct address_space *mapping, lcc->lcc_page = NULL; /* page will be queued */ /* Add it into write queue */ - cl_page_list_add(plist, page); + cl_page_list_add(plist, page, true); if (plist->pl_nr == 1) /* first page */ vio->u.readwrite.vui_from = from; else diff --git a/lustre/llite/vvp_io.c b/lustre/llite/vvp_io.c index 5cd5fcb..b7b953d 100644 --- a/lustre/llite/vvp_io.c +++ b/lustre/llite/vvp_io.c @@ -1510,7 +1510,7 @@ static int vvp_io_fault_start(const struct lu_env *env, cl_page_assume(env, io, page); cl_page_list_init(plist); - cl_page_list_add(plist, page); + cl_page_list_add(plist, page, true); /* size fixup */ if (last_index == vvp_index(vpg)) @@ -1530,7 +1530,7 @@ static int vvp_io_fault_start(const struct lu_env *env, if (result >= 0) { io->ci_noquota = 1; cl_page_own(env, io, page); - cl_page_list_add(plist, page); + cl_page_list_add(plist, page, true); lu_ref_add(&page->cp_reference, "cl_io", io); result = cl_io_commit_async(env, io, diff --git a/lustre/llite/vvp_page.c b/lustre/llite/vvp_page.c index 959f668..3a3bc9e 100644 --- a/lustre/llite/vvp_page.c +++ b/lustre/llite/vvp_page.c @@ -478,16 +478,20 @@ int vvp_page_init(const struct lu_env *env, struct cl_object *obj, vpg->vpg_page = vmpage; get_page(vmpage); - if (page->cp_type == CPT_CACHEABLE) { + if (page->cp_type == CPT_TRANSIENT) { + /* DIO pages are referenced by userspace, we don't need to take + * a reference on them. (contrast with get_page() call above) + */ + cl_page_slice_add(page, &vpg->vpg_cl, obj, + &vvp_transient_page_ops); + } else { /* in cache, decref in vvp_page_delete */ atomic_inc(&page->cp_ref); SetPagePrivate(vmpage); vmpage->private = (unsigned long)page; cl_page_slice_add(page, &vpg->vpg_cl, obj, &vvp_page_ops); - } else { - cl_page_slice_add(page, &vpg->vpg_cl, obj, - &vvp_transient_page_ops); } + return 0; } diff --git a/lustre/obdclass/cl_io.c b/lustre/obdclass/cl_io.c index 4ddf192..d3ce874 100644 --- a/lustre/obdclass/cl_io.c +++ b/lustre/obdclass/cl_io.c @@ -845,7 +845,8 @@ EXPORT_SYMBOL(cl_page_list_init); /** * Adds a page to a page list. */ -void cl_page_list_add(struct cl_page_list *plist, struct cl_page *page) +void cl_page_list_add(struct cl_page_list *plist, struct cl_page *page, + bool get_ref) { ENTRY; /* it would be better to check that page is owned by "current" io, but @@ -856,7 +857,8 @@ void cl_page_list_add(struct cl_page_list *plist, struct cl_page *page) list_add_tail(&page->cp_batch, &plist->pl_pages); ++plist->pl_nr; lu_ref_add_at(&page->cp_reference, &page->cp_queue_ref, "queue", plist); - cl_page_get(page); + if (get_ref) + cl_page_get(page); EXIT; } EXPORT_SYMBOL(cl_page_list_add); @@ -1033,11 +1035,9 @@ EXPORT_SYMBOL(cl_2queue_init); /** * Add a page to the incoming page list of 2-queue. */ -void cl_2queue_add(struct cl_2queue *queue, struct cl_page *page) +void cl_2queue_add(struct cl_2queue *queue, struct cl_page *page, bool get_ref) { - ENTRY; - cl_page_list_add(&queue->c2_qin, page); - EXIT; + cl_page_list_add(&queue->c2_qin, page, get_ref); } EXPORT_SYMBOL(cl_2queue_add); @@ -1094,10 +1094,10 @@ EXPORT_SYMBOL(cl_2queue_fini); */ void cl_2queue_init_page(struct cl_2queue *queue, struct cl_page *page) { - ENTRY; - cl_2queue_init(queue); - cl_2queue_add(queue, page); - EXIT; + ENTRY; + cl_2queue_init(queue); + cl_2queue_add(queue, page, true); + EXIT; } EXPORT_SYMBOL(cl_2queue_init_page); diff --git a/lustre/obdecho/echo_client.c b/lustre/obdecho/echo_client.c index 437466a..4042fe5 100644 --- a/lustre/obdecho/echo_client.c +++ b/lustre/obdecho/echo_client.c @@ -1328,7 +1328,7 @@ static void echo_commit_callback(const struct lu_env *env, struct cl_io *io, struct page *vmpage = pvec->pages[i]; struct cl_page *page = (struct cl_page *)vmpage->private; - cl_page_list_add(&queue->c2_qout, page); + cl_page_list_add(&queue->c2_qout, page, true); } } @@ -1391,7 +1391,7 @@ static int cl_echo_object_brw(struct echo_object *eco, int rw, u64 offset, break; } - cl_2queue_add(queue, clp); + cl_2queue_add(queue, clp, true); /* * drop the reference count for cl_page_find, so that the page -- 1.8.3.1