From 8a31964534358dd1a5db6cf86b9c6014d3c98d48 Mon Sep 17 00:00:00 2001 From: Patrick Farrell Date: Tue, 17 Aug 2021 11:54:01 -0400 Subject: [PATCH] LU-13799 llite: Adjust dio refcounting We get a page reference in cl_page_find, then immediately add another for cl_2queue_add and remove the first reference. This is pretty silly, since the life cycle is the same on these. This improves DIO/AIO page submission by around 2%. This patch reduces i/o time in ms/GiB by: Write: 2 ms/GiB Read: 2 ms/GiB Totals: Write: 170 ms/GiB Read: 162 ms/GiB mpirun -np 1 $IOR -w -r -t 64M -b 64G -o ./iorfile --posix.odirect With previous pa5ches in series: write 5955 MiB/s read 6218 MiB/s Plus this patch: write 6028 MiB/s read 6305 MiB/s Lustre-change: https://review.whamcloud.com/39447 Lustre-commit: 1e4d10af3909452b0eee1f99010d80aeb01d42a7 Signed-off-by: Patrick Farrell Change-Id: I228eca6d48c6007bbf2c8caae5e477b7d40521d1 Reviewed-on: https://review.whamcloud.com/44446 Reviewed-by: Li Xi Tested-by: Li Xi --- lustre/include/cl_object.h | 52 +++++++++++++++++++++++--------------------- lustre/llite/llite_lib.c | 2 +- lustre/llite/rw.c | 4 ++-- lustre/llite/rw26.c | 9 ++++---- lustre/llite/vvp_io.c | 4 ++-- lustre/llite/vvp_page.c | 10 ++++----- lustre/obdclass/cl_io.c | 20 ++++++++--------- lustre/obdecho/echo_client.c | 4 ++-- 8 files changed, 54 insertions(+), 51 deletions(-) diff --git a/lustre/include/cl_object.h b/lustre/include/cl_object.h index 10a70b0..00f4553 100644 --- a/lustre/include/cl_object.h +++ b/lustre/include/cl_object.h @@ -2528,33 +2528,35 @@ static inline struct cl_page *cl_page_list_first(struct cl_page_list *plist) #define cl_page_list_for_each_safe(page, temp, list) \ list_for_each_entry_safe((page), (temp), &(list)->pl_pages, cp_batch) -void cl_page_list_init (struct cl_page_list *plist); -void cl_page_list_add (struct cl_page_list *plist, struct cl_page *page); -void cl_page_list_move (struct cl_page_list *dst, struct cl_page_list *src, - struct cl_page *page); +void cl_page_list_init(struct cl_page_list *plist); +void cl_page_list_add(struct cl_page_list *plist, struct cl_page *page, + bool get_ref); +void cl_page_list_move(struct cl_page_list *dst, struct cl_page_list *src, + struct cl_page *page); void cl_page_list_move_head(struct cl_page_list *dst, struct cl_page_list *src, - struct cl_page *page); -void cl_page_list_splice (struct cl_page_list *list, - struct cl_page_list *head); -void cl_page_list_del (const struct lu_env *env, - struct cl_page_list *plist, struct cl_page *page); -void cl_page_list_disown (const struct lu_env *env, - struct cl_io *io, struct cl_page_list *plist); -void cl_page_list_assume (const struct lu_env *env, - struct cl_io *io, struct cl_page_list *plist); + struct cl_page *page); +void cl_page_list_splice(struct cl_page_list *list, + struct cl_page_list *head); +void cl_page_list_del(const struct lu_env *env, + struct cl_page_list *plist, struct cl_page *page); +void cl_page_list_disown(const struct lu_env *env, + struct cl_io *io, struct cl_page_list *plist); +void cl_page_list_assume(const struct lu_env *env, + struct cl_io *io, struct cl_page_list *plist); void cl_page_list_discard(const struct lu_env *env, - struct cl_io *io, struct cl_page_list *plist); -void cl_page_list_fini (const struct lu_env *env, struct cl_page_list *plist); - -void cl_2queue_init (struct cl_2queue *queue); -void cl_2queue_add (struct cl_2queue *queue, struct cl_page *page); -void cl_2queue_disown (const struct lu_env *env, - struct cl_io *io, struct cl_2queue *queue); -void cl_2queue_assume (const struct lu_env *env, - struct cl_io *io, struct cl_2queue *queue); -void cl_2queue_discard (const struct lu_env *env, - struct cl_io *io, struct cl_2queue *queue); -void cl_2queue_fini (const struct lu_env *env, struct cl_2queue *queue); + struct cl_io *io, struct cl_page_list *plist); +void cl_page_list_fini(const struct lu_env *env, struct cl_page_list *plist); + +void cl_2queue_init(struct cl_2queue *queue); +void cl_2queue_add(struct cl_2queue *queue, struct cl_page *page, + bool get_ref); +void cl_2queue_disown(const struct lu_env *env, struct cl_io *io, + struct cl_2queue *queue); +void cl_2queue_assume(const struct lu_env *env, struct cl_io *io, + struct cl_2queue *queue); +void cl_2queue_discard(const struct lu_env *env, struct cl_io *io, + struct cl_2queue *queue); +void cl_2queue_fini(const struct lu_env *env, struct cl_2queue *queue); void cl_2queue_init_page(struct cl_2queue *queue, struct cl_page *page); /** @} cl_page_list */ diff --git a/lustre/llite/llite_lib.c b/lustre/llite/llite_lib.c index cc92d13..2ae6c1e 100644 --- a/lustre/llite/llite_lib.c +++ b/lustre/llite/llite_lib.c @@ -1876,7 +1876,7 @@ int ll_io_zero_page(struct inode *inode, pgoff_t index, pgoff_t offset, anchor = &vvp_env_info(env)->vti_anchor; cl_sync_io_init(anchor, 1); clpage->cp_sync_io = anchor; - cl_2queue_add(queue, clpage); + cl_2queue_add(queue, clpage, true); rc = cl_io_submit_rw(env, io, CRT_WRITE, queue); if (rc) GOTO(queuefini1, rc); diff --git a/lustre/llite/rw.c b/lustre/llite/rw.c index 958d848..18981bb 100644 --- a/lustre/llite/rw.c +++ b/lustre/llite/rw.c @@ -248,7 +248,7 @@ static int ll_read_ahead_page(const struct lu_env *env, struct cl_io *io, vpg->vpg_defer_uptodate = 1; vpg->vpg_ra_used = 0; } - cl_page_list_add(queue, page); + cl_page_list_add(queue, page, true); } else { /* skip completed pages */ cl_page_unassume(env, io, page); @@ -1658,7 +1658,7 @@ int ll_io_read_page(const struct lu_env *env, struct cl_io *io, cl_sync_io_init(anchor, 1); page->cp_sync_io = anchor; - cl_2queue_add(queue, page); + cl_2queue_add(queue, page, true); } io_start_index = cl_index(io->ci_obj, io->u.ci_rw.crw_pos); diff --git a/lustre/llite/rw26.c b/lustre/llite/rw26.c index 39becc1..5b7417d 100644 --- a/lustre/llite/rw26.c +++ b/lustre/llite/rw26.c @@ -368,7 +368,10 @@ ll_direct_rw_pages(const struct lu_env *env, struct cl_io *io, size_t size, */ page->cp_inode = inode; } - cl_2queue_add(queue, page); + /* We keep the refcount from cl_page_find, so we don't need + * another one here + */ + cl_2queue_add(queue, page, false); /* * Set page clip to tell transfer formation engine * that page has to be sent even if it is beyond KMS. @@ -377,8 +380,6 @@ ll_direct_rw_pages(const struct lu_env *env, struct cl_io *io, size_t size, cl_page_clip(env, page, 0, size); ++io_pages; - /* drop the reference count for cl_page_find */ - cl_page_put(env, page); offset += page_size; size -= page_size; } @@ -874,7 +875,7 @@ static int ll_write_end(struct file *file, struct address_space *mapping, lcc->lcc_page = NULL; /* page will be queued */ /* Add it into write queue */ - cl_page_list_add(plist, page); + cl_page_list_add(plist, page, true); if (plist->pl_nr == 1) /* first page */ vio->u.readwrite.vui_from = from; else diff --git a/lustre/llite/vvp_io.c b/lustre/llite/vvp_io.c index d3ac0da..ac0e79b 100644 --- a/lustre/llite/vvp_io.c +++ b/lustre/llite/vvp_io.c @@ -1500,7 +1500,7 @@ static int vvp_io_fault_start(const struct lu_env *env, cl_page_assume(env, io, page); cl_page_list_init(plist); - cl_page_list_add(plist, page); + cl_page_list_add(plist, page, true); /* size fixup */ if (last_index == vvp_index(vpg)) @@ -1520,7 +1520,7 @@ static int vvp_io_fault_start(const struct lu_env *env, if (result >= 0) { io->ci_noquota = 1; cl_page_own(env, io, page); - cl_page_list_add(plist, page); + cl_page_list_add(plist, page, true); lu_ref_add(&page->cp_reference, "cl_io", io); result = cl_io_commit_async(env, io, diff --git a/lustre/llite/vvp_page.c b/lustre/llite/vvp_page.c index 529244f..be61460 100644 --- a/lustre/llite/vvp_page.c +++ b/lustre/llite/vvp_page.c @@ -477,18 +477,18 @@ int vvp_page_init(const struct lu_env *env, struct cl_object *obj, CLOBINVRNT(env, obj, vvp_object_invariant(obj)); vpg->vpg_page = vmpage; - get_page(vmpage); - if (page->cp_type == CPT_CACHEABLE) { + if (page->cp_type == CPT_TRANSIENT) { + cl_page_slice_add(page, &vpg->vpg_cl, obj, + &vvp_transient_page_ops); + } else { + get_page(vmpage); /* in cache, decref in vvp_page_delete */ atomic_inc(&page->cp_ref); SetPagePrivate(vmpage); vmpage->private = (unsigned long)page; cl_page_slice_add(page, &vpg->vpg_cl, obj, &vvp_page_ops); - } else { - cl_page_slice_add(page, &vpg->vpg_cl, obj, - &vvp_transient_page_ops); } return 0; } diff --git a/lustre/obdclass/cl_io.c b/lustre/obdclass/cl_io.c index fbc7825..dc78aba 100644 --- a/lustre/obdclass/cl_io.c +++ b/lustre/obdclass/cl_io.c @@ -832,7 +832,8 @@ EXPORT_SYMBOL(cl_page_list_init); /** * Adds a page to a page list. */ -void cl_page_list_add(struct cl_page_list *plist, struct cl_page *page) +void cl_page_list_add(struct cl_page_list *plist, struct cl_page *page, + bool get_ref) { ENTRY; /* it would be better to check that page is owned by "current" io, but @@ -843,7 +844,8 @@ void cl_page_list_add(struct cl_page_list *plist, struct cl_page *page) list_add_tail(&page->cp_batch, &plist->pl_pages); ++plist->pl_nr; lu_ref_add_at(&page->cp_reference, &page->cp_queue_ref, "queue", plist); - cl_page_get(page); + if (get_ref) + cl_page_get(page); EXIT; } EXPORT_SYMBOL(cl_page_list_add); @@ -1020,11 +1022,9 @@ EXPORT_SYMBOL(cl_2queue_init); /** * Add a page to the incoming page list of 2-queue. */ -void cl_2queue_add(struct cl_2queue *queue, struct cl_page *page) +void cl_2queue_add(struct cl_2queue *queue, struct cl_page *page, bool get_ref) { - ENTRY; - cl_page_list_add(&queue->c2_qin, page); - EXIT; + cl_page_list_add(&queue->c2_qin, page, get_ref); } EXPORT_SYMBOL(cl_2queue_add); @@ -1081,10 +1081,10 @@ EXPORT_SYMBOL(cl_2queue_fini); */ void cl_2queue_init_page(struct cl_2queue *queue, struct cl_page *page) { - ENTRY; - cl_2queue_init(queue); - cl_2queue_add(queue, page); - EXIT; + ENTRY; + cl_2queue_init(queue); + cl_2queue_add(queue, page, true); + EXIT; } EXPORT_SYMBOL(cl_2queue_init_page); diff --git a/lustre/obdecho/echo_client.c b/lustre/obdecho/echo_client.c index 820cd22..bdec008 100644 --- a/lustre/obdecho/echo_client.c +++ b/lustre/obdecho/echo_client.c @@ -1330,7 +1330,7 @@ static void echo_commit_callback(const struct lu_env *env, struct cl_io *io, struct page *vmpage = pvec->pages[i]; struct cl_page *page = (struct cl_page *)vmpage->private; - cl_page_list_add(&queue->c2_qout, page); + cl_page_list_add(&queue->c2_qout, page, true); } } @@ -1393,7 +1393,7 @@ static int cl_echo_object_brw(struct echo_object *eco, int rw, u64 offset, break; } - cl_2queue_add(queue, clp); + cl_2queue_add(queue, clp, true); /* * drop the reference count for cl_page_find, so that the page -- 1.8.3.1