LU-13799 llite: Adjust dio refcounting

We get a page reference in cl_page_find, then immediately
add another for cl_2queue_add and remove the first
reference.  This is pretty silly, since the life cycle is
the same on these.

This improves DIO/AIO page submission by around 2%.

This patch reduces i/o time in ms/GiB by:
Write: 2 ms/GiB
Read: 2 ms/GiB

Totals:
Write: 170 ms/GiB
Read: 162 ms/GiB

mpirun -np 1  $IOR -w -r -t 64M -b 64G -o ./iorfile --posix.odirect

With previous pa5ches in series:
write        5955 MiB/s
read         6218 MiB/s

Plus this patch:
write        6028 MiB/s
read         6305 MiB/s

Signed-off-by: Patrick Farrell <farr0186@gmail.com>
Change-Id: I228eca6d48c6007bbf2c8caae5e477b7d40521d1
Reviewed-on: https://review.whamcloud.com/39447
Reviewed-by: Wang Shilong <wshilong@whamcloud.com>
Reviewed-by: Andreas Dilger <adilger@whamcloud.com>
Tested-by: jenkins <devops@whamcloud.com>
Tested-by: Maloo <maloo@whamcloud.com>
Reviewed-by: Oleg Drokin <green@whamcloud.com>

diff --git a/lustre/include/cl_object.h b/lustre/include/cl_object.h
index 637015b..e180f68 100644 (file)
--- a/lustre/include/cl_object.h
+++ b/lustre/include/cl_object.h
@@ -2530,33 +2530,35 @@ static inline struct cl_page *cl_page_list_first(struct cl_page_list *plist)
 #define cl_page_list_for_each_safe(page, temp, list)                    \
        list_for_each_entry_safe((page), (temp), &(list)->pl_pages, cp_batch)
 
-void cl_page_list_init   (struct cl_page_list *plist);
-void cl_page_list_add    (struct cl_page_list *plist, struct cl_page *page);
-void cl_page_list_move   (struct cl_page_list *dst, struct cl_page_list *src,
-                          struct cl_page *page);
+void cl_page_list_init(struct cl_page_list *plist);
+void cl_page_list_add(struct cl_page_list *plist, struct cl_page *page,
+                     bool get_ref);
+void cl_page_list_move(struct cl_page_list *dst, struct cl_page_list *src,
+                      struct cl_page *page);
 void cl_page_list_move_head(struct cl_page_list *dst, struct cl_page_list *src,
-                         struct cl_page *page);
-void cl_page_list_splice (struct cl_page_list *list,
-                          struct cl_page_list *head);
-void cl_page_list_del    (const struct lu_env *env,
-                          struct cl_page_list *plist, struct cl_page *page);
-void cl_page_list_disown (const struct lu_env *env,
-                          struct cl_io *io, struct cl_page_list *plist);
-void cl_page_list_assume (const struct lu_env *env,
-                          struct cl_io *io, struct cl_page_list *plist);
+                           struct cl_page *page);
+void cl_page_list_splice(struct cl_page_list *list,
+                        struct cl_page_list *head);
+void cl_page_list_del(const struct lu_env *env,
+                     struct cl_page_list *plist, struct cl_page *page);
+void cl_page_list_disown(const struct lu_env *env,
+                        struct cl_io *io, struct cl_page_list *plist);
+void cl_page_list_assume(const struct lu_env *env,
+                        struct cl_io *io, struct cl_page_list *plist);
 void cl_page_list_discard(const struct lu_env *env,
-                          struct cl_io *io, struct cl_page_list *plist);
-void cl_page_list_fini   (const struct lu_env *env, struct cl_page_list *plist);
-
-void cl_2queue_init     (struct cl_2queue *queue);
-void cl_2queue_add      (struct cl_2queue *queue, struct cl_page *page);
-void cl_2queue_disown   (const struct lu_env *env,
-                         struct cl_io *io, struct cl_2queue *queue);
-void cl_2queue_assume   (const struct lu_env *env,
-                         struct cl_io *io, struct cl_2queue *queue);
-void cl_2queue_discard  (const struct lu_env *env,
-                         struct cl_io *io, struct cl_2queue *queue);
-void cl_2queue_fini     (const struct lu_env *env, struct cl_2queue *queue);
+                         struct cl_io *io, struct cl_page_list *plist);
+void cl_page_list_fini(const struct lu_env *env, struct cl_page_list *plist);
+
+void cl_2queue_init(struct cl_2queue *queue);
+void cl_2queue_add(struct cl_2queue *queue, struct cl_page *page,
+                  bool get_ref);
+void cl_2queue_disown(const struct lu_env *env, struct cl_io *io,
+                     struct cl_2queue *queue);
+void cl_2queue_assume(const struct lu_env *env, struct cl_io *io,
+                     struct cl_2queue *queue);
+void cl_2queue_discard(const struct lu_env *env, struct cl_io *io,
+                      struct cl_2queue *queue);
+void cl_2queue_fini(const struct lu_env *env, struct cl_2queue *queue);
 void cl_2queue_init_page(struct cl_2queue *queue, struct cl_page *page);
 
 /** @} cl_page_list */

diff --git a/lustre/llite/llite_lib.c b/lustre/llite/llite_lib.c
index d0597f2..e20a0c7 100644 (file)
--- a/lustre/llite/llite_lib.c
+++ b/lustre/llite/llite_lib.c
@@ -1970,7 +1970,7 @@ int ll_io_zero_page(struct inode *inode, pgoff_t index, pgoff_t offset,
                anchor = &vvp_env_info(env)->vti_anchor;
                cl_sync_io_init(anchor, 1);
                clpage->cp_sync_io = anchor;
-               cl_2queue_add(queue, clpage);
+               cl_2queue_add(queue, clpage, true);
                rc = cl_io_submit_rw(env, io, CRT_WRITE, queue);
                if (rc)
                        GOTO(queuefini1, rc);

diff --git a/lustre/llite/rw.c b/lustre/llite/rw.c
index 04cf0e7..3d69fec 100644 (file)
--- a/lustre/llite/rw.c
+++ b/lustre/llite/rw.c
@@ -247,7 +247,7 @@ static int ll_read_ahead_page(const struct lu_env *env, struct cl_io *io,
                        vpg->vpg_defer_uptodate = 1;
                        vpg->vpg_ra_used = 0;
                }
-               cl_page_list_add(queue, page);
+               cl_page_list_add(queue, page, true);
        } else {
                /* skip completed pages */
                cl_page_unassume(env, io, page);
@@ -1657,7 +1657,7 @@ int ll_io_read_page(const struct lu_env *env, struct cl_io *io,
                cl_sync_io_init(anchor, 1);
                page->cp_sync_io = anchor;
 
-               cl_2queue_add(queue, page);
+               cl_2queue_add(queue, page, true);
        }
 
        io_start_index = cl_index(io->ci_obj, io->u.ci_rw.crw_pos);

diff --git a/lustre/llite/rw26.c b/lustre/llite/rw26.c
index d412261..6fb6d53 100644 (file)
--- a/lustre/llite/rw26.c
+++ b/lustre/llite/rw26.c
@@ -367,7 +367,10 @@ ll_direct_rw_pages(const struct lu_env *env, struct cl_io *io, size_t size,
                         */
                        page->cp_inode = inode;
                }
-               cl_2queue_add(queue, page);
+               /* We keep the refcount from cl_page_find, so we don't need
+                * another one here
+                */
+               cl_2queue_add(queue, page, false);
                /*
                 * Set page clip to tell transfer formation engine
                 * that page has to be sent even if it is beyond KMS.
@@ -376,8 +379,6 @@ ll_direct_rw_pages(const struct lu_env *env, struct cl_io *io, size_t size,
                        cl_page_clip(env, page, 0, size);
                ++io_pages;
 
-               /* drop the reference count for cl_page_find */
-               cl_page_put(env, page);
                offset += page_size;
                size -= page_size;
        }
@@ -873,7 +874,7 @@ static int ll_write_end(struct file *file, struct address_space *mapping,
                lcc->lcc_page = NULL; /* page will be queued */
 
                /* Add it into write queue */
-               cl_page_list_add(plist, page);
+               cl_page_list_add(plist, page, true);
                if (plist->pl_nr == 1) /* first page */
                        vio->u.readwrite.vui_from = from;
                else

diff --git a/lustre/llite/vvp_io.c b/lustre/llite/vvp_io.c
index 5cd5fcb..b7b953d 100644 (file)
--- a/lustre/llite/vvp_io.c
+++ b/lustre/llite/vvp_io.c
@@ -1510,7 +1510,7 @@ static int vvp_io_fault_start(const struct lu_env *env,
                        cl_page_assume(env, io, page);
 
                        cl_page_list_init(plist);
-                       cl_page_list_add(plist, page);
+                       cl_page_list_add(plist, page, true);
 
                        /* size fixup */
                        if (last_index == vvp_index(vpg))
@@ -1530,7 +1530,7 @@ static int vvp_io_fault_start(const struct lu_env *env,
                                if (result >= 0) {
                                        io->ci_noquota = 1;
                                        cl_page_own(env, io, page);
-                                       cl_page_list_add(plist, page);
+                                       cl_page_list_add(plist, page, true);
                                        lu_ref_add(&page->cp_reference,
                                                   "cl_io", io);
                                        result = cl_io_commit_async(env, io,

diff --git a/lustre/llite/vvp_page.c b/lustre/llite/vvp_page.c
index 959f668..3a3bc9e 100644 (file)
--- a/lustre/llite/vvp_page.c
+++ b/lustre/llite/vvp_page.c
@@ -478,16 +478,20 @@ int vvp_page_init(const struct lu_env *env, struct cl_object *obj,
        vpg->vpg_page = vmpage;
        get_page(vmpage);
 
-       if (page->cp_type == CPT_CACHEABLE) {
+       if (page->cp_type == CPT_TRANSIENT) {
+               /* DIO pages are referenced by userspace, we don't need to take
+                * a reference on them. (contrast with get_page() call above)
+                */
+               cl_page_slice_add(page, &vpg->vpg_cl, obj,
+                                 &vvp_transient_page_ops);
+       } else {
                /* in cache, decref in vvp_page_delete */
                atomic_inc(&page->cp_ref);
                SetPagePrivate(vmpage);
                vmpage->private = (unsigned long)page;
                cl_page_slice_add(page, &vpg->vpg_cl, obj,
                                &vvp_page_ops);
-       } else {
-               cl_page_slice_add(page, &vpg->vpg_cl, obj,
-                                 &vvp_transient_page_ops);
        }
+
        return 0;
 }

diff --git a/lustre/obdclass/cl_io.c b/lustre/obdclass/cl_io.c
index 4ddf192..d3ce874 100644 (file)
--- a/lustre/obdclass/cl_io.c
+++ b/lustre/obdclass/cl_io.c
@@ -845,7 +845,8 @@ EXPORT_SYMBOL(cl_page_list_init);
 /**
  * Adds a page to a page list.
  */
-void cl_page_list_add(struct cl_page_list *plist, struct cl_page *page)
+void cl_page_list_add(struct cl_page_list *plist, struct cl_page *page,
+                     bool get_ref)
 {
        ENTRY;
        /* it would be better to check that page is owned by "current" io, but
@@ -856,7 +857,8 @@ void cl_page_list_add(struct cl_page_list *plist, struct cl_page *page)
        list_add_tail(&page->cp_batch, &plist->pl_pages);
        ++plist->pl_nr;
        lu_ref_add_at(&page->cp_reference, &page->cp_queue_ref, "queue", plist);
-       cl_page_get(page);
+       if (get_ref)
+               cl_page_get(page);
        EXIT;
 }
 EXPORT_SYMBOL(cl_page_list_add);
@@ -1033,11 +1035,9 @@ EXPORT_SYMBOL(cl_2queue_init);
 /**
  * Add a page to the incoming page list of 2-queue.
  */
-void cl_2queue_add(struct cl_2queue *queue, struct cl_page *page)
+void cl_2queue_add(struct cl_2queue *queue, struct cl_page *page, bool get_ref)
 {
-        ENTRY;
-        cl_page_list_add(&queue->c2_qin, page);
-        EXIT;
+       cl_page_list_add(&queue->c2_qin, page, get_ref);
 }
 EXPORT_SYMBOL(cl_2queue_add);
 
@@ -1094,10 +1094,10 @@ EXPORT_SYMBOL(cl_2queue_fini);
  */
 void cl_2queue_init_page(struct cl_2queue *queue, struct cl_page *page)
 {
-        ENTRY;
-        cl_2queue_init(queue);
-        cl_2queue_add(queue, page);
-        EXIT;
+       ENTRY;
+       cl_2queue_init(queue);
+       cl_2queue_add(queue, page, true);
+       EXIT;
 }
 EXPORT_SYMBOL(cl_2queue_init_page);
 

diff --git a/lustre/obdecho/echo_client.c b/lustre/obdecho/echo_client.c
index 437466a..4042fe5 100644 (file)
--- a/lustre/obdecho/echo_client.c
+++ b/lustre/obdecho/echo_client.c
@@ -1328,7 +1328,7 @@ static void echo_commit_callback(const struct lu_env *env, struct cl_io *io,
                struct page *vmpage = pvec->pages[i];
                struct cl_page *page = (struct cl_page *)vmpage->private;
 
-               cl_page_list_add(&queue->c2_qout, page);
+               cl_page_list_add(&queue->c2_qout, page, true);
        }
 }
 
@@ -1391,7 +1391,7 @@ static int cl_echo_object_brw(struct echo_object *eco, int rw, u64 offset,
                        break;
                }
 
-               cl_2queue_add(queue, clp);
+               cl_2queue_add(queue, clp, true);
 
                /*
                 * drop the reference count for cl_page_find, so that the page