LU-13799 lov: Cache stripe offset calculation

Calculating the page offset relative to the stripe (etc)
in a file is surprisingly expensive.  Because i/o has
already been split up to stripes by the cl_io code,
calculating the stripe each time is unnecessary.

We cache most of the values requiring calculation.

This improves AIO/DIO page submission significantly,
improving performance by a bit over 10%.

Also remove lpg_generation, which isn't doing anything
useful.  This suggests the possibility of removing
lov_page, but that's for another patch.

This patch reduces i/o time in ms/GiB by:
Write: 17 ms/GiB
Read: 22 ms/GiB

Totals:
Write: 119 ms/GiB
Read: 121 ms/GiB

mpirun -np 1  $IOR -w -r -t 64M -b 64G -o ./iorfile --posix.odirect

With previous patches in series:
write        7531 MiB/s
read         7179 MiB/s

Plus this patch:
write        8637 MiB/s
read         8488 MiB/s

Lustre-change: https://review.whamcloud.com/39445
Lustre-commit: 14db1faa0fbe813fed616435303753d390f45827

Signed-off-by: Patrick Farrell <pfarrell@whamcloud.com>
Change-Id: I046af5eda9560da0fb8daa8ca025484851538a50
Reviewed-on: https://review.whamcloud.com/44681
Tested-by: jenkins <devops@whamcloud.com>
Tested-by: Maloo <maloo@whamcloud.com>
Reviewed-by: Andreas Dilger <adilger@whamcloud.com>

diff --git a/lustre/lov/lov_cl_internal.h b/lustre/lov/lov_cl_internal.h
index 8740d12..a74d61b 100644 (file)
--- a/lustre/lov/lov_cl_internal.h
+++ b/lustre/lov/lov_cl_internal.h
@@ -453,8 +453,6 @@ struct lov_lock {
 
 struct lov_page {
        struct cl_page_slice    lps_cl;
-       /* the layout gen when this page was created */
-       __u32                   lps_layout_gen;
 };
 
 /*
@@ -524,6 +522,7 @@ struct lov_io_sub {
 /**
  * IO state private for LOV.
  */
+#define LIS_CACHE_ENTRY_NONE   -ENOENT
 struct lov_io {
         /** super-class */
         struct cl_io_slice lis_cl;
@@ -589,7 +588,12 @@ struct lov_io {
         * All sub-io's created in this lov_io.
         */
        struct list_head        lis_subios;
-
+       /* Cached results from stripe & offset calculations for page init */
+       int                     lis_cached_entry;
+       int                     lis_cached_stripe;
+       loff_t                  lis_cached_off;
+       loff_t                  lis_cached_suboff;
+       struct lov_io_sub       *lis_cached_sub;
 };
 
 struct lov_session {

diff --git a/lustre/lov/lov_io.c b/lustre/lov/lov_io.c
index da78b08..efe5071 100644 (file)
--- a/lustre/lov/lov_io.c
+++ b/lustre/lov/lov_io.c
@@ -481,6 +481,7 @@ static int lov_io_slice_init(struct lov_io *lio,
 
        io->ci_result = 0;
        lio->lis_object = obj;
+       lio->lis_cached_entry = LIS_CACHE_ENTRY_NONE;
 
        rdonly = lsm_is_rdonly(obj->lo_lsm);
        switch (io->ci_type) {
@@ -1090,6 +1091,11 @@ static void lov_io_end(const struct lu_env *env, const struct cl_io_slice *ios)
 {
        int rc;
 
+       /* Before ending each i/o, we must set lis_cached_entry to tell the
+        * next i/o not to use stale cached lis information.
+        */
+       cl2lov_io(env, ios)->lis_cached_entry = LIS_CACHE_ENTRY_NONE;
+
        rc = lov_io_call(env, cl2lov_io(env, ios), lov_io_end_wrapper);
        LASSERT(rc == 0);
 }

diff --git a/lustre/lov/lov_page.c b/lustre/lov/lov_page.c
index ed0226d..dbdb03e 100644 (file)
--- a/lustre/lov/lov_page.c
+++ b/lustre/lov/lov_page.c
@@ -56,8 +56,7 @@ static int lov_comp_page_print(const struct lu_env *env,
        struct lov_page *lp = cl2lov_page(slice);
 
        return (*printer)(env, cookie,
-                         LUSTRE_LOV_NAME"-page@%p, gen: %u\n",
-                         lp, lp->lps_layout_gen);
+                         LUSTRE_LOV_NAME"-page@%p\n", lp);
 }
 
 static const struct cl_page_operations lov_comp_page_ops = {
@@ -76,34 +75,65 @@ int lov_page_init_composite(const struct lu_env *env, struct cl_object *obj,
        struct lov_layout_raid0 *r0;
        loff_t offset;
        loff_t suboff;
+       bool stripe_cached = false;
        int entry;
        int stripe;
        int rc;
 
        ENTRY;
 
+               /* Direct i/o (CPT_TRANSIENT) is split strictly to stripes, so we can
+        * cache the stripe information.  Buffered i/o is differently
+        * organized, and stripe calculation isn't a significant cost for
+        * buffered i/o, so we only cache this for direct i/o.
+        */
+       stripe_cached = lio->lis_cached_entry != LIS_CACHE_ENTRY_NONE &&
+                       page->cp_type == CPT_TRANSIENT;
+
        offset = cl_offset(obj, index);
-       entry = lov_io_layout_at(lio, offset);
-       LASSERT(equi(entry >= 0, !lsm_entry_is_foreign(loo->lo_lsm, entry)));
+
+       if (stripe_cached) {
+               entry = lio->lis_cached_entry;
+               stripe = lio->lis_cached_stripe;
+               /* Offset can never go backwards in an i/o, so this is valid */
+               suboff = lio->lis_cached_suboff + offset - lio->lis_cached_off;
+       } else {
+               entry = lov_io_layout_at(lio, offset);
+
+               stripe = lov_stripe_number(loo->lo_lsm, entry, offset);
+               rc = lov_stripe_offset(loo->lo_lsm, entry, offset, stripe,
+                                      &suboff);
+               LASSERT(rc == 0);
+               lio->lis_cached_entry = entry;
+               lio->lis_cached_stripe = stripe;
+               lio->lis_cached_off = offset;
+               lio->lis_cached_suboff = suboff;
+       }
+
        if (entry < 0 || !lsm_entry_inited(loo->lo_lsm, entry)) {
                /* non-existing layout component */
                lov_page_init_empty(env, obj, page, index);
                RETURN(0);
        }
 
-       r0 = lov_r0(loo, entry);
-       stripe = lov_stripe_number(loo->lo_lsm, entry, offset);
-       LASSERT(stripe < r0->lo_nr);
-       rc = lov_stripe_offset(loo->lo_lsm, entry, offset, stripe, &suboff);
-       LASSERT(rc == 0);
+       CDEBUG(D_PAGE, "offset %llu, entry %d, stripe %d, suboff %llu\n",
+              offset, entry, stripe, suboff);
 
        page->cp_lov_index = lov_comp_index(entry, stripe);
-       lpg->lps_layout_gen = loo->lo_lsm->lsm_layout_gen;
        cl_page_slice_add(page, &lpg->lps_cl, obj, &lov_comp_page_ops);
 
-       sub = lov_sub_get(env, lio, page->cp_lov_index);
-       if (IS_ERR(sub))
-               RETURN(PTR_ERR(sub));
+       if (!stripe_cached) {
+               sub = lov_sub_get(env, lio, page->cp_lov_index);
+               if (IS_ERR(sub))
+                       RETURN(PTR_ERR(sub));
+       } else {
+               sub = lio->lis_cached_sub;
+       }
+
+       lio->lis_cached_sub = sub;
+
+       r0 = lov_r0(loo, entry);
+       LASSERT(stripe < r0->lo_nr);
 
        subobj = lovsub2cl(r0->lo_sub[stripe]);
        list_for_each_entry(o, &subobj->co_lu.lo_header->loh_layers,