LU-14128 lov: correctly set OST obj size

When extends a PFL file to a size locating at a boundary of a stripe
in a component, the truncate won't set the size of the OST object
in the prior stripe.

This patch record the prior stripe in
lov_layout_raid0::lo_trunc_stripeno and add the stripe in the
truncate IO and enqueue the lock covering it.

Signed-off-by: Bobi Jam <bobijam@whamcloud.com>
Change-Id: Ic5d8e3c16f950003736cd6dbd5af404613f818c7
Reviewed-on: https://review.whamcloud.com/40581
Tested-by: jenkins <devops@whamcloud.com>
Tested-by: Maloo <maloo@whamcloud.com>
Reviewed-by: Andreas Dilger <adilger@whamcloud.com>
Reviewed-by: Mike Pershin <mpershin@whamcloud.com>
Reviewed-by: Oleg Drokin <green@whamcloud.com>

diff --git a/lustre/lov/lov_cl_internal.h b/lustre/lov/lov_cl_internal.h
index 707333f..3a112da 100644 (file)
--- a/lustre/lov/lov_cl_internal.h
+++ b/lustre/lov/lov_cl_internal.h
@@ -176,6 +176,11 @@ struct lov_comp_layout_entry_ops {
 struct lov_layout_raid0 {
        unsigned               lo_nr;
        /**
 struct lov_layout_raid0 {
        unsigned               lo_nr;
        /**

+        * record the stripe no before the truncate size, used for setting OST
+        * object size for truncate. LU-14128.
+        */
+       int                    lo_trunc_stripeno;
+       /**

         * When this is true, lov_object::lo_attr contains
         * valid up to date attributes for a top-level
         * object. This field is reset to 0 when attributes of
         * When this is true, lov_object::lo_attr contains
         * valid up to date attributes for a top-level
         * object. This field is reset to 0 when attributes of

diff --git a/lustre/lov/lov_internal.h b/lustre/lov/lov_internal.h
index d5cff26..8016610 100644 (file)
--- a/lustre/lov/lov_internal.h
+++ b/lustre/lov/lov_internal.h
@@ -274,6 +274,7 @@ int lov_merge_lvb_kms(struct lov_stripe_md *lsm, int index,
                       struct ost_lvb *lvb, __u64 *kms_place);
 
 /* lov_offset.c */
                       struct ost_lvb *lvb, __u64 *kms_place);
 
 /* lov_offset.c */

+loff_t stripe_width(struct lov_stripe_md *lsm, unsigned int index);

 u64 lov_stripe_size(struct lov_stripe_md *lsm, int index,
                    u64 ost_size, int stripeno);
 int lov_stripe_offset(struct lov_stripe_md *lsm, int index, loff_t lov_off,
 u64 lov_stripe_size(struct lov_stripe_md *lsm, int index,
                    u64 ost_size, int stripeno);
 int lov_stripe_offset(struct lov_stripe_md *lsm, int index, loff_t lov_off,

diff --git a/lustre/lov/lov_io.c b/lustre/lov/lov_io.c
index f20c8f4..b9592d1 100644 (file)
--- a/lustre/lov/lov_io.c
+++ b/lustre/lov/lov_io.c
@@ -763,6 +763,23 @@ static loff_t lov_offset_mod(loff_t val, int delta)
        return val;
 }
 
        return val;
 }
 

+static int lov_io_add_sub(const struct lu_env *env, struct lov_io *lio,
+                         struct lov_io_sub *sub, u64 start, u64 end)
+{
+       int rc;
+
+       end = lov_offset_mod(end, 1);
+       lov_io_sub_inherit(sub, lio, start, end);
+       rc = cl_io_iter_init(sub->sub_env, &sub->sub_io);
+       if (rc != 0) {
+               cl_io_iter_fini(sub->sub_env, &sub->sub_io);
+               return rc;
+       }
+
+       list_add_tail(&sub->sub_linkage, &lio->lis_active);
+
+       return rc;
+}

 static int lov_io_iter_init(const struct lu_env *env,
                            const struct cl_io_slice *ios)
 {
 static int lov_io_iter_init(const struct lu_env *env,
                            const struct cl_io_slice *ios)
 {


@@ -784,6 +801,9 @@ static int lov_io_iter_init(const struct lu_env *env,
                u64 start;
                u64 end;
                int stripe;
                u64 start;
                u64 end;
                int stripe;

+               bool tested_trunc_stripe = false;
+
+               r0->lo_trunc_stripeno = -1;

 
                CDEBUG(D_VFSTRACE, "component[%d] flags %#x\n",
                       index, lsm->lsm_entries[index]->lsme_flags);
 
                CDEBUG(D_VFSTRACE, "component[%d] flags %#x\n",
                       index, lsm->lsm_entries[index]->lsme_flags);


@@ -815,28 +835,79 @@ static int lov_io_iter_init(const struct lu_env *env,
                                continue;
                        }
 
                                continue;
                        }
 

-                       end = lov_offset_mod(end, 1);
+                       if (cl_io_is_trunc(ios->cis_io) &&
+                           !tested_trunc_stripe) {
+                               int prev;
+                               u64 tr_start;
+
+                               prev = (stripe == 0) ? r0->lo_nr - 1 :
+                                                       stripe - 1;
+                               /**
+                                * Only involving previous stripe if the
+                                * truncate in this component is at the
+                                * beginning of this stripe.
+                                */
+                               tested_trunc_stripe = true;
+                               if (ext.e_start < lsm->lsm_entries[index]->
+                                                       lsme_extent.e_start) {
+                                       /* need previous stripe involvement */
+                                       r0->lo_trunc_stripeno = prev;
+                               } else {
+                                       tr_start = ext.e_start;
+                                       tr_start = lov_do_div64(tr_start,
+                                                     stripe_width(lsm, index));
+                                       /* tr_start %= stripe_swidth */
+                                       if (tr_start == stripe * lsm->
+                                                       lsm_entries[index]->
+                                                       lsme_stripe_size)
+                                               r0->lo_trunc_stripeno = prev;
+                               }
+                       }
+
+                       /* if the last stripe is the trunc stripeno */
+                       if (r0->lo_trunc_stripeno == stripe)
+                               r0->lo_trunc_stripeno = -1;
+

                        sub = lov_sub_get(env, lio,
                                          lov_comp_index(index, stripe));
                        sub = lov_sub_get(env, lio,
                                          lov_comp_index(index, stripe));

-                       if (IS_ERR(sub)) {
-                               rc = PTR_ERR(sub);
+                       if (IS_ERR(sub))
+                               return PTR_ERR(sub);
+
+                       rc = lov_io_add_sub(env, lio, sub, start, end);
+                       if (rc != 0)

                                break;
                                break;

+               }
+               if (rc != 0)
+                       break;
+
+               if (r0->lo_trunc_stripeno != -1) {
+                       stripe = r0->lo_trunc_stripeno;
+                       if (unlikely(!r0->lo_sub[stripe])) {
+                               r0->lo_trunc_stripeno = -1;
+                               continue;

                        }
                        }

+                       sub = lov_sub_get(env, lio,
+                                         lov_comp_index(index, stripe));
+                       if (IS_ERR(sub))
+                               return PTR_ERR(sub);

 
 

-                       lov_io_sub_inherit(sub, lio, start, end);
-                       rc = cl_io_iter_init(sub->sub_env, &sub->sub_io);
-                       if (rc != 0)
-                               cl_io_iter_fini(sub->sub_env, &sub->sub_io);
+                       /**
+                        * the prev sub could be used by another truncate, we'd
+                        * skip it. LU-14128 happends when expand truncate +
+                        * read get wrong kms.
+                        */
+                       if (!list_empty(&sub->sub_linkage)) {
+                               r0->lo_trunc_stripeno = -1;
+                               continue;
+                       }
+
+                       (void)lov_stripe_intersects(lsm, index, stripe, &ext,
+                                                   &start, &end);
+                       rc = lov_io_add_sub(env, lio, sub, start, end);

                        if (rc != 0)
                                break;
 
                        if (rc != 0)
                                break;
 

-                       CDEBUG(D_VFSTRACE, "shrink: %d [%llu, %llu)\n",
-                              stripe, start, end);
-
-                       list_add_tail(&sub->sub_linkage, &lio->lis_active);

                }
                }

-               if (rc != 0)
-                       break;

        }
        RETURN(rc);
 }
        }
        RETURN(rc);
 }

diff --git a/lustre/lov/lov_lock.c b/lustre/lov/lov_lock.c
index 2e39c05..1b4a958 100644 (file)
--- a/lustre/lov/lov_lock.c
+++ b/lustre/lov/lov_lock.c
@@ -112,6 +112,7 @@ static int lov_sublock_init(const struct lu_env *env,
  * through already created sub-locks (possibly shared with other top-locks).
  */
 static struct lov_lock *lov_lock_sub_init(const struct lu_env *env,
  * through already created sub-locks (possibly shared with other top-locks).
  */
 static struct lov_lock *lov_lock_sub_init(const struct lu_env *env,

+                                         const struct cl_io *io,

                                          const struct cl_object *obj,
                                          struct cl_lock *lock)
 {
                                          const struct cl_object *obj,
                                          struct cl_lock *lock)
 {


@@ -138,10 +139,14 @@ static struct lov_lock *lov_lock_sub_init(const struct lu_env *env,
                struct lov_layout_raid0 *r0 = lov_r0(lov, index);
 
                for (i = 0; i < r0->lo_nr; i++) {
                struct lov_layout_raid0 *r0 = lov_r0(lov, index);
 
                for (i = 0; i < r0->lo_nr; i++) {

-                       if (likely(r0->lo_sub[i]) && /* spare layout */
-                           lov_stripe_intersects(lov->lo_lsm, index, i,
-                                                 &ext, &start, &end))
-                               nr++;
+                       if (likely(r0->lo_sub[i])) {/* spare layout */
+                               if (lov_stripe_intersects(lov->lo_lsm, index, i,
+                                                         &ext, &start, &end))
+                                       nr++;
+                               else if (cl_io_is_trunc(io) &&
+                                        r0->lo_trunc_stripeno == i)
+                                       nr++;
+                       }

                }
        }
        /**
                }
        }
        /**


@@ -162,12 +167,22 @@ static struct lov_lock *lov_lock_sub_init(const struct lu_env *env,
                for (i = 0; i < r0->lo_nr; ++i) {
                        struct lov_lock_sub *lls = &lovlck->lls_sub[nr];
                        struct cl_lock_descr *descr = &lls->sub_lock.cll_descr;
                for (i = 0; i < r0->lo_nr; ++i) {
                        struct lov_lock_sub *lls = &lovlck->lls_sub[nr];
                        struct cl_lock_descr *descr = &lls->sub_lock.cll_descr;

+                       bool intersect = false;

 
 

-                       if (unlikely(!r0->lo_sub[i]) ||
-                           !lov_stripe_intersects(lov->lo_lsm, index, i,
-                                                  &ext, &start, &end))
+                       if (unlikely(!r0->lo_sub[i]))

                                continue;
 
                                continue;
 

+                       intersect = lov_stripe_intersects(lov->lo_lsm, index, i,
+                                                         &ext, &start, &end);
+                       if (intersect)
+                               goto init_sublock;
+
+                       if (cl_io_is_trunc(io) && i == r0->lo_trunc_stripeno)
+                               goto init_sublock;
+
+                       continue;
+
+init_sublock:

                        LASSERT(descr->cld_obj == NULL);
                        descr->cld_obj   = lovsub2cl(r0->lo_sub[i]);
                        descr->cld_start = cl_index(descr->cld_obj, start);
                        LASSERT(descr->cld_obj == NULL);
                        descr->cld_obj   = lovsub2cl(r0->lo_sub[i]);
                        descr->cld_start = cl_index(descr->cld_obj, start);


@@ -321,7 +336,7 @@ int lov_lock_init_composite(const struct lu_env *env, struct cl_object *obj,
        int result = 0;
 
        ENTRY;
        int result = 0;
 
        ENTRY;

-       lck = lov_lock_sub_init(env, obj, lock);
+       lck = lov_lock_sub_init(env, io, obj, lock);

        if (!IS_ERR(lck))
                cl_lock_slice_add(lock, &lck->lls_cl, obj, &lov_lock_ops);
        else
        if (!IS_ERR(lck))
                cl_lock_slice_add(lock, &lck->lls_cl, obj, &lov_lock_ops);
        else

diff --git a/lustre/lov/lov_object.c b/lustre/lov/lov_object.c
index eb385d3..1491851 100644 (file)
--- a/lustre/lov/lov_object.c
+++ b/lustre/lov/lov_object.c
@@ -216,6 +216,7 @@ static int lov_init_raid0(const struct lu_env *env, struct lov_device *dev,
 
        spin_lock_init(&r0->lo_sub_lock);
        r0->lo_nr = lse->lsme_stripe_count;
 
        spin_lock_init(&r0->lo_sub_lock);
        r0->lo_nr = lse->lsme_stripe_count;

+       r0->lo_trunc_stripeno = -1;

 
        OBD_ALLOC_PTR_ARRAY_LARGE(r0->lo_sub, r0->lo_nr);
        if (r0->lo_sub == NULL)
 
        OBD_ALLOC_PTR_ARRAY_LARGE(r0->lo_sub, r0->lo_nr);
        if (r0->lo_sub == NULL)

diff --git a/lustre/lov/lov_offset.c b/lustre/lov/lov_offset.c
index 8540ccd..bdab64e 100644 (file)
--- a/lustre/lov/lov_offset.c
+++ b/lustre/lov/lov_offset.c
@@ -38,7 +38,7 @@
 
 #include "lov_internal.h"
 
 
 #include "lov_internal.h"
 

-static loff_t stripe_width(struct lov_stripe_md *lsm, unsigned int index)
+loff_t stripe_width(struct lov_stripe_md *lsm, unsigned int index)

 {
        struct lov_stripe_md_entry *entry = lsm->lsm_entries[index];
 
 {
        struct lov_stripe_md_entry *entry = lsm->lsm_entries[index];
 

diff --git a/lustre/tests/Makefile.am b/lustre/tests/Makefile.am
index f736fca..9831e7a 100644 (file)
--- a/lustre/tests/Makefile.am
+++ b/lustre/tests/Makefile.am
@@ -76,7 +76,7 @@ THETESTS += group_lock_test llapi_fid_test sendfile_grouplock mmap_cat
 THETESTS += swap_lock_test lockahead_test mirror_io mmap_mknod_test
 THETESTS += create_foreign_file parse_foreign_file
 THETESTS += create_foreign_dir parse_foreign_dir
 THETESTS += swap_lock_test lockahead_test mirror_io mmap_mknod_test
 THETESTS += create_foreign_file parse_foreign_file
 THETESTS += create_foreign_dir parse_foreign_dir

-THETESTS += check_fallocate splice-test lseek_test
+THETESTS += check_fallocate splice-test lseek_test expand_truncate_test

 
 if LIBAIO
 THETESTS += aiocp
 
 if LIBAIO
 THETESTS += aiocp

diff --git a/lustre/tests/expand_truncate_test.c b/lustre/tests/expand_truncate_test.c
new file mode 100644 (file)
index 0000000..d59e67b
--- /dev/null
+++ b/lustre/tests/expand_truncate_test.c
@@ -0,0 +1,55 @@
+#include <stdio.h>
+#include <unistd.h>
+#include <errno.h>
+#include <sys/file.h>
+
+int main(int argc, char **argv)
+{
+       char *fname = argv[1];
+       char buf[5];
+       int fd;
+       off_t off;
+       int rc;
+
+       if (argc != 2) {
+               fprintf(stdout, "usage: %s file\n", argv[0]);
+               return 1;
+       }
+
+       fd = open(fname, O_RDWR | O_CREAT, 0666);
+       if (fd < 0) {
+               fprintf(stderr, "open %s failed:%d\n", fname, errno);
+               return fd;
+       }
+
+       off = 1021 * 1024 * 1024;
+       if (ftruncate(fd, off) < 0) {
+               fprintf(stderr, "ftruncate %ld failed:%d\n", off, errno);
+               rc = -1;
+               goto close;
+       }
+
+       off -= 4;
+       off = lseek(fd, off, SEEK_SET);
+       if (off == (off_t)-1) {
+               fprintf(stderr, "lseek %ld failed:%d\n", off, errno);
+               rc = -1;
+               goto close;
+       }
+
+       rc = read(fd, buf, 4);
+       if (rc < 0) {
+               fprintf(stderr, "read 4 bytes failed:%d\n", errno);
+               goto close;
+       } else if (rc != 4) {
+               fprintf(stderr, "read returns %d, not 4 bytes\n", rc);
+               rc = -1;
+       } else {
+               rc = 0;
+       }
+
+close:
+       close(fd);
+
+       return rc;
+}

diff --git a/lustre/tests/sanity-pfl.sh b/lustre/tests/sanity-pfl.sh
index d70edaa..664acc2 100644 (file)
--- a/lustre/tests/sanity-pfl.sh
+++ b/lustre/tests/sanity-pfl.sh
@@ -556,7 +556,7 @@ test_10() {
 }
 run_test 10 "Inherit composite template from root"
 
 }
 run_test 10 "Inherit composite template from root"
 

-test_11() {
+test_11a() {

        local comp_file=$DIR/$tdir/$tfile
        test_mkdir $DIR/$tdir
        rm -f $comp_file
        local comp_file=$DIR/$tdir/$tfile
        test_mkdir $DIR/$tdir
        rm -f $comp_file


@@ -607,7 +607,22 @@ test_11() {
 
        return 0
 }
 
        return 0
 }

-run_test 11 "Verify component instantiation with write/truncate"
+run_test 11a "Verify component instantiation with write/truncate"
+
+test_11b() {
+       [ $OSTCOUNT -lt 4 ] && skip "needs >= 4 OSTs"
+
+       local file=$DIR/$tdir/$tfile
+
+       test_mkdir $DIR/$tdir
+       rm -f $file
+
+       $LFS setstripe -E 1m -E 1g -c 4 -E eof $DIR/$tdir ||
+               error "setstripe dir $DIR/$tdir failed"
+       expand_truncate_test $file ||
+               error "expand_truncate_test failed on $file"
+}
+run_test 11b "truncate file set file size correctly"

 
 test_12() {
        [ $OSTCOUNT -lt 3 ] && skip "needs >= 3 OSTs"
 
 test_12() {
        [ $OSTCOUNT -lt 3 ] && skip "needs >= 3 OSTs"