X-Git-Url: https://git.whamcloud.com/?a=blobdiff_plain;f=lustre%2Fllite%2Frw26.c;h=ab6bec998f03aaa03fcd18bb9c57fb45e83164c8;hb=6bce536725efd166d2772f13fe954f271f9c53b8;hp=ee17a73cd409a97b2dce210c35b74c2a2bbc01f2;hpb=0b1ad400c8f64575292a7ff54a8ce872a124b19e;p=fs%2Flustre-release.git diff --git a/lustre/llite/rw26.c b/lustre/llite/rw26.c index ee17a73..ab6bec9 100644 --- a/lustre/llite/rw26.c +++ b/lustre/llite/rw26.c @@ -15,11 +15,7 @@ * * You should have received a copy of the GNU General Public License * version 2 along with this program; If not, see - * http://www.sun.com/software/products/lustre/docs/GPLv2.pdf - * - * Please contact Sun Microsystems, Inc., 4150 Network Circle, Santa Clara, - * CA 95054 USA or visit www.sun.com if you need additional information or - * have any questions. + * http://www.gnu.org/licenses/gpl-2.0.html * * GPL HEADER END */ @@ -27,7 +23,7 @@ * Copyright (c) 2003, 2010, Oracle and/or its affiliates. All rights reserved. * Use is subject to license terms. * - * Copyright (c) 2011, 2014, Intel Corporation. + * Copyright (c) 2011, 2017, Intel Corporation. */ /* * This file is part of Lustre, http://www.lustre.org/ @@ -38,27 +34,22 @@ * Lustre Lite I/O page cache routines for the 2.5/2.6 kernel version */ +#include +#include +#include #include #include +#include +#include #include -#include -#include #include -#include +#include #ifdef HAVE_MIGRATE_H #include #elif defined(HAVE_MIGRATE_MODE_H) #include #endif -#include -#include -#include -#include -#include -#include -#include -#include #define DEBUG_SUBSYSTEM S_LLITE @@ -88,8 +79,6 @@ static void ll_invalidatepage(struct page *vmpage, struct cl_page *page; struct cl_object *obj; - __u16 refcheck; - LASSERT(PageLocked(vmpage)); LASSERT(!PageWriteback(vmpage)); @@ -99,24 +88,26 @@ static void ll_invalidatepage(struct page *vmpage, * happening with locked page too */ #ifdef HAVE_INVALIDATE_RANGE - if (offset == 0 && length == PAGE_CACHE_SIZE) { + if (offset == 0 && length == PAGE_SIZE) { #else if (offset == 0) { #endif - env = cl_env_get(&refcheck); - if (!IS_ERR(env)) { - inode = vmpage->mapping->host; - obj = ll_i2info(inode)->lli_clob; - if (obj != NULL) { - page = cl_vmpage_page(vmpage, obj); - if (page != NULL) { - cl_page_delete(env, page); - cl_page_put(env, page); - } - } else - LASSERT(vmpage->private == 0); - cl_env_put(env, &refcheck); - } + /* See the comment in ll_releasepage() */ + env = cl_env_percpu_get(); + LASSERT(!IS_ERR(env)); + + inode = vmpage->mapping->host; + obj = ll_i2info(inode)->lli_clob; + if (obj != NULL) { + page = cl_vmpage_page(vmpage, obj); + if (page != NULL) { + cl_page_delete(env, page); + cl_page_put(env, page); + } + } else + LASSERT(vmpage->private == 0); + + cl_env_percpu_put(env); } } @@ -128,7 +119,6 @@ static void ll_invalidatepage(struct page *vmpage, static int ll_releasepage(struct page *vmpage, RELEASEPAGE_ARG_TYPE gfp_mask) { struct lu_env *env; - void *cookie; struct cl_object *obj; struct cl_page *page; struct address_space *mapping; @@ -146,15 +136,10 @@ static int ll_releasepage(struct page *vmpage, RELEASEPAGE_ARG_TYPE gfp_mask) if (obj == NULL) return 1; - /* 1 for caller, 1 for cl_page and 1 for page cache */ - if (page_count(vmpage) > 3) - return 0; - page = cl_vmpage_page(vmpage, obj); if (page == NULL) return 1; - cookie = cl_env_reenter(); env = cl_env_percpu_get(); LASSERT(!IS_ERR(env)); @@ -180,131 +165,112 @@ static int ll_releasepage(struct page *vmpage, RELEASEPAGE_ARG_TYPE gfp_mask) cl_page_put(env, page); cl_env_percpu_put(env); - cl_env_reexit(cookie); return result; } #define MAX_DIRECTIO_SIZE 2*1024*1024*1024UL -ssize_t ll_direct_rw_pages(const struct lu_env *env, struct cl_io *io, - int rw, struct inode *inode, - struct ll_dio_pages *pv) +static ssize_t +ll_direct_IO_seg(const struct lu_env *env, struct cl_io *io, int rw, + struct inode *inode, size_t size, loff_t file_offset, + struct page **pages, int page_count) { - struct cl_page *clp; - struct cl_2queue *queue; - struct cl_object *obj = io->ci_obj; + struct cl_page *clp; + struct cl_2queue *queue; + struct cl_object *obj = io->ci_obj; int i; ssize_t rc = 0; - loff_t file_offset = pv->ldp_start_offset; - size_t size = pv->ldp_size; - int page_count = pv->ldp_nr; - struct page **pages = pv->ldp_pages; - size_t page_size = cl_page_size(obj); + size_t page_size = cl_page_size(obj); + size_t orig_size = size; bool do_io; - int io_pages = 0; - ENTRY; + int io_pages = 0; - queue = &io->ci_queue; - cl_2queue_init(queue); - for (i = 0; i < page_count; i++) { - if (pv->ldp_offsets) - file_offset = pv->ldp_offsets[i]; - - LASSERT(!(file_offset & (page_size - 1))); - clp = cl_page_find(env, obj, cl_index(obj, file_offset), - pv->ldp_pages[i], CPT_TRANSIENT); - if (IS_ERR(clp)) { - rc = PTR_ERR(clp); - break; - } + ENTRY; + queue = &io->ci_queue; + cl_2queue_init(queue); + for (i = 0; i < page_count; i++) { + LASSERT(!(file_offset & (page_size - 1))); + clp = cl_page_find(env, obj, cl_index(obj, file_offset), + pages[i], CPT_TRANSIENT); + if (IS_ERR(clp)) { + rc = PTR_ERR(clp); + break; + } - rc = cl_page_own(env, io, clp); - if (rc) { - LASSERT(clp->cp_state == CPS_FREEING); - cl_page_put(env, clp); - break; - } + rc = cl_page_own(env, io, clp); + if (rc) { + LASSERT(clp->cp_state == CPS_FREEING); + cl_page_put(env, clp); + break; + } - do_io = true; + do_io = true; - /* check the page type: if the page is a host page, then do - * write directly */ - if (clp->cp_type == CPT_CACHEABLE) { + /* check the page type: if the page is a host page, then do + * write directly + */ + if (clp->cp_type == CPT_CACHEABLE) { struct page *vmpage = cl_page_vmpage(clp); struct page *src_page; struct page *dst_page; - void *src; - void *dst; - - src_page = (rw == WRITE) ? pages[i] : vmpage; - dst_page = (rw == WRITE) ? vmpage : pages[i]; - - src = ll_kmap_atomic(src_page, KM_USER0); - dst = ll_kmap_atomic(dst_page, KM_USER1); - memcpy(dst, src, min(page_size, size)); - ll_kunmap_atomic(dst, KM_USER1); - ll_kunmap_atomic(src, KM_USER0); - - /* make sure page will be added to the transfer by - * cl_io_submit()->...->vvp_page_prep_write(). */ - if (rw == WRITE) - set_page_dirty(vmpage); - - if (rw == READ) { - /* do not issue the page for read, since it - * may reread a ra page which has NOT uptodate - * bit set. */ - cl_page_disown(env, io, clp); - do_io = false; - } - } + void *src; + void *dst; + + src_page = (rw == WRITE) ? pages[i] : vmpage; + dst_page = (rw == WRITE) ? vmpage : pages[i]; + + src = ll_kmap_atomic(src_page, KM_USER0); + dst = ll_kmap_atomic(dst_page, KM_USER1); + memcpy(dst, src, min(page_size, size)); + ll_kunmap_atomic(dst, KM_USER1); + ll_kunmap_atomic(src, KM_USER0); + + /* make sure page will be added to the transfer by + * cl_io_submit()->...->vvp_page_prep_write(). + */ + if (rw == WRITE) + set_page_dirty(vmpage); + + if (rw == READ) { + /* do not issue the page for read, since it + * may reread a ra page which has NOT uptodate + * bit set. + */ + cl_page_disown(env, io, clp); + do_io = false; + } + } - if (likely(do_io)) { - cl_2queue_add(queue, clp); + if (likely(do_io)) { + cl_2queue_add(queue, clp); - /* - * Set page clip to tell transfer formation engine - * that page has to be sent even if it is beyond KMS. - */ - cl_page_clip(env, clp, 0, min(size, page_size)); + /* + * Set page clip to tell transfer formation engine + * that page has to be sent even if it is beyond KMS. + */ + cl_page_clip(env, clp, 0, min(size, page_size)); - ++io_pages; - } + ++io_pages; + } - /* drop the reference count for cl_page_find */ - cl_page_put(env, clp); - size -= page_size; - file_offset += page_size; - } + /* drop the reference count for cl_page_find */ + cl_page_put(env, clp); + size -= page_size; + file_offset += page_size; + } - if (rc == 0 && io_pages) { - rc = cl_io_submit_sync(env, io, - rw == READ ? CRT_READ : CRT_WRITE, + if (rc == 0 && io_pages) { + rc = cl_io_submit_sync(env, io, + rw == READ ? CRT_READ : CRT_WRITE, queue, 0); - } - if (rc == 0) - rc = pv->ldp_size; - - cl_2queue_discard(env, io, queue); - cl_2queue_disown(env, io, queue); - cl_2queue_fini(env, queue); - RETURN(rc); -} -EXPORT_SYMBOL(ll_direct_rw_pages); + } + if (rc == 0) + rc = orig_size; -static ssize_t -ll_direct_IO_seg(const struct lu_env *env, struct cl_io *io, int rw, - struct inode *inode, size_t size, loff_t file_offset, - struct page **pages, int page_count) -{ - struct ll_dio_pages pvec = { .ldp_pages = pages, - .ldp_nr = page_count, - .ldp_size = size, - .ldp_offsets = NULL, - .ldp_start_offset = file_offset - }; - - return ll_direct_rw_pages(env, io, rw, inode, &pvec); + cl_2queue_discard(env, io, queue); + cl_2queue_disown(env, io, queue); + cl_2queue_fini(env, queue); + RETURN(rc); } /* ll_free_user_pages - tear down page struct array @@ -318,7 +284,7 @@ static void ll_free_user_pages(struct page **pages, int npages, int do_dirty) break; if (do_dirty) set_page_dirty_lock(pages[i]); - page_cache_release(pages[i]); + put_page(pages[i]); } #if defined(HAVE_DIRECTIO_ITER) || defined(HAVE_IOV_ITER_RW) @@ -339,7 +305,7 @@ static void ll_free_user_pages(struct page **pages, int npages, int do_dirty) * representing PAGE_SIZE worth of user data, into a single buffer, and * then truncate this to be a full-sized RPC. For 4kB PAGE_SIZE this is * up to 22MB for 128kB kmalloc and up to 682MB for 4MB kmalloc. */ -#define MAX_DIO_SIZE ((MAX_MALLOC / sizeof(struct brw_page) * PAGE_CACHE_SIZE) & \ +#define MAX_DIO_SIZE ((MAX_MALLOC / sizeof(struct brw_page) * PAGE_SIZE) & \ ~(DT_MAX_BRW_SIZE - 1)) #ifndef HAVE_IOV_ITER_RW @@ -352,18 +318,27 @@ ll_direct_IO( # ifndef HAVE_IOV_ITER_RW int rw, # endif - struct kiocb *iocb, struct iov_iter *iter, - loff_t file_offset) + struct kiocb *iocb, struct iov_iter *iter +# ifndef HAVE_DIRECTIO_2ARGS + , loff_t file_offset +# endif + ) { - struct lu_env *env; +#ifdef HAVE_DIRECTIO_2ARGS + loff_t file_offset = iocb->ki_pos; +#endif + struct ll_cl_context *lcc; + const struct lu_env *env; struct cl_io *io; struct file *file = iocb->ki_filp; struct inode *inode = file->f_mapping->host; ssize_t count = iov_iter_count(iter); ssize_t tot_bytes = 0, result = 0; size_t size = MAX_DIO_SIZE; - __u16 refcheck; + /* Check EOF by ourselves */ + if (iov_iter_rw(iter) == READ && file_offset >= i_size_read(inode)) + return 0; /* FIXME: io smaller than PAGE_SIZE is broken on ia64 ??? */ if ((file_offset & ~PAGE_MASK) || (count & ~PAGE_MASK)) return -EINVAL; @@ -371,16 +346,20 @@ ll_direct_IO( CDEBUG(D_VFSTRACE, "VFS Op:inode="DFID"(%p), size=%zd (max %lu), " "offset=%lld=%llx, pages %zd (max %lu)\n", PFID(ll_inode2fid(inode)), inode, count, MAX_DIO_SIZE, - file_offset, file_offset, count >> PAGE_CACHE_SHIFT, - MAX_DIO_SIZE >> PAGE_CACHE_SHIFT); + file_offset, file_offset, count >> PAGE_SHIFT, + MAX_DIO_SIZE >> PAGE_SHIFT); /* Check that all user buffers are aligned as well */ if (iov_iter_alignment(iter) & ~PAGE_MASK) return -EINVAL; - env = cl_env_get(&refcheck); + lcc = ll_cl_find(file); + if (lcc == NULL) + RETURN(-EIO); + + env = lcc->lcc_env; LASSERT(!IS_ERR(env)); - io = vvp_env_io(env)->vui_cl.cis_io; + io = lcc->lcc_io; LASSERT(io != NULL); /* 0. Need locking between buffered and direct access. and race with @@ -388,7 +367,7 @@ ll_direct_IO( * 1. Need inode mutex to operate transient pages. */ if (iov_iter_rw(iter) == READ) - mutex_lock(&inode->i_mutex); + inode_lock(inode); while (iov_iter_count(iter)) { struct page **pages; @@ -421,8 +400,8 @@ ll_direct_IO( * We should always be able to kmalloc for a * page worth of page pointers = 4MB on i386. */ if (result == -ENOMEM && - size > (PAGE_CACHE_SIZE / sizeof(*pages)) * - PAGE_CACHE_SIZE) { + size > (PAGE_SIZE / sizeof(*pages)) * + PAGE_SIZE) { size = ((((size / 2) - 1) | ~PAGE_MASK) + 1) & PAGE_MASK; CDEBUG(D_VFSTRACE, "DIO size now %zu\n", @@ -439,7 +418,7 @@ ll_direct_IO( } out: if (iov_iter_rw(iter) == READ) - mutex_unlock(&inode->i_mutex); + inode_unlock(inode); if (tot_bytes > 0) { struct vvp_io *vio = vvp_env_io(env); @@ -448,7 +427,6 @@ out: vio->u.write.vui_written += tot_bytes; } - cl_env_put(env, &refcheck); return tot_bytes ? : result; } #else /* !HAVE_DIRECTIO_ITER && !HAVE_IOV_ITER_RW */ @@ -465,9 +443,9 @@ static inline int ll_get_user_pages(int rw, unsigned long user_addr, return -EFBIG; } - *max_pages = (user_addr + size + PAGE_CACHE_SIZE - 1) >> - PAGE_CACHE_SHIFT; - *max_pages -= user_addr >> PAGE_CACHE_SHIFT; + *max_pages = (user_addr + size + PAGE_SIZE - 1) >> + PAGE_SHIFT; + *max_pages -= user_addr >> PAGE_SHIFT; OBD_ALLOC_LARGE(*pages, *max_pages * sizeof(**pages)); if (*pages) { @@ -487,7 +465,8 @@ static ssize_t ll_direct_IO(int rw, struct kiocb *iocb, const struct iovec *iov, loff_t file_offset, unsigned long nr_segs) { - struct lu_env *env; + struct ll_cl_context *lcc; + const struct lu_env *env; struct cl_io *io; struct file *file = iocb->ki_filp; struct inode *inode = file->f_mapping->host; @@ -495,7 +474,6 @@ ll_direct_IO(int rw, struct kiocb *iocb, const struct iovec *iov, ssize_t tot_bytes = 0, result = 0; unsigned long seg = 0; size_t size = MAX_DIO_SIZE; - __u16 refcheck; ENTRY; /* FIXME: io smaller than PAGE_SIZE is broken on ia64 ??? */ @@ -505,8 +483,8 @@ ll_direct_IO(int rw, struct kiocb *iocb, const struct iovec *iov, CDEBUG(D_VFSTRACE, "VFS Op:inode="DFID"(%p), size=%zd (max %lu), " "offset=%lld=%llx, pages %zd (max %lu)\n", PFID(ll_inode2fid(inode)), inode, count, MAX_DIO_SIZE, - file_offset, file_offset, count >> PAGE_CACHE_SHIFT, - MAX_DIO_SIZE >> PAGE_CACHE_SHIFT); + file_offset, file_offset, count >> PAGE_SHIFT, + MAX_DIO_SIZE >> PAGE_SHIFT); /* Check that all user buffers are aligned as well */ for (seg = 0; seg < nr_segs; seg++) { @@ -515,10 +493,14 @@ ll_direct_IO(int rw, struct kiocb *iocb, const struct iovec *iov, RETURN(-EINVAL); } - env = cl_env_get(&refcheck); - LASSERT(!IS_ERR(env)); - io = vvp_env_io(env)->vui_cl.cis_io; - LASSERT(io != NULL); + lcc = ll_cl_find(file); + if (lcc == NULL) + RETURN(-EIO); + + env = lcc->lcc_env; + LASSERT(!IS_ERR(env)); + io = lcc->lcc_io; + LASSERT(io != NULL); for (seg = 0; seg < nr_segs; seg++) { size_t iov_left = iov[seg].iov_len; @@ -541,7 +523,7 @@ ll_direct_IO(int rw, struct kiocb *iocb, const struct iovec *iov, &pages, &max_pages); if (likely(page_count > 0)) { if (unlikely(page_count < max_pages)) - bytes = page_count << PAGE_CACHE_SHIFT; + bytes = page_count << PAGE_SHIFT; result = ll_direct_IO_seg(env, io, rw, inode, bytes, file_offset, pages, page_count); @@ -558,11 +540,11 @@ ll_direct_IO(int rw, struct kiocb *iocb, const struct iovec *iov, * We should always be able to kmalloc for a * page worth of page pointers = 4MB on i386. */ if (result == -ENOMEM && - size > (PAGE_CACHE_SIZE / sizeof(*pages)) * - PAGE_CACHE_SIZE) { + size > (PAGE_SIZE / sizeof(*pages)) * + PAGE_SIZE) { size = ((((size / 2) - 1) | - ~PAGE_CACHE_MASK) + 1) & - PAGE_CACHE_MASK; + ~PAGE_MASK) + 1) & + PAGE_MASK; CDEBUG(D_VFSTRACE, "DIO size now %zu\n", size); continue; @@ -585,58 +567,95 @@ out: vio->u.write.vui_written += tot_bytes; } - cl_env_put(env, &refcheck); RETURN(tot_bytes ? tot_bytes : result); } #endif /* HAVE_DIRECTIO_ITER || HAVE_IOV_ITER_RW */ /** * Prepare partially written-to page for a write. + * @pg is owned when passed in and disowned when it returns non-zero result to + * the caller. */ static int ll_prepare_partial_page(const struct lu_env *env, struct cl_io *io, - struct cl_page *pg) + struct cl_page *pg, struct file *file) { struct cl_attr *attr = vvp_env_thread_attr(env); struct cl_object *obj = io->ci_obj; struct vvp_page *vpg = cl_object_page_slice(obj, pg); loff_t offset = cl_offset(obj, vvp_index(vpg)); int result; + ENTRY; cl_object_attr_lock(obj); result = cl_object_attr_get(env, obj, attr); cl_object_attr_unlock(obj); - if (result == 0) { - /* - * If are writing to a new page, no need to read old data. - * The extent locking will have updated the KMS, and for our - * purposes here we can treat it like i_size. - */ - if (attr->cat_kms <= offset) { - char *kaddr = ll_kmap_atomic(vpg->vpg_page, KM_USER0); + if (result) { + cl_page_disown(env, io, pg); + GOTO(out, result); + } - memset(kaddr, 0, cl_page_size(obj)); - ll_kunmap_atomic(kaddr, KM_USER0); - } else if (vpg->vpg_defer_uptodate) - vpg->vpg_ra_used = 1; - else - result = ll_page_sync_io(env, io, pg, CRT_READ); + /* + * If are writing to a new page, no need to read old data. + * The extent locking will have updated the KMS, and for our + * purposes here we can treat it like i_size. + */ + if (attr->cat_kms <= offset) { + char *kaddr = ll_kmap_atomic(vpg->vpg_page, KM_USER0); + + memset(kaddr, 0, cl_page_size(obj)); + ll_kunmap_atomic(kaddr, KM_USER0); + GOTO(out, result = 0); + } + + if (vpg->vpg_defer_uptodate) { + vpg->vpg_ra_used = 1; + GOTO(out, result = 0); } + + result = ll_io_read_page(env, io, pg, file); + if (result) + GOTO(out, result); + + /* ll_io_read_page() disowns the page */ + result = cl_page_own(env, io, pg); + if (!result) { + if (!PageUptodate(cl_page_vmpage(pg))) { + cl_page_disown(env, io, pg); + result = -EIO; + } + } else if (result == -ENOENT) { + /* page was truncated */ + result = -EAGAIN; + } + EXIT; + +out: return result; } +static int ll_tiny_write_begin(struct page *vmpage) +{ + /* Page must be present, up to date, dirty, and not in writeback. */ + if (!vmpage || !PageUptodate(vmpage) || !PageDirty(vmpage) || + PageWriteback(vmpage)) + return -ENODATA; + + return 0; +} + static int ll_write_begin(struct file *file, struct address_space *mapping, loff_t pos, unsigned len, unsigned flags, struct page **pagep, void **fsdata) { - struct ll_cl_context *lcc; - const struct lu_env *env; - struct cl_io *io; - struct cl_page *page; + struct ll_cl_context *lcc = NULL; + const struct lu_env *env = NULL; + struct cl_io *io = NULL; + struct cl_page *page = NULL; struct cl_object *clob = ll_i2info(mapping->host)->lli_clob; - pgoff_t index = pos >> PAGE_CACHE_SHIFT; + pgoff_t index = pos >> PAGE_SHIFT; struct page *vmpage = NULL; - unsigned from = pos & (PAGE_CACHE_SIZE - 1); + unsigned from = pos & (PAGE_SIZE - 1); unsigned to = from + len; int result = 0; ENTRY; @@ -645,13 +664,34 @@ static int ll_write_begin(struct file *file, struct address_space *mapping, lcc = ll_cl_find(file); if (lcc == NULL) { - io = NULL; - GOTO(out, result = -EIO); + vmpage = grab_cache_page_nowait(mapping, index); + result = ll_tiny_write_begin(vmpage); + GOTO(out, result); } env = lcc->lcc_env; io = lcc->lcc_io; + if (file->f_flags & O_DIRECT) { + /* direct IO failed because it couldn't clean up cached pages, + * this causes a problem for mirror write because the cached + * page may belong to another mirror, which will result in + * problem submitting the I/O. */ + if (io->ci_designated_mirror > 0) + GOTO(out, result = -EBUSY); + + /** + * Direct read can fall back to buffered read, but DIO is done + * with lockless i/o, and buffered requires LDLM locking, so + * in this case we must restart without lockless. + */ + if (!io->ci_ignore_lockless) { + io->ci_ignore_lockless = 1; + io->ci_need_restart = 1; + GOTO(out, result = -ENOLCK); + } + } +again: /* To avoid deadlock, try to lock page first. */ vmpage = grab_cache_page_nowait(mapping, index); @@ -667,7 +707,7 @@ static int ll_write_begin(struct file *file, struct address_space *mapping, * one in commit page list, though. */ if (vmpage != NULL && plist->pl_nr > 0) { unlock_page(vmpage); - page_cache_release(vmpage); + put_page(vmpage); vmpage = NULL; } @@ -704,19 +744,29 @@ static int ll_write_begin(struct file *file, struct address_space *mapping, /* TODO: can be optimized at OSC layer to check if it * is a lockless IO. In that case, it's not necessary * to read the data. */ - result = ll_prepare_partial_page(env, io, page); - if (result == 0) - SetPageUptodate(vmpage); + result = ll_prepare_partial_page(env, io, page, file); + if (result) { + /* vmpage should have been unlocked */ + put_page(vmpage); + vmpage = NULL; + + if (result == -EAGAIN) + goto again; + GOTO(out, result); + } } } - if (result < 0) - cl_page_unassume(env, io, page); EXIT; out: if (result < 0) { if (vmpage != NULL) { unlock_page(vmpage); - page_cache_release(vmpage); + put_page(vmpage); + } + /* On tiny_write failure, page and io are always null. */ + if (!IS_ERR_OR_NULL(page)) { + lu_ref_del(&page->cp_reference, "cl_io", io); + cl_page_put(env, page); } if (io) io->ci_result = result; @@ -727,6 +777,47 @@ out: RETURN(result); } +static int ll_tiny_write_end(struct file *file, struct address_space *mapping, + loff_t pos, unsigned int len, unsigned int copied, + struct page *vmpage) +{ + struct cl_page *clpage = (struct cl_page *) vmpage->private; + loff_t kms = pos+copied; + loff_t to = kms & (PAGE_SIZE-1) ? kms & (PAGE_SIZE-1) : PAGE_SIZE; + __u16 refcheck; + struct lu_env *env = cl_env_get(&refcheck); + int rc = 0; + + ENTRY; + + if (IS_ERR(env)) { + rc = PTR_ERR(env); + goto out; + } + + /* This page is dirty in cache, so it should have a cl_page pointer + * set in vmpage->private. + */ + LASSERT(clpage != NULL); + + if (copied == 0) + goto out_env; + + /* Update the underlying size information in the OSC/LOV objects this + * page is part of. + */ + cl_page_touch(env, clpage, to); + +out_env: + cl_env_put(env, &refcheck); + +out: + /* Must return page unlocked. */ + unlock_page(vmpage); + + RETURN(rc); +} + static int ll_write_end(struct file *file, struct address_space *mapping, loff_t pos, unsigned len, unsigned copied, struct page *vmpage, void *fsdata) @@ -736,12 +827,20 @@ static int ll_write_end(struct file *file, struct address_space *mapping, struct cl_io *io; struct vvp_io *vio; struct cl_page *page; - unsigned from = pos & (PAGE_CACHE_SIZE - 1); + unsigned from = pos & (PAGE_SIZE - 1); bool unplug = false; int result = 0; ENTRY; - page_cache_release(vmpage); + put_page(vmpage); + + CDEBUG(D_VFSTRACE, "pos %llu, len %u, copied %u\n", pos, len, copied); + + if (lcc == NULL) { + result = ll_tiny_write_end(file, mapping, pos, len, copied, + vmpage); + GOTO(out, result); + } LASSERT(lcc != NULL); env = lcc->lcc_env; @@ -784,12 +883,14 @@ static int ll_write_end(struct file *file, struct address_space *mapping, /* page list is not contiguous now, commit it now */ unplug = true; } - if (unplug || - file->f_flags & O_SYNC || IS_SYNC(file->f_path.dentry->d_inode)) + if (unplug || io->u.ci_wr.wr_sync) result = vvp_io_write_commit(env, io); if (result < 0) io->ci_result = result; + + +out: RETURN(result >= 0 ? copied : result); } @@ -806,7 +907,6 @@ static int ll_migratepage(struct address_space *mapping, } #endif -#ifndef MS_HAS_NEW_AOPS const struct address_space_operations ll_aops = { .readpage = ll_readpage, .direct_IO = ll_direct_IO, @@ -821,19 +921,3 @@ const struct address_space_operations ll_aops = { .migratepage = ll_migratepage, #endif }; -#else -const struct address_space_operations_ext ll_aops = { - .orig_aops.readpage = ll_readpage, - .orig_aops.direct_IO = ll_direct_IO, - .orig_aops.writepage = ll_writepage, - .orig_aops.writepages = ll_writepages, - .orig_aops.set_page_dirty = __set_page_dirty_nobuffers, - .orig_aops.invalidatepage = ll_invalidatepage, - .orig_aops.releasepage = ll_releasepage, -#ifdef CONFIG_MIGRATION - .orig_aops.migratepage = ll_migratepage, -#endif - .write_begin = ll_write_begin, - .write_end = ll_write_end -}; -#endif