* Copyright (c) 2003, 2010, Oracle and/or its affiliates. All rights reserved.
* Use is subject to license terms.
*
- * Copyright (c) 2011, 2016, Intel Corporation.
+ * Copyright (c) 2011, 2017, Intel Corporation.
*/
/*
* This file is part of Lustre, http://www.lustre.org/
* Lustre Lite I/O page cache routines for the 2.5/2.6 kernel version
*/
+#include <linux/buffer_head.h>
+#include <linux/errno.h>
+#include <linux/fs.h>
#include <linux/kernel.h>
#include <linux/mm.h>
+#include <linux/mpage.h>
+#include <linux/pagemap.h>
#include <linux/string.h>
-#include <linux/stat.h>
-#include <linux/errno.h>
#include <linux/unistd.h>
-#include <asm/uaccess.h>
+#include <linux/writeback.h>
#ifdef HAVE_MIGRATE_H
#include <linux/migrate.h>
#elif defined(HAVE_MIGRATE_MODE_H)
#include <linux/migrate_mode.h>
#endif
-#include <linux/fs.h>
-#include <linux/buffer_head.h>
-#include <linux/mpage.h>
-#include <linux/writeback.h>
-#include <linux/stat.h>
-#include <asm/uaccess.h>
-#include <linux/mm.h>
-#include <linux/pagemap.h>
#define DEBUG_SUBSYSTEM S_LLITE
if (obj == NULL)
return 1;
- /* 1 for caller, 1 for cl_page and 1 for page cache */
- if (page_count(vmpage) > 3)
- return 0;
-
page = cl_vmpage_page(vmpage, obj);
if (page == NULL)
return 1;
#define MAX_DIRECTIO_SIZE 2*1024*1024*1024UL
-ssize_t ll_direct_rw_pages(const struct lu_env *env, struct cl_io *io,
- int rw, struct inode *inode,
- struct ll_dio_pages *pv)
+static ssize_t
+ll_direct_IO_seg(const struct lu_env *env, struct cl_io *io, int rw,
+ struct inode *inode, size_t size, loff_t file_offset,
+ struct page **pages, int page_count)
{
- struct cl_page *clp;
- struct cl_2queue *queue;
- struct cl_object *obj = io->ci_obj;
+ struct cl_page *clp;
+ struct cl_2queue *queue;
+ struct cl_object *obj = io->ci_obj;
int i;
ssize_t rc = 0;
- loff_t file_offset = pv->ldp_start_offset;
- size_t size = pv->ldp_size;
- int page_count = pv->ldp_nr;
- struct page **pages = pv->ldp_pages;
- size_t page_size = cl_page_size(obj);
+ size_t page_size = cl_page_size(obj);
+ size_t orig_size = size;
bool do_io;
- int io_pages = 0;
- ENTRY;
+ int io_pages = 0;
- queue = &io->ci_queue;
- cl_2queue_init(queue);
- for (i = 0; i < page_count; i++) {
- if (pv->ldp_offsets)
- file_offset = pv->ldp_offsets[i];
-
- LASSERT(!(file_offset & (page_size - 1)));
- clp = cl_page_find(env, obj, cl_index(obj, file_offset),
- pv->ldp_pages[i], CPT_TRANSIENT);
- if (IS_ERR(clp)) {
- rc = PTR_ERR(clp);
- break;
- }
+ ENTRY;
+ queue = &io->ci_queue;
+ cl_2queue_init(queue);
+ for (i = 0; i < page_count; i++) {
+ LASSERT(!(file_offset & (page_size - 1)));
+ clp = cl_page_find(env, obj, cl_index(obj, file_offset),
+ pages[i], CPT_TRANSIENT);
+ if (IS_ERR(clp)) {
+ rc = PTR_ERR(clp);
+ break;
+ }
- rc = cl_page_own(env, io, clp);
- if (rc) {
- LASSERT(clp->cp_state == CPS_FREEING);
- cl_page_put(env, clp);
- break;
- }
+ rc = cl_page_own(env, io, clp);
+ if (rc) {
+ LASSERT(clp->cp_state == CPS_FREEING);
+ cl_page_put(env, clp);
+ break;
+ }
- do_io = true;
+ do_io = true;
- /* check the page type: if the page is a host page, then do
- * write directly */
- if (clp->cp_type == CPT_CACHEABLE) {
+ /* check the page type: if the page is a host page, then do
+ * write directly
+ */
+ if (clp->cp_type == CPT_CACHEABLE) {
struct page *vmpage = cl_page_vmpage(clp);
struct page *src_page;
struct page *dst_page;
- void *src;
- void *dst;
-
- src_page = (rw == WRITE) ? pages[i] : vmpage;
- dst_page = (rw == WRITE) ? vmpage : pages[i];
-
- src = ll_kmap_atomic(src_page, KM_USER0);
- dst = ll_kmap_atomic(dst_page, KM_USER1);
- memcpy(dst, src, min(page_size, size));
- ll_kunmap_atomic(dst, KM_USER1);
- ll_kunmap_atomic(src, KM_USER0);
-
- /* make sure page will be added to the transfer by
- * cl_io_submit()->...->vvp_page_prep_write(). */
- if (rw == WRITE)
- set_page_dirty(vmpage);
-
- if (rw == READ) {
- /* do not issue the page for read, since it
- * may reread a ra page which has NOT uptodate
- * bit set. */
- cl_page_disown(env, io, clp);
- do_io = false;
- }
- }
+ void *src;
+ void *dst;
+
+ src_page = (rw == WRITE) ? pages[i] : vmpage;
+ dst_page = (rw == WRITE) ? vmpage : pages[i];
+
+ src = ll_kmap_atomic(src_page, KM_USER0);
+ dst = ll_kmap_atomic(dst_page, KM_USER1);
+ memcpy(dst, src, min(page_size, size));
+ ll_kunmap_atomic(dst, KM_USER1);
+ ll_kunmap_atomic(src, KM_USER0);
+
+ /* make sure page will be added to the transfer by
+ * cl_io_submit()->...->vvp_page_prep_write().
+ */
+ if (rw == WRITE)
+ set_page_dirty(vmpage);
+
+ if (rw == READ) {
+ /* do not issue the page for read, since it
+ * may reread a ra page which has NOT uptodate
+ * bit set.
+ */
+ cl_page_disown(env, io, clp);
+ do_io = false;
+ }
+ }
- if (likely(do_io)) {
- cl_2queue_add(queue, clp);
+ if (likely(do_io)) {
+ cl_2queue_add(queue, clp);
- /*
- * Set page clip to tell transfer formation engine
- * that page has to be sent even if it is beyond KMS.
- */
- cl_page_clip(env, clp, 0, min(size, page_size));
+ /*
+ * Set page clip to tell transfer formation engine
+ * that page has to be sent even if it is beyond KMS.
+ */
+ cl_page_clip(env, clp, 0, min(size, page_size));
- ++io_pages;
- }
+ ++io_pages;
+ }
- /* drop the reference count for cl_page_find */
- cl_page_put(env, clp);
- size -= page_size;
- file_offset += page_size;
- }
+ /* drop the reference count for cl_page_find */
+ cl_page_put(env, clp);
+ size -= page_size;
+ file_offset += page_size;
+ }
- if (rc == 0 && io_pages) {
- rc = cl_io_submit_sync(env, io,
- rw == READ ? CRT_READ : CRT_WRITE,
+ if (rc == 0 && io_pages) {
+ rc = cl_io_submit_sync(env, io,
+ rw == READ ? CRT_READ : CRT_WRITE,
queue, 0);
- }
- if (rc == 0)
- rc = pv->ldp_size;
-
- cl_2queue_discard(env, io, queue);
- cl_2queue_disown(env, io, queue);
- cl_2queue_fini(env, queue);
- RETURN(rc);
-}
-EXPORT_SYMBOL(ll_direct_rw_pages);
+ }
+ if (rc == 0)
+ rc = orig_size;
-static ssize_t
-ll_direct_IO_seg(const struct lu_env *env, struct cl_io *io, int rw,
- struct inode *inode, size_t size, loff_t file_offset,
- struct page **pages, int page_count)
-{
- struct ll_dio_pages pvec = { .ldp_pages = pages,
- .ldp_nr = page_count,
- .ldp_size = size,
- .ldp_offsets = NULL,
- .ldp_start_offset = file_offset
- };
-
- return ll_direct_rw_pages(env, io, rw, inode, &pvec);
+ cl_2queue_discard(env, io, queue);
+ cl_2queue_disown(env, io, queue);
+ cl_2queue_fini(env, queue);
+ RETURN(rc);
}
/* ll_free_user_pages - tear down page struct array
/**
* Prepare partially written-to page for a write.
+ * @pg is owned when passed in and disowned when it returns non-zero result to
+ * the caller.
*/
static int ll_prepare_partial_page(const struct lu_env *env, struct cl_io *io,
- struct cl_page *pg)
+ struct cl_page *pg, struct file *file)
{
struct cl_attr *attr = vvp_env_thread_attr(env);
struct cl_object *obj = io->ci_obj;
struct vvp_page *vpg = cl_object_page_slice(obj, pg);
loff_t offset = cl_offset(obj, vvp_index(vpg));
int result;
+ ENTRY;
cl_object_attr_lock(obj);
result = cl_object_attr_get(env, obj, attr);
cl_object_attr_unlock(obj);
- if (result == 0) {
- /*
- * If are writing to a new page, no need to read old data.
- * The extent locking will have updated the KMS, and for our
- * purposes here we can treat it like i_size.
- */
- if (attr->cat_kms <= offset) {
- char *kaddr = ll_kmap_atomic(vpg->vpg_page, KM_USER0);
+ if (result) {
+ cl_page_disown(env, io, pg);
+ GOTO(out, result);
+ }
- memset(kaddr, 0, cl_page_size(obj));
- ll_kunmap_atomic(kaddr, KM_USER0);
- } else if (vpg->vpg_defer_uptodate)
- vpg->vpg_ra_used = 1;
- else
- result = ll_page_sync_io(env, io, pg, CRT_READ);
+ /*
+ * If are writing to a new page, no need to read old data.
+ * The extent locking will have updated the KMS, and for our
+ * purposes here we can treat it like i_size.
+ */
+ if (attr->cat_kms <= offset) {
+ char *kaddr = ll_kmap_atomic(vpg->vpg_page, KM_USER0);
+
+ memset(kaddr, 0, cl_page_size(obj));
+ ll_kunmap_atomic(kaddr, KM_USER0);
+ GOTO(out, result = 0);
+ }
+
+ if (vpg->vpg_defer_uptodate) {
+ vpg->vpg_ra_used = 1;
+ GOTO(out, result = 0);
+ }
+
+ result = ll_io_read_page(env, io, pg, file);
+ if (result)
+ GOTO(out, result);
+
+ /* ll_io_read_page() disowns the page */
+ result = cl_page_own(env, io, pg);
+ if (!result) {
+ if (!PageUptodate(cl_page_vmpage(pg))) {
+ cl_page_disown(env, io, pg);
+ result = -EIO;
+ }
+ } else if (result == -ENOENT) {
+ /* page was truncated */
+ result = -EAGAIN;
}
+ EXIT;
+
+out:
return result;
}
+static int ll_tiny_write_begin(struct page *vmpage)
+{
+ /* Page must be present, up to date, dirty, and not in writeback. */
+ if (!vmpage || !PageUptodate(vmpage) || !PageDirty(vmpage) ||
+ PageWriteback(vmpage))
+ return -ENODATA;
+
+ return 0;
+}
+
static int ll_write_begin(struct file *file, struct address_space *mapping,
loff_t pos, unsigned len, unsigned flags,
struct page **pagep, void **fsdata)
{
- struct ll_cl_context *lcc;
+ struct ll_cl_context *lcc = NULL;
const struct lu_env *env = NULL;
- struct cl_io *io;
+ struct cl_io *io = NULL;
struct cl_page *page = NULL;
struct cl_object *clob = ll_i2info(mapping->host)->lli_clob;
int result = 0;
ENTRY;
- CDEBUG(D_PAGE, "Writing %lu of %d to %d bytes\n", index, from, len);
+ CDEBUG(D_VFSTRACE, "Writing %lu of %d to %d bytes\n", index, from, len);
lcc = ll_cl_find(file);
if (lcc == NULL) {
- io = NULL;
- GOTO(out, result = -EIO);
+ vmpage = grab_cache_page_nowait(mapping, index);
+ result = ll_tiny_write_begin(vmpage);
+ GOTO(out, result);
}
env = lcc->lcc_env;
io = lcc->lcc_io;
+ if (file->f_flags & O_DIRECT) {
+ /* direct IO failed because it couldn't clean up cached pages,
+ * this causes a problem for mirror write because the cached
+ * page may belong to another mirror, which will result in
+ * problem submitting the I/O. */
+ if (io->ci_designated_mirror > 0)
+ GOTO(out, result = -EBUSY);
+
+ /**
+ * Direct read can fall back to buffered read, but DIO is done
+ * with lockless i/o, and buffered requires LDLM locking, so
+ * in this case we must restart without lockless.
+ */
+ if (!io->ci_ignore_lockless) {
+ io->ci_ignore_lockless = 1;
+ io->ci_need_restart = 1;
+ GOTO(out, result = -ENOLCK);
+ }
+ }
+again:
/* To avoid deadlock, try to lock page first. */
vmpage = grab_cache_page_nowait(mapping, index);
/* TODO: can be optimized at OSC layer to check if it
* is a lockless IO. In that case, it's not necessary
* to read the data. */
- result = ll_prepare_partial_page(env, io, page);
- if (result == 0)
- SetPageUptodate(vmpage);
+ result = ll_prepare_partial_page(env, io, page, file);
+ if (result) {
+ /* vmpage should have been unlocked */
+ put_page(vmpage);
+ vmpage = NULL;
+
+ if (result == -EAGAIN)
+ goto again;
+ GOTO(out, result);
+ }
}
}
- if (result < 0)
- cl_page_unassume(env, io, page);
EXIT;
out:
if (result < 0) {
unlock_page(vmpage);
put_page(vmpage);
}
+ /* On tiny_write failure, page and io are always null. */
if (!IS_ERR_OR_NULL(page)) {
lu_ref_del(&page->cp_reference, "cl_io", io);
cl_page_put(env, page);
RETURN(result);
}
+static int ll_tiny_write_end(struct file *file, struct address_space *mapping,
+ loff_t pos, unsigned int len, unsigned int copied,
+ struct page *vmpage)
+{
+ struct cl_page *clpage = (struct cl_page *) vmpage->private;
+ loff_t kms = pos+copied;
+ loff_t to = kms & (PAGE_SIZE-1) ? kms & (PAGE_SIZE-1) : PAGE_SIZE;
+ __u16 refcheck;
+ struct lu_env *env = cl_env_get(&refcheck);
+ int rc = 0;
+
+ ENTRY;
+
+ if (IS_ERR(env)) {
+ rc = PTR_ERR(env);
+ goto out;
+ }
+
+ /* This page is dirty in cache, so it should have a cl_page pointer
+ * set in vmpage->private.
+ */
+ LASSERT(clpage != NULL);
+
+ if (copied == 0)
+ goto out_env;
+
+ /* Update the underlying size information in the OSC/LOV objects this
+ * page is part of.
+ */
+ cl_page_touch(env, clpage, to);
+
+out_env:
+ cl_env_put(env, &refcheck);
+
+out:
+ /* Must return page unlocked. */
+ unlock_page(vmpage);
+
+ RETURN(rc);
+}
+
static int ll_write_end(struct file *file, struct address_space *mapping,
loff_t pos, unsigned len, unsigned copied,
struct page *vmpage, void *fsdata)
put_page(vmpage);
+ CDEBUG(D_VFSTRACE, "pos %llu, len %u, copied %u\n", pos, len, copied);
+
+ if (lcc == NULL) {
+ result = ll_tiny_write_end(file, mapping, pos, len, copied,
+ vmpage);
+ GOTO(out, result);
+ }
+
LASSERT(lcc != NULL);
env = lcc->lcc_env;
page = lcc->lcc_page;
if (plist->pl_nr >= PTLRPC_MAX_BRW_PAGES)
unplug = true;
- CL_PAGE_DEBUG(D_PAGE, env, page,
+ CL_PAGE_DEBUG(D_VFSTRACE, env, page,
"queued page: %d.\n", plist->pl_nr);
} else {
cl_page_disown(env, io, page);
/* page list is not contiguous now, commit it now */
unplug = true;
}
- if (unplug ||
- file->f_flags & O_SYNC || IS_SYNC(file_inode(file)))
+ if (unplug || io->u.ci_wr.wr_sync)
result = vvp_io_write_commit(env, io);
if (result < 0)
io->ci_result = result;
+
+
+out:
RETURN(result >= 0 ? copied : result);
}
git://git.whamcloud.com / fs / lustre-release.git / blobdiff