#include <obd.h>
+#include <linux/pagevec.h>
+#include <linux/memcontrol.h>
#include "llite_internal.h"
#include "vvp_internal.h"
+#include <libcfs/linux/linux-misc.h>
static struct vvp_io *cl2vvp_io(const struct lu_env *env,
const struct cl_io_slice *slice)
struct inode *inode = vvp_object_inode(ios->cis_obj);
LASSERT(inode == file_inode(vio->vui_fd->fd_file));
- vio->u.fault.ft_mtime = inode->i_mtime.tv_sec;
return 0;
}
union ldlm_policy_data policy;
struct iovec iov;
struct iov_iter i;
+ unsigned long addr;
+ ssize_t count;
int result = 0;
ENTRY;
if (mm == NULL)
RETURN(0);
- iov_for_each(iov, i, *(vio->vui_iter)) {
- unsigned long addr = (unsigned long)iov.iov_base;
- size_t count = iov.iov_len;
+ if (!iter_is_iovec(vio->vui_iter) && !iov_iter_is_kvec(vio->vui_iter))
+ RETURN(0);
+
+ for (i = *vio->vui_iter;
+ iov_iter_count(&i);
+ iov_iter_advance(&i, iov.iov_len)) {
+ iov = iov_iter_iovec(&i);
+ addr = (unsigned long)iov.iov_base;
+ count = iov.iov_len;
if (count == 0)
continue;
ast_flags |= CEF_NONBLOCK;
if (io->ci_lock_no_expand)
ast_flags |= CEF_LOCK_NO_EXPAND;
+ if (vio->vui_fd) {
+ /* Group lock held means no lockless any more */
+ if (vio->vui_fd->fd_flags & LL_FILE_GROUP_LOCKED)
+ io->ci_ignore_lockless = 1;
+
+ if (ll_file_nolock(vio->vui_fd->fd_file) ||
+ (vio->vui_fd->fd_flags & LL_FILE_LOCKLESS_IO &&
+ !io->ci_ignore_lockless))
+ ast_flags |= CEF_NEVER;
+ }
result = vvp_mmap_locks(env, vio, io);
if (result == 0)
struct ll_inode_info *lli = ll_i2info(inode);
if (cl_io_is_trunc(io)) {
- down_write(&lli->lli_trunc_sem);
+ trunc_sem_down_write(&lli->lli_trunc_sem);
inode_lock(inode);
inode_dio_wait(inode);
} else {
vvp_do_vmtruncate(inode, io->u.ci_setattr.sa_attr.lvb_size);
inode_dio_write_done(inode);
inode_unlock(inode);
- up_write(&lli->lli_trunc_sem);
+ trunc_sem_up_write(&lli->lli_trunc_sem);
} else {
inode_unlock(inode);
}
static int vvp_io_read_start(const struct lu_env *env,
const struct cl_io_slice *ios)
{
- struct vvp_io *vio = cl2vvp_io(env, ios);
- struct cl_io *io = ios->cis_io;
- struct cl_object *obj = io->ci_obj;
- struct inode *inode = vvp_object_inode(obj);
- struct ll_inode_info *lli = ll_i2info(inode);
- struct file *file = vio->vui_fd->fd_file;
- loff_t pos = io->u.ci_rd.rd.crw_pos;
- long cnt = io->u.ci_rd.rd.crw_count;
- long tot = vio->vui_tot_count;
- int exceed = 0;
- int result;
+ struct vvp_io *vio = cl2vvp_io(env, ios);
+ struct cl_io *io = ios->cis_io;
+ struct cl_object *obj = io->ci_obj;
+ struct inode *inode = vvp_object_inode(obj);
+ struct ll_inode_info *lli = ll_i2info(inode);
+ struct file *file = vio->vui_fd->fd_file;
+ loff_t pos = io->u.ci_rd.rd.crw_pos;
+ size_t cnt = io->u.ci_rd.rd.crw_count;
+ size_t tot = vio->vui_tot_count;
+ int exceed = 0;
+ int result;
ENTRY;
CLOBINVRNT(env, obj, vvp_object_invariant(obj));
pos, pos + cnt);
if (vio->vui_io_subtype == IO_NORMAL)
- down_read(&lli->lli_trunc_sem);
+ trunc_sem_down_read(&lli->lli_trunc_sem);
+
+ if (io->ci_async_readahead) {
+ file_accessed(file);
+ RETURN(0);
+ }
if (!can_populate_pages(env, io, inode))
RETURN(0);
/* initialize read-ahead window once per syscall */
if (!vio->vui_ra_valid) {
vio->vui_ra_valid = true;
- vio->vui_ra_start = cl_index(obj, pos);
- vio->vui_ra_count = cl_index(obj, tot + PAGE_SIZE - 1);
- ll_ras_enter(file);
+ vio->vui_ra_start_idx = cl_index(obj, pos);
+ vio->vui_ra_pages = cl_index(obj, tot + PAGE_SIZE - 1);
+ /* If both start and end are unaligned, we read one more page
+ * than the index math suggests. */
+ if ((pos & ~PAGE_MASK) != 0 && ((pos + tot) & ~PAGE_MASK) != 0)
+ vio->vui_ra_pages++;
+
+ CDEBUG(D_READA, "tot %zu, ra_start %lu, ra_count %lu\n",
+ tot, vio->vui_ra_start_idx, vio->vui_ra_pages);
}
/* BUG: 5972 */
if (result < cnt)
io->ci_continue = 0;
io->ci_nob += result;
- ll_rw_stats_tally(ll_i2sbi(inode), current->pid, vio->vui_fd,
- pos, result, READ);
result = 0;
}
RETURN(bytes > 0 ? bytes : rc);
}
+/*
+ * Kernels 4.2 - 4.5 pass memcg argument to account_page_dirtied()
+ * Kernel v5.2-5678-gac1c3e4 no longer exports account_page_dirtied
+ */
+static inline void ll_account_page_dirtied(struct page *page,
+ struct address_space *mapping)
+{
+#ifdef HAVE_ACCOUNT_PAGE_DIRTIED_3ARGS
+ struct mem_cgroup *memcg = mem_cgroup_begin_page_stat(page);
+
+ account_page_dirtied(page, mapping, memcg);
+ mem_cgroup_end_page_stat(memcg);
+#elif defined HAVE_ACCOUNT_PAGE_DIRTIED
+ account_page_dirtied(page, mapping, memcg);
+#else
+ typedef unsigned int (dirtied_t)(struct page *pg,
+ struct address_space *as);
+ const char *symbol = "account_page_dirtied";
+ static dirtied_t *dirtied = NULL;
+
+ if (!dirtied)
+ dirtied = (dirtied_t *)symbol_get(symbol);
+
+ if (dirtied)
+ dirtied(page, mapping);
+#endif
+}
+
+/*
+ * From kernel v4.19-rc5-248-g9b89a0355144 use XArrary
+ * Prior kernels use radix_tree for tags
+ */
+static inline void ll_page_tag_dirty(struct page *page,
+ struct address_space *mapping)
+{
+#ifdef HAVE___XA_SET_MARK
+ __xa_set_mark(&mapping->i_pages, page_index(page), PAGECACHE_TAG_DIRTY);
+#else
+ radix_tree_tag_set(&mapping->page_tree, page_index(page),
+ PAGECACHE_TAG_DIRTY);
+#endif
+}
+
+/* Taken from kernel set_page_dirty, __set_page_dirty_nobuffers
+ * Last change to this area: b93b016313b3ba8003c3b8bb71f569af91f19fc7
+ *
+ * Current with Linus tip of tree (7/13/2019):
+ * v5.2-rc4-224-ge01e060fe0
+ *
+ * Backwards compat for 3.x, 5.x kernels relating to memcg handling
+ * & rename of radix tree to xarray.
+ */
+void vvp_set_pagevec_dirty(struct pagevec *pvec)
+{
+ struct page *page = pvec->pages[0];
+ struct address_space *mapping = page->mapping;
+ unsigned long flags;
+ int count = pagevec_count(pvec);
+ int dirtied = 0;
+ int i = 0;
+
+ ENTRY;
+
+ /* From set_page_dirty */
+ for (i = 0; i < count; i++)
+ ClearPageReclaim(pvec->pages[i]);
+
+ LASSERTF(page->mapping,
+ "mapping must be set. page %p, page->private (cl_page) %p",
+ page, (void *) page->private);
+
+ /* Rest of code derived from __set_page_dirty_nobuffers */
+ xa_lock_irqsave(&mapping->i_pages, flags);
+
+ /* Notes on differences with __set_page_dirty_nobuffers:
+ * 1. We don't need to call page_mapping because we know this is a page
+ * cache page.
+ * 2. We have the pages locked, so there is no need for the careful
+ * mapping/mapping2 dance.
+ * 3. No mapping is impossible. (Race w/truncate mentioned in
+ * dirty_nobuffers should be impossible because we hold the page lock.)
+ * 4. All mappings are the same because i/o is only to one file.
+ * 5. We invert the lock order on lock_page_memcg(page) and the mapping
+ * xa_lock, but this is the only function that should use that pair of
+ * locks and it can't race because Lustre locks pages throughout i/o.
+ */
+ for (i = 0; i < count; i++) {
+ page = pvec->pages[i];
+ lock_page_memcg(page);
+ if (TestSetPageDirty(page)) {
+ unlock_page_memcg(page);
+ continue;
+ }
+ LASSERTF(page->mapping == mapping,
+ "all pages must have the same mapping. page %p, mapping %p, first mapping %p\n",
+ page, page->mapping, mapping);
+ WARN_ON_ONCE(!PagePrivate(page) && !PageUptodate(page));
+ ll_account_page_dirtied(page, mapping);
+ ll_page_tag_dirty(page, mapping);
+ dirtied++;
+ unlock_page_memcg(page);
+ }
+ xa_unlock_irqrestore(&mapping->i_pages, flags);
+
+ CDEBUG(D_VFSTRACE, "mapping %p, count %d, dirtied %d\n", mapping,
+ count, dirtied);
+
+ if (mapping->host && dirtied) {
+ /* !PageAnon && !swapper_space */
+ __mark_inode_dirty(mapping->host, I_DIRTY_PAGES);
+ }
+
+ EXIT;
+}
+
static void write_commit_callback(const struct lu_env *env, struct cl_io *io,
- struct cl_page *page)
+ struct pagevec *pvec)
{
- struct page *vmpage = page->cp_vmpage;
+ int count = 0;
+ int i = 0;
+
+ ENTRY;
- SetPageUptodate(vmpage);
- set_page_dirty(vmpage);
+ count = pagevec_count(pvec);
+ LASSERT(count > 0);
- cl_page_disown(env, io, page);
+ for (i = 0; i < count; i++) {
+ struct page *vmpage = pvec->pages[i];
+ SetPageUptodate(vmpage);
+ }
+
+ vvp_set_pagevec_dirty(pvec);
+
+ for (i = 0; i < count; i++) {
+ struct page *vmpage = pvec->pages[i];
+ struct cl_page *page = (struct cl_page *) vmpage->private;
+ cl_page_disown(env, io, page);
+ lu_ref_del(&page->cp_reference, "cl_io", cl_io_top(io));
+ cl_page_put(env, page);
+ }
- /* held in ll_cl_init() */
- lu_ref_del(&page->cp_reference, "cl_io", cl_io_top(io));
- cl_page_put(env, page);
+ EXIT;
}
/* make sure the page list is contiguous */
loff_t pos = io->u.ci_wr.wr.crw_pos;
size_t cnt = io->u.ci_wr.wr.crw_count;
bool lock_inode = !IS_NOSEC(inode);
+ size_t nob = io->ci_nob;
+ size_t written = 0;
ENTRY;
if (vio->vui_io_subtype == IO_NORMAL)
- down_read(&lli->lli_trunc_sem);
+ trunc_sem_down_read(&lli->lli_trunc_sem);
if (!can_populate_pages(env, io, inode))
RETURN(0);
if (pos + cnt > ll_file_maxbytes(inode)) {
CDEBUG(D_INODE,
"%s: file %s ("DFID") offset %llu > maxbytes %llu\n",
- ll_get_fsname(inode->i_sb, NULL, 0),
+ ll_i2sbi(inode)->ll_fsname,
file_dentry(file)->d_name.name,
PFID(ll_inode2fid(inode)), pos + cnt,
ll_file_maxbytes(inode));
if (unlikely(lock_inode))
inode_unlock(inode);
- if (result > 0 || result == -EIOCBQUEUED)
+ written = result;
+ if (result > 0)
#ifdef HAVE_GENERIC_WRITE_SYNC_2ARGS
result = generic_write_sync(vio->vui_iocb, result);
#else
io->ci_nob += result;
}
}
- if (result > 0) {
+ if (vio->vui_iocb->ki_pos != (pos + io->ci_nob - nob)) {
+ CDEBUG(D_VFSTRACE, "%s: write position mismatch: "
+ "ki_pos %lld vs. pos %lld, written %ld, commit %ld "
+ "rc %ld\n",
+ file_dentry(file)->d_name.name,
+ vio->vui_iocb->ki_pos, pos + io->ci_nob - nob,
+ written, io->ci_nob - nob, result);
+ /* rewind ki_pos to where it has successfully committed */
+ vio->vui_iocb->ki_pos = pos + io->ci_nob - nob;
+ }
+ if (result > 0 || result == -EIOCBQUEUED) {
ll_file_set_flag(ll_i2info(inode), LLIF_DATA_MODIFIED);
if (result < cnt)
io->ci_continue = 0;
- ll_rw_stats_tally(ll_i2sbi(inode), current->pid,
- vio->vui_fd, pos, result, WRITE);
- result = 0;
+ if (result > 0)
+ result = 0;
}
RETURN(result);
struct ll_inode_info *lli = ll_i2info(inode);
if (vio->vui_io_subtype == IO_NORMAL)
- up_read(&lli->lli_trunc_sem);
+ trunc_sem_up_read(&lli->lli_trunc_sem);
}
static int vvp_io_kernel_fault(struct vvp_fault_io *cfio)
}
static void mkwrite_commit_callback(const struct lu_env *env, struct cl_io *io,
- struct cl_page *page)
+ struct pagevec *pvec)
{
- set_page_dirty(page->cp_vmpage);
+ vvp_set_pagevec_dirty(pvec);
}
static int vvp_io_fault_start(const struct lu_env *env,
pgoff_t last_index;
ENTRY;
- down_read(&lli->lli_trunc_sem);
+ trunc_sem_down_read_nowait(&lli->lli_trunc_sem);
/* offset of the last byte on the page */
offset = cl_offset(obj, fio->ft_index + 1) - 1;
LASSERT(PageLocked(vmpage));
if (OBD_FAIL_CHECK(OBD_FAIL_LLITE_FAULT_TRUNC_RACE))
- ll_invalidate_page(vmpage);
+ generic_error_remove_page(vmpage->mapping, vmpage);
size = i_size_read(inode);
/* Though we have already held a cl_lock upon this page, but
CLOBINVRNT(env, ios->cis_io->ci_obj,
vvp_object_invariant(ios->cis_io->ci_obj));
- up_read(&lli->lli_trunc_sem);
+ trunc_sem_up_read(&lli->lli_trunc_sem);
}
static int vvp_io_fsync_start(const struct lu_env *env,
struct vvp_io *vio = cl2vvp_io(env, ios);
if (unlikely(vio->vui_fd->fd_flags & LL_FILE_GROUP_LOCKED)) {
- ra->cra_end = CL_PAGE_EOF;
+ ra->cra_end_idx = CL_PAGE_EOF;
result = +1; /* no need to call down */
}
}
result = 0;
if (result < 0)
CERROR("%s: refresh file layout " DFID " error %d.\n",
- ll_get_fsname(inode->i_sb, NULL, 0),
- PFID(lu_object_fid(&obj->co_lu)), result);
+ ll_i2sbi(inode)->ll_fsname,
+ PFID(lu_object_fid(&obj->co_lu)), result);
}
io->ci_result = result < 0 ? result : 0;
git://git.whamcloud.com / fs / lustre-release.git / blobdiff