X-Git-Url: https://git.whamcloud.com/?p=fs%2Flustre-release.git;a=blobdiff_plain;f=lustre%2Fllite%2Fvvp_io.c;h=55e21fa4e412d0b7c7b83ca560cd107d3a7871d1;hp=a6c355a7bd36a49cca0cfab5f1c414fd5badf0a7;hb=985de582849dfc25ccbf2ef6ea923f984ad7fd89;hpb=8bc4b26453fb9d3296475ab3b9c9f3a9bf905afc diff --git a/lustre/llite/vvp_io.c b/lustre/llite/vvp_io.c index a6c355a..55e21fa 100644 --- a/lustre/llite/vvp_io.c +++ b/lustre/llite/vvp_io.c @@ -23,7 +23,7 @@ * Copyright (c) 2008, 2010, Oracle and/or its affiliates. All rights reserved. * Use is subject to license terms. * - * Copyright (c) 2011, 2016, Intel Corporation. + * Copyright (c) 2011, 2017, Intel Corporation. */ /* * This file is part of Lustre, http://www.lustre.org/ @@ -37,10 +37,13 @@ #define DEBUG_SUBSYSTEM S_LLITE - #include +#include +#include + #include "llite_internal.h" #include "vvp_internal.h" +#include static struct vvp_io *cl2vvp_io(const struct lu_env *env, const struct cl_io_slice *slice) @@ -287,7 +290,6 @@ static int vvp_io_fault_iter_init(const struct lu_env *env, struct inode *inode = vvp_object_inode(ios->cis_obj); LASSERT(inode == file_inode(vio->vui_fd->fd_file)); - vio->u.fault.ft_mtime = inode->i_mtime.tv_sec; return 0; } @@ -298,12 +300,14 @@ static void vvp_io_fini(const struct lu_env *env, const struct cl_io_slice *ios) struct cl_object *obj = io->ci_obj; struct vvp_io *vio = cl2vvp_io(env, ios); struct inode *inode = vvp_object_inode(obj); + __u32 gen = 0; int rc; + ENTRY; CLOBINVRNT(env, obj, vvp_object_invariant(obj)); CDEBUG(D_VFSTRACE, DFID" ignore/verify layout %d/%d, layout version %d " - "need write layout %d, restore needed %d\n", + "need write layout %d, restore needed %d\n", PFID(lu_object_fid(&obj->co_lu)), io->ci_ignore_layout, io->ci_verify_layout, vio->vui_layout_gen, io->ci_need_write_intent, @@ -320,18 +324,40 @@ static void vvp_io_fini(const struct lu_env *env, const struct cl_io_slice *ios) * block on layout lock held by the MDT * as MDT will not send new layout in lvb (see LU-3124) * we have to explicitly fetch it, all this will be done - * by ll_layout_refresh() + * by ll_layout_refresh(). + * Even if ll_layout_restore() returns zero, it doesn't mean + * that restore has been successful. Therefore it sets + * ci_verify_layout so that it will check layout at the end + * of this function. */ - if (rc == 0) { - io->ci_restore_needed = 0; - io->ci_need_restart = 1; - io->ci_verify_layout = 1; - } else { + if (rc) { io->ci_restore_needed = 1; io->ci_need_restart = 0; io->ci_verify_layout = 0; io->ci_result = rc; + GOTO(out, rc); } + + io->ci_restore_needed = 0; + + /* Even if ll_layout_restore() returns zero, it doesn't mean + * that restore has been successful. Therefore it should verify + * if there was layout change and restart I/O correspondingly. + */ + ll_layout_refresh(inode, &gen); + io->ci_need_restart = vio->vui_layout_gen != gen; + if (io->ci_need_restart) { + CDEBUG(D_VFSTRACE, + DFID" layout changed from %d to %d.\n", + PFID(lu_object_fid(&obj->co_lu)), + vio->vui_layout_gen, gen); + /* today successful restore is the only possible + * case */ + /* restore was done, clear restoring state */ + ll_file_clear_flag(ll_i2info(vvp_object_inode(obj)), + LLIF_FILE_RESTORING); + } + GOTO(out, 0); } /** @@ -339,40 +365,29 @@ static void vvp_io_fini(const struct lu_env *env, const struct cl_io_slice *ios) * RPC. */ if (io->ci_need_write_intent) { - loff_t start = 0; - loff_t end = OBD_OBJECT_EOF; + enum layout_intent_opc opc = LAYOUT_INTENT_WRITE; io->ci_need_write_intent = 0; LASSERT(io->ci_type == CIT_WRITE || cl_io_is_trunc(io) || cl_io_is_mkwrite(io)); - if (io->ci_type == CIT_WRITE) { - if (!cl_io_is_append(io)) { - start = io->u.ci_rw.rw_range.cir_pos; - end = start + io->u.ci_rw.rw_range.cir_count; - } - } else if (cl_io_is_trunc(io)) { - end = io->u.ci_setattr.sa_attr.lvb_size; - } else { /* mkwrite */ - pgoff_t index = io->u.ci_fault.ft_index; + CDEBUG(D_VFSTRACE, DFID" write layout, type %u "DEXT"\n", + PFID(lu_object_fid(&obj->co_lu)), io->ci_type, + PEXT(&io->ci_write_intent)); - start = cl_offset(io->ci_obj, index); - end = cl_offset(io->ci_obj, index + 1); - } + if (cl_io_is_trunc(io)) + opc = LAYOUT_INTENT_TRUNC; - CDEBUG(D_VFSTRACE, DFID" write layout, type %u [%llu, %llu)\n", - PFID(lu_object_fid(&obj->co_lu)), io->ci_type, - start, end); - rc = ll_layout_write_intent(inode, start, end); + rc = ll_layout_write_intent(inode, opc, &io->ci_write_intent); io->ci_result = rc; if (!rc) io->ci_need_restart = 1; + GOTO(out, rc); } - if (!io->ci_ignore_layout && io->ci_verify_layout) { - __u32 gen = 0; - + if (!io->ci_need_restart && + !io->ci_ignore_layout && io->ci_verify_layout) { /* check layout version */ ll_layout_refresh(inode, &gen); io->ci_need_restart = vio->vui_layout_gen != gen; @@ -381,13 +396,11 @@ static void vvp_io_fini(const struct lu_env *env, const struct cl_io_slice *ios) DFID" layout changed from %d to %d.\n", PFID(lu_object_fid(&obj->co_lu)), vio->vui_layout_gen, gen); - /* today successful restore is the only possible - * case */ - /* restore was done, clear restoring state */ - ll_file_clear_flag(ll_i2info(vvp_object_inode(obj)), - LLIF_FILE_RESTORING); } + GOTO(out, 0); } +out: + EXIT; } static void vvp_io_fault_fini(const struct lu_env *env, @@ -418,7 +431,8 @@ static enum cl_lock_mode vvp_mode_from_vma(struct vm_area_struct *vma) return CLM_READ; } -static int vvp_mmap_locks(const struct lu_env *env, struct cl_io *io) +static int vvp_mmap_locks(const struct lu_env *env, + struct vvp_io *vio, struct cl_io *io) { struct vvp_thread_info *vti = vvp_env_info(env); struct mm_struct *mm = current->mm; @@ -427,6 +441,8 @@ static int vvp_mmap_locks(const struct lu_env *env, struct cl_io *io) union ldlm_policy_data policy; struct iovec iov; struct iov_iter i; + unsigned long addr; + ssize_t count; int result = 0; ENTRY; @@ -435,13 +451,23 @@ static int vvp_mmap_locks(const struct lu_env *env, struct cl_io *io) if (!cl_is_normalio(env, io)) RETURN(0); + /* nfs or loop back device write */ + if (vio->vui_iter == NULL) + RETURN(0); + /* No MM (e.g. NFS)? No vmas too. */ if (mm == NULL) RETURN(0); - iov_for_each(iov, i, io->u.ci_rw.rw_iter) { - unsigned long addr = (unsigned long)iov.iov_base; - size_t count = iov.iov_len; + if (!iter_is_iovec(vio->vui_iter) && !iov_iter_is_kvec(vio->vui_iter)) + RETURN(0); + + for (i = *vio->vui_iter; + iov_iter_count(&i); + iov_iter_advance(&i, iov.iov_len)) { + iov = iov_iter_iovec(&i); + addr = (unsigned long)iov.iov_base; + count = iov.iov_len; if (count == 0) continue; @@ -512,37 +538,48 @@ static void vvp_io_advance(const struct lu_env *env, return; vio->vui_tot_count -= nob; - if (io->ci_pio) { - iov_iter_advance(&io->u.ci_rw.rw_iter, nob); - io->u.ci_rw.rw_iocb.ki_pos = io->u.ci_rw.rw_range.cir_pos; -#ifdef HAVE_KIOCB_KI_LEFT - io->u.ci_rw.rw_iocb.ki_left = vio->vui_tot_count; -#elif defined(HAVE_KI_NBYTES) - io->u.ci_rw.rw_iocb.ki_nbytes = vio->vui_tot_count; -#endif - } else { - /* It was truncated to stripe size in vvp_io_rw_lock() */ - iov_iter_reexpand(&io->u.ci_rw.rw_iter, vio->vui_tot_count); - } + iov_iter_reexpand(vio->vui_iter, vio->vui_tot_count); +} + +static void vvp_io_update_iov(const struct lu_env *env, + struct vvp_io *vio, struct cl_io *io) +{ + size_t size = io->u.ci_rw.crw_count; + + if (!cl_is_normalio(env, io) || vio->vui_iter == NULL) + return; + + iov_iter_truncate(vio->vui_iter, size); } static int vvp_io_rw_lock(const struct lu_env *env, struct cl_io *io, enum cl_lock_mode mode, loff_t start, loff_t end) { + struct vvp_io *vio = vvp_env_io(env); int result; int ast_flags = 0; LASSERT(io->ci_type == CIT_READ || io->ci_type == CIT_WRITE); ENTRY; - if (cl_is_normalio(env, io)) - iov_iter_truncate(&io->u.ci_rw.rw_iter, - io->u.ci_rw.rw_range.cir_count); + vvp_io_update_iov(env, vio, io); - if (io->u.ci_rw.rw_nonblock) + if (io->u.ci_rw.crw_nonblock) ast_flags |= CEF_NONBLOCK; + if (io->ci_lock_no_expand) + ast_flags |= CEF_LOCK_NO_EXPAND; + if (vio->vui_fd) { + /* Group lock held means no lockless any more */ + if (vio->vui_fd->fd_flags & LL_FILE_GROUP_LOCKED) + io->ci_ignore_lockless = 1; + + if (ll_file_nolock(vio->vui_fd->fd_file) || + (vio->vui_fd->fd_flags & LL_FILE_LOCKLESS_IO && + !io->ci_ignore_lockless)) + ast_flags |= CEF_NEVER; + } - result = vvp_mmap_locks(env, io); + result = vvp_mmap_locks(env, vio, io); if (result == 0) result = vvp_io_one_lock(env, io, ast_flags, mode, start, end); @@ -553,13 +590,13 @@ static int vvp_io_read_lock(const struct lu_env *env, const struct cl_io_slice *ios) { struct cl_io *io = ios->cis_io; - struct cl_io_range *range = &io->u.ci_rw.rw_range; - int rc; + struct cl_io_rw_common *rd = &io->u.ci_rd.rd; + int result; ENTRY; - rc = vvp_io_rw_lock(env, io, CLM_READ, range->cir_pos, - range->cir_pos + range->cir_count - 1); - RETURN(rc); + result = vvp_io_rw_lock(env, io, CLM_READ, rd->crw_pos, + rd->crw_pos + rd->crw_count - 1); + RETURN(result); } static int vvp_io_fault_lock(const struct lu_env *env, @@ -578,27 +615,26 @@ static int vvp_io_fault_lock(const struct lu_env *env, } static int vvp_io_write_lock(const struct lu_env *env, - const struct cl_io_slice *ios) + const struct cl_io_slice *ios) { struct cl_io *io = ios->cis_io; loff_t start; loff_t end; - int rc; - ENTRY; - if (io->u.ci_rw.rw_append) { + if (io->u.ci_wr.wr_append) { start = 0; end = OBD_OBJECT_EOF; } else { - start = io->u.ci_rw.rw_range.cir_pos; - end = start + io->u.ci_rw.rw_range.cir_count - 1; + start = io->u.ci_wr.wr.crw_pos; + end = start + io->u.ci_wr.wr.crw_count - 1; } - rc = vvp_io_rw_lock(env, io, CLM_WRITE, start, end); - RETURN(rc); + + RETURN(vvp_io_rw_lock(env, io, CLM_WRITE, start, end)); } static int vvp_io_setattr_iter_init(const struct lu_env *env, const struct cl_io_slice *ios) + { return 0; } @@ -612,15 +648,18 @@ static int vvp_io_setattr_lock(const struct lu_env *env, const struct cl_io_slice *ios) { struct cl_io *io = ios->cis_io; - __u64 new_size; + __u64 lock_start = 0; + __u64 lock_end = OBD_OBJECT_EOF; __u32 enqflags = 0; - if (cl_io_is_trunc(io)) { - new_size = io->u.ci_setattr.sa_attr.lvb_size; - if (new_size == 0) - enqflags = CEF_DISCARD_DATA; - } else { - unsigned int valid = io->u.ci_setattr.sa_valid; + if (cl_io_is_trunc(io)) { + if (io->u.ci_setattr.sa_attr.lvb_size == 0) + enqflags = CEF_DISCARD_DATA; + } else if (cl_io_is_fallocate(io)) { + lock_start = io->u.ci_setattr.sa_falloc_offset; + lock_end = lock_start + io->u.ci_setattr.sa_attr.lvb_size; + } else { + unsigned int valid = io->u.ci_setattr.sa_avalid; if (!(valid & TIMES_SET_FLAGS)) return 0; @@ -632,12 +671,10 @@ static int vvp_io_setattr_lock(const struct lu_env *env, io->u.ci_setattr.sa_attr.lvb_atime >= io->u.ci_setattr.sa_attr.lvb_ctime)) return 0; - - new_size = 0; } return vvp_io_one_lock(env, io, enqflags, CLM_WRITE, - new_size, OBD_OBJECT_EOF); + lock_start, lock_end); } static int vvp_do_vmtruncate(struct inode *inode, size_t size) @@ -669,16 +706,16 @@ static int vvp_io_setattr_time(const struct lu_env *env, int result; unsigned valid = CAT_CTIME; - cl_object_attr_lock(obj); - attr->cat_ctime = io->u.ci_setattr.sa_attr.lvb_ctime; - if (io->u.ci_setattr.sa_valid & ATTR_ATIME_SET) { - attr->cat_atime = io->u.ci_setattr.sa_attr.lvb_atime; - valid |= CAT_ATIME; - } - if (io->u.ci_setattr.sa_valid & ATTR_MTIME_SET) { - attr->cat_mtime = io->u.ci_setattr.sa_attr.lvb_mtime; - valid |= CAT_MTIME; - } + cl_object_attr_lock(obj); + attr->cat_ctime = io->u.ci_setattr.sa_attr.lvb_ctime; + if (io->u.ci_setattr.sa_avalid & ATTR_ATIME_SET) { + attr->cat_atime = io->u.ci_setattr.sa_attr.lvb_atime; + valid |= CAT_ATIME; + } + if (io->u.ci_setattr.sa_avalid & ATTR_MTIME_SET) { + attr->cat_mtime = io->u.ci_setattr.sa_attr.lvb_mtime; + valid |= CAT_MTIME; + } result = cl_object_attr_update(env, obj, attr, valid); cl_object_attr_unlock(obj); @@ -693,14 +730,17 @@ static int vvp_io_setattr_start(const struct lu_env *env, struct ll_inode_info *lli = ll_i2info(inode); if (cl_io_is_trunc(io)) { - down_write(&lli->lli_trunc_sem); + trunc_sem_down_write(&lli->lli_trunc_sem); + inode_lock(inode); + inode_dio_wait(inode); + } else if (cl_io_is_fallocate(io)) { inode_lock(inode); inode_dio_wait(inode); } else { inode_lock(inode); } - if (io->u.ci_setattr.sa_valid & TIMES_SET_FLAGS) + if (io->u.ci_setattr.sa_avalid & TIMES_SET_FLAGS) return vvp_io_setattr_time(env, ios); return 0; @@ -717,9 +757,10 @@ static void vvp_io_setattr_end(const struct lu_env *env, /* Truncate in memory pages - they must be clean pages * because osc has already notified to destroy osc_extents. */ vvp_do_vmtruncate(inode, io->u.ci_setattr.sa_attr.lvb_size); - inode_dio_write_done(inode); inode_unlock(inode); - up_write(&lli->lli_trunc_sem); + trunc_sem_up_write(&lli->lli_trunc_sem); + } else if (cl_io_is_fallocate(io)) { + inode_unlock(inode); } else { inode_unlock(inode); } @@ -742,40 +783,47 @@ static void vvp_io_setattr_fini(const struct lu_env *env, static int vvp_io_read_start(const struct lu_env *env, const struct cl_io_slice *ios) { - struct vvp_io *vio = cl2vvp_io(env, ios); - struct cl_io *io = ios->cis_io; - struct cl_object *obj = io->ci_obj; - struct inode *inode = vvp_object_inode(obj); - struct ll_inode_info *lli = ll_i2info(inode); - struct file *file = vio->vui_fd->fd_file; - struct cl_io_range *range = &io->u.ci_rw.rw_range; - loff_t pos = range->cir_pos; /* for generic_file_splice_read() only */ + struct vvp_io *vio = cl2vvp_io(env, ios); + struct cl_io *io = ios->cis_io; + struct cl_object *obj = io->ci_obj; + struct inode *inode = vvp_object_inode(obj); + struct ll_inode_info *lli = ll_i2info(inode); + struct file *file = vio->vui_fd->fd_file; + loff_t pos = io->u.ci_rd.rd.crw_pos; + size_t cnt = io->u.ci_rd.rd.crw_count; size_t tot = vio->vui_tot_count; int exceed = 0; int result; + ENTRY; CLOBINVRNT(env, obj, vvp_object_invariant(obj)); CDEBUG(D_VFSTRACE, "%s: read [%llu, %llu)\n", file_dentry(file)->d_name.name, - range->cir_pos, range->cir_pos + range->cir_count); + pos, pos + cnt); if (vio->vui_io_subtype == IO_NORMAL) - down_read(&lli->lli_trunc_sem); + trunc_sem_down_read(&lli->lli_trunc_sem); + + if (io->ci_async_readahead) { + file_accessed(file); + RETURN(0); + } if (!can_populate_pages(env, io, inode)) - return 0; + RETURN(0); - result = vvp_prep_size(env, obj, io, range->cir_pos, tot, &exceed); + /* Unless this is reading a sparse file, otherwise the lock has already + * been acquired so vvp_prep_size() is an empty op. */ + result = vvp_prep_size(env, obj, io, pos, cnt, &exceed); if (result != 0) - return result; + RETURN(result); else if (exceed != 0) - goto out; + GOTO(out, result); LU_OBJECT_HEADER(D_INODE, env, &obj->co_lu, "Read ino %lu, %lu bytes, offset %lld, size %llu\n", - inode->i_ino, range->cir_count, range->cir_pos, - i_size_read(inode)); + inode->i_ino, cnt, pos, i_size_read(inode)); /* turn off the kernel's read-ahead */ vio->vui_fd->fd_file->f_ra.ra_pages = 0; @@ -783,26 +831,27 @@ static int vvp_io_read_start(const struct lu_env *env, /* initialize read-ahead window once per syscall */ if (!vio->vui_ra_valid) { vio->vui_ra_valid = true; - vio->vui_ra_start = cl_index(obj, range->cir_pos); - vio->vui_ra_count = cl_index(obj, tot + PAGE_SIZE - 1); - ll_ras_enter(file); + vio->vui_ra_start_idx = cl_index(obj, pos); + vio->vui_ra_pages = cl_index(obj, tot + PAGE_SIZE - 1); + /* If both start and end are unaligned, we read one more page + * than the index math suggests. */ + if ((pos & ~PAGE_MASK) != 0 && ((pos + tot) & ~PAGE_MASK) != 0) + vio->vui_ra_pages++; + + CDEBUG(D_READA, "tot %zu, ra_start %lu, ra_count %lu\n", + tot, vio->vui_ra_start_idx, vio->vui_ra_pages); } /* BUG: 5972 */ file_accessed(file); switch (vio->vui_io_subtype) { case IO_NORMAL: - LASSERTF(io->u.ci_rw.rw_iocb.ki_pos == range->cir_pos, - "ki_pos %lld [%lld, %lld)\n", - io->u.ci_rw.rw_iocb.ki_pos, - range->cir_pos, range->cir_pos + range->cir_count); - result = generic_file_read_iter(&io->u.ci_rw.rw_iocb, - &io->u.ci_rw.rw_iter); + LASSERT(vio->vui_iocb->ki_pos == pos); + result = generic_file_read_iter(vio->vui_iocb, vio->vui_iter); break; case IO_SPLICE: result = generic_file_splice_read(file, &pos, - vio->u.splice.vui_pipe, - range->cir_count, + vio->u.splice.vui_pipe, cnt, vio->u.splice.vui_flags); /* LU-1109: do splice read stripe by stripe otherwise if it * may make nfsd stuck if this read occupied all internal pipe @@ -813,14 +862,13 @@ static int vvp_io_read_start(const struct lu_env *env, CERROR("Wrong IO type %u\n", vio->vui_io_subtype); LBUG(); } + GOTO(out, result); out: if (result >= 0) { - if (result < range->cir_count) + if (result < cnt) io->ci_continue = 0; io->ci_nob += result; - ll_rw_stats_tally(ll_i2sbi(inode), current->pid, vio->vui_fd, - range->cir_pos, result, READ); result = 0; } @@ -876,6 +924,7 @@ static int vvp_io_commit_sync(const struct lu_env *env, struct cl_io *io, SetPageUptodate(cl_page_vmpage(page)); cl_page_disown(env, io, page); + /* held in ll_cl_init() */ lu_ref_del(&page->cp_reference, "cl_io", io); cl_page_put(env, page); } @@ -884,18 +933,139 @@ static int vvp_io_commit_sync(const struct lu_env *env, struct cl_io *io, RETURN(bytes > 0 ? bytes : rc); } +/* + * Kernels 4.2 - 4.5 pass memcg argument to account_page_dirtied() + * Kernel v5.2-5678-gac1c3e4 no longer exports account_page_dirtied + */ +static inline void ll_account_page_dirtied(struct page *page, + struct address_space *mapping) +{ +#ifdef HAVE_ACCOUNT_PAGE_DIRTIED_3ARGS + struct mem_cgroup *memcg = mem_cgroup_begin_page_stat(page); + + account_page_dirtied(page, mapping, memcg); + mem_cgroup_end_page_stat(memcg); +#elif defined(HAVE_ACCOUNT_PAGE_DIRTIED_EXPORT) + account_page_dirtied(page, mapping); +#else + vvp_account_page_dirtied(page, mapping); +#endif +} + +/* + * From kernel v4.19-rc5-248-g9b89a0355144 use XArrary + * Prior kernels use radix_tree for tags + */ +static inline void ll_page_tag_dirty(struct page *page, + struct address_space *mapping) +{ +#ifndef HAVE_RADIX_TREE_TAG_SET + __xa_set_mark(&mapping->i_pages, page_index(page), PAGECACHE_TAG_DIRTY); +#else + radix_tree_tag_set(&mapping->page_tree, page_index(page), + PAGECACHE_TAG_DIRTY); +#endif +} + +/* Taken from kernel set_page_dirty, __set_page_dirty_nobuffers + * Last change to this area: b93b016313b3ba8003c3b8bb71f569af91f19fc7 + * + * Current with Linus tip of tree (7/13/2019): + * v5.2-rc4-224-ge01e060fe0 + * + * Backwards compat for 3.x, 5.x kernels relating to memcg handling + * & rename of radix tree to xarray. + */ +void vvp_set_pagevec_dirty(struct pagevec *pvec) +{ + struct page *page = pvec->pages[0]; + struct address_space *mapping = page->mapping; + unsigned long flags; + int count = pagevec_count(pvec); + int dirtied = 0; + int i = 0; + + ENTRY; + + /* From set_page_dirty */ + for (i = 0; i < count; i++) + ClearPageReclaim(pvec->pages[i]); + + LASSERTF(page->mapping, + "mapping must be set. page %p, page->private (cl_page) %p\n", + page, (void *) page->private); + + /* Rest of code derived from __set_page_dirty_nobuffers */ + ll_xa_lock_irqsave(&mapping->i_pages, flags); + + /* Notes on differences with __set_page_dirty_nobuffers: + * 1. We don't need to call page_mapping because we know this is a page + * cache page. + * 2. We have the pages locked, so there is no need for the careful + * mapping/mapping2 dance. + * 3. No mapping is impossible. (Race w/truncate mentioned in + * dirty_nobuffers should be impossible because we hold the page lock.) + * 4. All mappings are the same because i/o is only to one file. + * 5. We invert the lock order on lock_page_memcg(page) and the mapping + * xa_lock, but this is the only function that should use that pair of + * locks and it can't race because Lustre locks pages throughout i/o. + */ + for (i = 0; i < count; i++) { + page = pvec->pages[i]; + lock_page_memcg(page); + if (TestSetPageDirty(page)) { + unlock_page_memcg(page); + continue; + } + LASSERTF(page->mapping == mapping, + "all pages must have the same mapping. page %p, mapping %p, first mapping %p\n", + page, page->mapping, mapping); + WARN_ON_ONCE(!PagePrivate(page) && !PageUptodate(page)); + ll_account_page_dirtied(page, mapping); + ll_page_tag_dirty(page, mapping); + dirtied++; + unlock_page_memcg(page); + } + ll_xa_unlock_irqrestore(&mapping->i_pages, flags); + + CDEBUG(D_VFSTRACE, "mapping %p, count %d, dirtied %d\n", mapping, + count, dirtied); + + if (mapping->host && dirtied) { + /* !PageAnon && !swapper_space */ + __mark_inode_dirty(mapping->host, I_DIRTY_PAGES); + } + + EXIT; +} + static void write_commit_callback(const struct lu_env *env, struct cl_io *io, - struct cl_page *page) + struct pagevec *pvec) { - struct page *vmpage = page->cp_vmpage; + int count = 0; + int i = 0; + + ENTRY; - SetPageUptodate(vmpage); - set_page_dirty(vmpage); + count = pagevec_count(pvec); + LASSERT(count > 0); - cl_page_disown(env, io, page); + for (i = 0; i < count; i++) { + struct page *vmpage = pvec->pages[i]; + SetPageUptodate(vmpage); + } + + vvp_set_pagevec_dirty(pvec); + + for (i = 0; i < count; i++) { + struct page *vmpage = pvec->pages[i]; + struct cl_page *page = (struct cl_page *) vmpage->private; + cl_page_disown(env, io, page); + lu_ref_del(&page->cp_reference, "cl_io", cl_io_top(io)); + cl_page_put(env, page); + } - lu_ref_del(&page->cp_reference, "cl_io", cl_io_top(io)); - cl_page_put(env, page); + EXIT; } /* make sure the page list is contiguous */ @@ -994,6 +1164,7 @@ int vvp_io_write_commit(const struct lu_env *env, struct cl_io *io) cl_page_disown(env, io, page); + /* held in ll_cl_init() */ lu_ref_del(&page->cp_reference, "cl_io", io); cl_page_put(env, page); } @@ -1011,14 +1182,18 @@ static int vvp_io_write_start(const struct lu_env *env, struct inode *inode = vvp_object_inode(obj); struct ll_inode_info *lli = ll_i2info(inode); struct file *file = vio->vui_fd->fd_file; - struct cl_io_range *range = &io->u.ci_rw.rw_range; - bool lock_inode = !lli->lli_inode_locked && - !IS_NOSEC(inode); ssize_t result = 0; + loff_t pos = io->u.ci_wr.wr.crw_pos; + size_t cnt = io->u.ci_wr.wr.crw_count; + bool lock_inode = !IS_NOSEC(inode); + size_t nob = io->ci_nob; + struct iov_iter iter; + size_t written = 0; + ENTRY; if (vio->vui_io_subtype == IO_NORMAL) - down_read(&lli->lli_trunc_sem); + trunc_sem_down_read(&lli->lli_trunc_sem); if (!can_populate_pages(env, io, inode)) RETURN(0); @@ -1029,29 +1204,28 @@ static int vvp_io_write_start(const struct lu_env *env, * out-of-order writes. */ ll_merge_attr(env, inode); - range->cir_pos = i_size_read(inode); - io->u.ci_rw.rw_iocb.ki_pos = range->cir_pos; + pos = io->u.ci_wr.wr.crw_pos = i_size_read(inode); + vio->vui_iocb->ki_pos = pos; } else { - LASSERTF(io->u.ci_rw.rw_iocb.ki_pos == range->cir_pos, + LASSERTF(vio->vui_iocb->ki_pos == pos, "ki_pos %lld [%lld, %lld)\n", - io->u.ci_rw.rw_iocb.ki_pos, - range->cir_pos, range->cir_pos + range->cir_count); + vio->vui_iocb->ki_pos, + pos, pos + cnt); } CDEBUG(D_VFSTRACE, "%s: write [%llu, %llu)\n", file_dentry(file)->d_name.name, - range->cir_pos, range->cir_pos + range->cir_count); + pos, pos + cnt); /* The maximum Lustre file size is variable, based on the OST maximum * object size and number of stripes. This needs another check in * addition to the VFS checks earlier. */ - if (range->cir_pos + range->cir_count > ll_file_maxbytes(inode)) { + if (pos + cnt > ll_file_maxbytes(inode)) { CDEBUG(D_INODE, "%s: file %s ("DFID") offset %llu > maxbytes %llu\n", - ll_get_fsname(inode->i_sb, NULL, 0), + ll_i2sbi(inode)->ll_fsname, file_dentry(file)->d_name.name, - PFID(ll_inode2fid(inode)), - range->cir_pos + range->cir_count, + PFID(ll_inode2fid(inode)), pos + cnt, ll_file_maxbytes(inode)); RETURN(-EFBIG); } @@ -1063,53 +1237,87 @@ static int vvp_io_write_start(const struct lu_env *env, if (OBD_FAIL_CHECK(OBD_FAIL_LLITE_IMUTEX_NOSEC) && lock_inode) RETURN(-EINVAL); - /* - * When using the locked AIO function (generic_file_aio_write()) - * testing has shown the inode mutex to be a limiting factor - * with multi-threaded single shared file performance. To get - * around this, we now use the lockless version. To maintain - * consistency, proper locking to protect against writes, - * trucates, etc. is handled in the higher layers of lustre. - */ - if (lock_inode) - inode_lock(inode); - result = __generic_file_write_iter(&io->u.ci_rw.rw_iocb, - &io->u.ci_rw.rw_iter); - if (lock_inode) - inode_unlock(inode); - - if (result > 0 || result == -EIOCBQUEUED) + if (vio->vui_iter == NULL) { + /* from a temp io in ll_cl_init(). */ + result = 0; + } else { + /* + * When using the locked AIO function (generic_file_aio_write()) + * testing has shown the inode mutex to be a limiting factor + * with multi-threaded single shared file performance. To get + * around this, we now use the lockless version. To maintain + * consistency, proper locking to protect against writes, + * trucates, etc. is handled in the higher layers of lustre. + */ + lock_inode = !IS_NOSEC(inode); + iter = *vio->vui_iter; + + if (unlikely(lock_inode)) + inode_lock(inode); + result = __generic_file_write_iter(vio->vui_iocb, + vio->vui_iter); + if (unlikely(lock_inode)) + inode_unlock(inode); + + written = result; + if (result > 0) #ifdef HAVE_GENERIC_WRITE_SYNC_2ARGS - result = generic_write_sync(&io->u.ci_rw.rw_iocb, result); + result = generic_write_sync(vio->vui_iocb, result); #else - { - ssize_t err; + { + ssize_t err; - err = generic_write_sync(io->u.ci_rw.rw_iocb.ki_filp, - range->cir_pos, result); - if (err < 0 && result > 0) - result = err; - } + err = generic_write_sync(vio->vui_iocb->ki_filp, pos, + result); + if (err < 0 && result > 0) + result = err; + } #endif + } if (result > 0) { result = vvp_io_write_commit(env, io); + /* Simulate short commit */ + if (CFS_FAULT_CHECK(OBD_FAIL_LLITE_SHORT_COMMIT)) { + vio->u.write.vui_written >>= 1; + if (vio->u.write.vui_written > 0) + io->ci_need_restart = 1; + } if (vio->u.write.vui_written > 0) { result = vio->u.write.vui_written; CDEBUG(D_VFSTRACE, "%s: write nob %zd, result: %zd\n", file_dentry(file)->d_name.name, io->ci_nob, result); io->ci_nob += result; + } else { + io->ci_continue = 0; } } - if (result > 0) { + if (vio->vui_iocb->ki_pos != (pos + io->ci_nob - nob)) { + CDEBUG(D_VFSTRACE, "%s: write position mismatch: " + "ki_pos %lld vs. pos %lld, written %ld, commit %ld " + "rc %ld\n", + file_dentry(file)->d_name.name, + vio->vui_iocb->ki_pos, pos + io->ci_nob - nob, + written, io->ci_nob - nob, result); + /* + * Rewind ki_pos and vui_iter to where it has + * successfully committed. + */ + vio->vui_iocb->ki_pos = pos + io->ci_nob - nob; + iov_iter_advance(&iter, io->ci_nob - nob); + vio->vui_iter->iov = iter.iov; + vio->vui_iter->nr_segs = iter.nr_segs; + vio->vui_iter->iov_offset = iter.iov_offset; + vio->vui_iter->count = iter.count; + } + if (result > 0 || result == -EIOCBQUEUED) { ll_file_set_flag(ll_i2info(inode), LLIF_DATA_MODIFIED); - if (result < range->cir_count) + if (result < cnt) io->ci_continue = 0; - ll_rw_stats_tally(ll_i2sbi(inode), current->pid, - vio->vui_fd, range->cir_pos, result, WRITE); - result = 0; + if (result > 0) + result = 0; } RETURN(result); @@ -1123,19 +1331,19 @@ static void vvp_io_rw_end(const struct lu_env *env, struct ll_inode_info *lli = ll_i2info(inode); if (vio->vui_io_subtype == IO_NORMAL) - up_read(&lli->lli_trunc_sem); + trunc_sem_up_read(&lli->lli_trunc_sem); } static int vvp_io_kernel_fault(struct vvp_fault_io *cfio) { struct vm_fault *vmf = cfio->ft_vmf; - cfio->ft_flags = filemap_fault(cfio->ft_vma, vmf); + cfio->ft_flags = ll_filemap_fault(cfio->ft_vma, vmf); cfio->ft_flags_valid = 1; if (vmf->page) { LL_CDEBUG_PAGE(D_PAGE, vmf->page, "got addr %p type NOPAGE\n", - vmf->virtual_address); + get_vmf_address(vmf)); if (unlikely(!(cfio->ft_flags & VM_FAULT_LOCKED))) { lock_page(vmf->page); cfio->ft_flags |= VM_FAULT_LOCKED; @@ -1147,12 +1355,12 @@ static int vvp_io_kernel_fault(struct vvp_fault_io *cfio) } if (cfio->ft_flags & VM_FAULT_SIGBUS) { - CDEBUG(D_PAGE, "got addr %p - SIGBUS\n", vmf->virtual_address); + CDEBUG(D_PAGE, "got addr %p - SIGBUS\n", get_vmf_address(vmf)); return -EFAULT; } if (cfio->ft_flags & VM_FAULT_OOM) { - CDEBUG(D_PAGE, "got addr %p - OOM\n", vmf->virtual_address); + CDEBUG(D_PAGE, "got addr %p - OOM\n", get_vmf_address(vmf)); return -ENOMEM; } @@ -1165,9 +1373,9 @@ static int vvp_io_kernel_fault(struct vvp_fault_io *cfio) } static void mkwrite_commit_callback(const struct lu_env *env, struct cl_io *io, - struct cl_page *page) + struct pagevec *pvec) { - set_page_dirty(page->cp_vmpage); + vvp_set_pagevec_dirty(pvec); } static int vvp_io_fault_start(const struct lu_env *env, @@ -1188,7 +1396,7 @@ static int vvp_io_fault_start(const struct lu_env *env, pgoff_t last_index; ENTRY; - down_read(&lli->lli_trunc_sem); + trunc_sem_down_read_nowait(&lli->lli_trunc_sem); /* offset of the last byte on the page */ offset = cl_offset(obj, fio->ft_index + 1) - 1; @@ -1211,7 +1419,7 @@ static int vvp_io_fault_start(const struct lu_env *env, LASSERT(PageLocked(vmpage)); if (OBD_FAIL_CHECK(OBD_FAIL_LLITE_FAULT_TRUNC_RACE)) - ll_invalidate_page(vmpage); + generic_error_remove_page(vmpage->mapping, vmpage); size = i_size_read(inode); /* Though we have already held a cl_lock upon this page, but @@ -1263,7 +1471,7 @@ static int vvp_io_fault_start(const struct lu_env *env, if (fio->ft_mkwrite) { wait_on_page_writeback(vmpage); if (!PageDirty(vmpage)) { - struct cl_page_list *plist = &io->ci_queue.c2_qin; + struct cl_page_list *plist = &vio->u.fault.ft_queue; struct vvp_page *vpg = cl_object_page_slice(obj, page); int to = PAGE_SIZE; @@ -1275,13 +1483,34 @@ static int vvp_io_fault_start(const struct lu_env *env, /* size fixup */ if (last_index == vvp_index(vpg)) - to = size & ~PAGE_MASK; + to = ((size - 1) & ~PAGE_MASK) + 1; /* Do not set Dirty bit here so that in case IO is * started before the page is really made dirty, we * still have chance to detect it. */ result = cl_io_commit_async(env, io, plist, 0, to, mkwrite_commit_callback); + /* Have overquota flag, trying sync write to check + * whether indeed out of quota */ + if (result == -EDQUOT) { + cl_page_get(page); + result = vvp_io_commit_sync(env, io, + plist, 0, to); + if (result >= 0) { + io->ci_noquota = 1; + cl_page_own(env, io, page); + cl_page_list_add(plist, page); + lu_ref_add(&page->cp_reference, + "cl_io", io); + result = cl_io_commit_async(env, io, + plist, 0, to, + mkwrite_commit_callback); + io->ci_noquota = 0; + } else { + cl_page_put(env, page); + } + } + LASSERT(cl_page_is_owned(page, io)); cl_page_list_fini(env, plist); @@ -1296,8 +1525,9 @@ static int vvp_io_fault_start(const struct lu_env *env, if (result == -EDQUOT) result = -ENOSPC; GOTO(out, result); - } else + } else { cl_page_disown(env, io, page); + } } } @@ -1338,7 +1568,7 @@ static void vvp_io_fault_end(const struct lu_env *env, CLOBINVRNT(env, ios->cis_io->ci_obj, vvp_object_invariant(ios->cis_io->ci_obj)); - up_read(&lli->lli_trunc_sem); + trunc_sem_up_read(&lli->lli_trunc_sem); } static int vvp_io_fsync_start(const struct lu_env *env, @@ -1362,7 +1592,7 @@ static int vvp_io_read_ahead(const struct lu_env *env, struct vvp_io *vio = cl2vvp_io(env, ios); if (unlikely(vio->vui_fd->fd_flags & LL_FILE_GROUP_LOCKED)) { - ra->cra_end = CL_PAGE_EOF; + ra->cra_end_idx = CL_PAGE_EOF; result = +1; /* no need to call down */ } } @@ -1406,6 +1636,9 @@ static const struct cl_io_operations vvp_io_ops = { .cio_start = vvp_io_fsync_start, .cio_fini = vvp_io_fini }, + [CIT_GLIMPSE] = { + .cio_fini = vvp_io_fini + }, [CIT_MISC] = { .cio_fini = vvp_io_fini }, @@ -1437,13 +1670,16 @@ int vvp_io_init(const struct lu_env *env, struct cl_object *obj, vio->vui_ra_valid = false; result = 0; if (io->ci_type == CIT_READ || io->ci_type == CIT_WRITE) { + size_t count; struct ll_inode_info *lli = ll_i2info(inode); - vio->vui_tot_count = io->u.ci_rw.rw_range.cir_count; + count = io->u.ci_rw.crw_count; /* "If nbyte is 0, read() will return 0 and have no other * results." -- Single Unix Spec */ - if (vio->vui_tot_count == 0) + if (count == 0) result = 1; + else + vio->vui_tot_count = count; /* for read/write, we store the jobid in the inode, and * it'll be fetched by osc when building RPC. @@ -1451,7 +1687,7 @@ int vvp_io_init(const struct lu_env *env, struct cl_object *obj, * it's not accurate if the file is shared by different * jobs. */ - lustre_get_jobid(lli->lli_jobid); + lustre_get_jobid(lli->lli_jobid, sizeof(lli->lli_jobid)); } else if (io->ci_type == CIT_SETATTR) { if (!cl_io_is_trunc(io)) io->ci_lockreq = CILR_MANDATORY; @@ -1470,9 +1706,10 @@ int vvp_io_init(const struct lu_env *env, struct cl_object *obj, result = 0; if (result < 0) CERROR("%s: refresh file layout " DFID " error %d.\n", - ll_get_fsname(inode->i_sb, NULL, 0), - PFID(lu_object_fid(&obj->co_lu)), result); + ll_i2sbi(inode)->ll_fsname, + PFID(lu_object_fid(&obj->co_lu)), result); } + io->ci_result = result < 0 ? result : 0; RETURN(result); }