X-Git-Url: https://git.whamcloud.com/?p=fs%2Flustre-release.git;a=blobdiff_plain;f=lustre%2Fllite%2Fvvp_io.c;h=94df8b5ac926a41d2d8380053a2f440c6f6e9a63;hp=b641fa950678104f33d52cc4dd69d09e5e29fe99;hb=8958ecee22010dfa1b99220c2288f7369f490871;hpb=500edcada7e4101a21a28078611597fdb50b4791

diff --git a/lustre/llite/vvp_io.c b/lustre/llite/vvp_io.c
index b641fa9..94df8b5 100644
--- a/lustre/llite/vvp_io.c
+++ b/lustre/llite/vvp_io.c
@@ -37,10 +37,13 @@
 
 #define DEBUG_SUBSYSTEM S_LLITE
 
-
 #include <obd.h>
+#include <linux/pagevec.h>
+#include <linux/memcontrol.h>
+
 #include "llite_internal.h"
 #include "vvp_internal.h"
+#include <libcfs/linux/linux-misc.h>
 
 static struct vvp_io *cl2vvp_io(const struct lu_env *env,
 				const struct cl_io_slice *slice)
@@ -287,7 +290,6 @@ static int vvp_io_fault_iter_init(const struct lu_env *env,
 	struct inode  *inode = vvp_object_inode(ios->cis_obj);
 
 	LASSERT(inode == file_inode(vio->vui_fd->fd_file));
-	vio->u.fault.ft_mtime = inode->i_mtime.tv_sec;
 
 	return 0;
 }
@@ -439,6 +441,8 @@ static int vvp_mmap_locks(const struct lu_env *env,
 	union ldlm_policy_data policy;
 	struct iovec iov;
 	struct iov_iter i;
+	unsigned long addr;
+	ssize_t count;
 	int result = 0;
 	ENTRY;
 
@@ -455,9 +459,15 @@ static int vvp_mmap_locks(const struct lu_env *env,
 	if (mm == NULL)
 		RETURN(0);
 
-	iov_for_each(iov, i, *(vio->vui_iter)) {
-		unsigned long addr = (unsigned long)iov.iov_base;
-		size_t count = iov.iov_len;
+	if (!iter_is_iovec(vio->vui_iter) && !iov_iter_is_kvec(vio->vui_iter))
+		RETURN(0);
+
+	for (i = *vio->vui_iter;
+	     iov_iter_count(&i);
+	     iov_iter_advance(&i, iov.iov_len)) {
+		iov = iov_iter_iovec(&i);
+		addr = (unsigned long)iov.iov_base;
+		count = iov.iov_len;
 
                 if (count == 0)
                         continue;
@@ -558,6 +568,16 @@ static int vvp_io_rw_lock(const struct lu_env *env, struct cl_io *io,
 		ast_flags |= CEF_NONBLOCK;
 	if (io->ci_lock_no_expand)
 		ast_flags |= CEF_LOCK_NO_EXPAND;
+	if (vio->vui_fd) {
+		/* Group lock held means no lockless any more */
+		if (vio->vui_fd->fd_flags & LL_FILE_GROUP_LOCKED)
+			io->ci_ignore_lockless = 1;
+
+		if (ll_file_nolock(vio->vui_fd->fd_file) ||
+		    (vio->vui_fd->fd_flags & LL_FILE_LOCKLESS_IO &&
+		     !io->ci_ignore_lockless))
+			ast_flags |= CEF_NEVER;
+	}
 
 	result = vvp_mmap_locks(env, vio, io);
 	if (result == 0)
@@ -628,13 +648,23 @@ static int vvp_io_setattr_lock(const struct lu_env *env,
                                const struct cl_io_slice *ios)
 {
 	struct cl_io  *io  = ios->cis_io;
-	__u64 new_size;
+	__u64 lock_start = 0;
+	__u64 lock_end = OBD_OBJECT_EOF;
 	__u32 enqflags = 0;
 
 	if (cl_io_is_trunc(io)) {
-		new_size = io->u.ci_setattr.sa_attr.lvb_size;
-		if (new_size == 0)
+		struct inode *inode = vvp_object_inode(io->ci_obj);
+
+		/* set enqueue flags to CEF_MUST in case of encrypted file,
+		 * to prevent lockless truncate
+		 */
+		if (S_ISREG(inode->i_mode) && IS_ENCRYPTED(inode))
+			enqflags = CEF_MUST;
+		else if (io->u.ci_setattr.sa_attr.lvb_size == 0)
 			enqflags = CEF_DISCARD_DATA;
+	} else if (cl_io_is_fallocate(io)) {
+		lock_start = io->u.ci_setattr.sa_falloc_offset;
+		lock_end = lock_start + io->u.ci_setattr.sa_attr.lvb_size;
 	} else {
 		unsigned int valid = io->u.ci_setattr.sa_avalid;
 
@@ -648,12 +678,10 @@ static int vvp_io_setattr_lock(const struct lu_env *env,
 		     io->u.ci_setattr.sa_attr.lvb_atime >=
 		     io->u.ci_setattr.sa_attr.lvb_ctime))
 			return 0;
-
-		new_size = 0;
 	}
 
 	return vvp_io_one_lock(env, io, enqflags, CLM_WRITE,
-			       new_size, OBD_OBJECT_EOF);
+			       lock_start, lock_end);
 }
 
 static int vvp_do_vmtruncate(struct inode *inode, size_t size)
@@ -709,11 +737,14 @@ static int vvp_io_setattr_start(const struct lu_env *env,
 	struct ll_inode_info	*lli   = ll_i2info(inode);
 
 	if (cl_io_is_trunc(io)) {
-		down_write(&lli->lli_trunc_sem);
+		trunc_sem_down_write(&lli->lli_trunc_sem);
+		mutex_lock(&lli->lli_setattr_mutex);
+		inode_dio_wait(inode);
+	} else if (cl_io_is_fallocate(io)) {
 		inode_lock(inode);
 		inode_dio_wait(inode);
 	} else {
-		inode_lock(inode);
+		mutex_lock(&lli->lli_setattr_mutex);
 	}
 
 	if (io->u.ci_setattr.sa_avalid & TIMES_SET_FLAGS)
@@ -733,11 +764,12 @@ static void vvp_io_setattr_end(const struct lu_env *env,
 		/* Truncate in memory pages - they must be clean pages
 		 * because osc has already notified to destroy osc_extents. */
 		vvp_do_vmtruncate(inode, io->u.ci_setattr.sa_attr.lvb_size);
-		inode_dio_write_done(inode);
+		mutex_unlock(&lli->lli_setattr_mutex);
+		trunc_sem_up_write(&lli->lli_trunc_sem);
+	} else if (cl_io_is_fallocate(io)) {
 		inode_unlock(inode);
-		up_write(&lli->lli_trunc_sem);
 	} else {
-		inode_unlock(inode);
+		mutex_unlock(&lli->lli_setattr_mutex);
 	}
 }
 
@@ -758,17 +790,17 @@ static void vvp_io_setattr_fini(const struct lu_env *env,
 static int vvp_io_read_start(const struct lu_env *env,
 			     const struct cl_io_slice *ios)
 {
-	struct vvp_io		*vio   = cl2vvp_io(env, ios);
-	struct cl_io		*io    = ios->cis_io;
-	struct cl_object	*obj   = io->ci_obj;
-	struct inode		*inode = vvp_object_inode(obj);
-	struct ll_inode_info	*lli   = ll_i2info(inode);
-	struct file		*file  = vio->vui_fd->fd_file;
-	loff_t  pos = io->u.ci_rd.rd.crw_pos;
-	long    cnt = io->u.ci_rd.rd.crw_count;
-	long    tot = vio->vui_tot_count;
-	int     exceed = 0;
-	int     result;
+	struct vvp_io *vio = cl2vvp_io(env, ios);
+	struct cl_io *io = ios->cis_io;
+	struct cl_object *obj = io->ci_obj;
+	struct inode *inode = vvp_object_inode(obj);
+	struct ll_inode_info *lli = ll_i2info(inode);
+	struct file *file = vio->vui_fd->fd_file;
+	loff_t pos = io->u.ci_rd.rd.crw_pos;
+	size_t cnt = io->u.ci_rd.rd.crw_count;
+	size_t tot = vio->vui_tot_count;
+	int exceed = 0;
+	int result;
 	ENTRY;
 
 	CLOBINVRNT(env, obj, vvp_object_invariant(obj));
@@ -778,7 +810,12 @@ static int vvp_io_read_start(const struct lu_env *env,
 		pos, pos + cnt);
 
 	if (vio->vui_io_subtype == IO_NORMAL)
-		down_read(&lli->lli_trunc_sem);
+		trunc_sem_down_read(&lli->lli_trunc_sem);
+
+	if (io->ci_async_readahead) {
+		file_accessed(file);
+		RETURN(0);
+	}
 
 	if (!can_populate_pages(env, io, inode))
 		RETURN(0);
@@ -801,8 +838,15 @@ static int vvp_io_read_start(const struct lu_env *env,
 	/* initialize read-ahead window once per syscall */
 	if (!vio->vui_ra_valid) {
 		vio->vui_ra_valid = true;
-		vio->vui_ra_start = cl_index(obj, pos);
-		vio->vui_ra_count = cl_index(obj, tot + PAGE_SIZE - 1);
+		vio->vui_ra_start_idx = cl_index(obj, pos);
+		vio->vui_ra_pages = cl_index(obj, tot + PAGE_SIZE - 1);
+		/* If both start and end are unaligned, we read one more page
+		 * than the index math suggests. */
+		if ((pos & ~PAGE_MASK) != 0 && ((pos + tot) & ~PAGE_MASK) != 0)
+			vio->vui_ra_pages++;
+
+		CDEBUG(D_READA, "tot %zu, ra_start %lu, ra_count %lu\n",
+		       tot, vio->vui_ra_start_idx, vio->vui_ra_pages);
 	}
 
 	/* BUG: 5972 */
@@ -832,8 +876,6 @@ out:
 		if (result < cnt)
 			io->ci_continue = 0;
 		io->ci_nob += result;
-		ll_rw_stats_tally(ll_i2sbi(inode), current->pid, vio->vui_fd,
-				  pos, result, READ);
 		result = 0;
 	}
 
@@ -898,19 +940,139 @@ static int vvp_io_commit_sync(const struct lu_env *env, struct cl_io *io,
 	RETURN(bytes > 0 ? bytes : rc);
 }
 
+/*
+ * Kernels 4.2 - 4.5 pass memcg argument to account_page_dirtied()
+ * Kernel v5.2-5678-gac1c3e4 no longer exports account_page_dirtied
+ */
+static inline void ll_account_page_dirtied(struct page *page,
+					   struct address_space *mapping)
+{
+#ifdef HAVE_ACCOUNT_PAGE_DIRTIED_3ARGS
+	struct mem_cgroup *memcg = mem_cgroup_begin_page_stat(page);
+
+	account_page_dirtied(page, mapping, memcg);
+	mem_cgroup_end_page_stat(memcg);
+#elif defined(HAVE_ACCOUNT_PAGE_DIRTIED_EXPORT)
+	account_page_dirtied(page, mapping);
+#else
+	vvp_account_page_dirtied(page, mapping);
+#endif
+}
+
+/*
+ * From kernel v4.19-rc5-248-g9b89a0355144 use XArrary
+ * Prior kernels use radix_tree for tags
+ */
+static inline void ll_page_tag_dirty(struct page *page,
+				     struct address_space *mapping)
+{
+#ifndef HAVE_RADIX_TREE_TAG_SET
+	__xa_set_mark(&mapping->i_pages, page_index(page), PAGECACHE_TAG_DIRTY);
+#else
+	radix_tree_tag_set(&mapping->page_tree, page_index(page),
+			   PAGECACHE_TAG_DIRTY);
+#endif
+}
+
+/* Taken from kernel set_page_dirty, __set_page_dirty_nobuffers
+ * Last change to this area: b93b016313b3ba8003c3b8bb71f569af91f19fc7
+ *
+ * Current with Linus tip of tree (7/13/2019):
+ * v5.2-rc4-224-ge01e060fe0
+ *
+ * Backwards compat for 3.x, 5.x kernels relating to memcg handling
+ * & rename of radix tree to xarray.
+ */
+void vvp_set_pagevec_dirty(struct pagevec *pvec)
+{
+	struct page *page = pvec->pages[0];
+	struct address_space *mapping = page->mapping;
+	unsigned long flags;
+	int count = pagevec_count(pvec);
+	int dirtied = 0;
+	int i = 0;
+
+	ENTRY;
+
+	/* From set_page_dirty */
+	for (i = 0; i < count; i++)
+		ClearPageReclaim(pvec->pages[i]);
+
+	LASSERTF(page->mapping,
+		 "mapping must be set. page %p, page->private (cl_page) %p\n",
+		 page, (void *) page->private);
+
+	/* Rest of code derived from __set_page_dirty_nobuffers */
+	ll_xa_lock_irqsave(&mapping->i_pages, flags);
+
+	/* Notes on differences with __set_page_dirty_nobuffers:
+	 * 1. We don't need to call page_mapping because we know this is a page
+	 * cache page.
+	 * 2. We have the pages locked, so there is no need for the careful
+	 * mapping/mapping2 dance.
+	 * 3. No mapping is impossible. (Race w/truncate mentioned in
+	 * dirty_nobuffers should be impossible because we hold the page lock.)
+	 * 4. All mappings are the same because i/o is only to one file.
+	 * 5. We invert the lock order on lock_page_memcg(page) and the mapping
+	 * xa_lock, but this is the only function that should use that pair of
+	 * locks and it can't race because Lustre locks pages throughout i/o.
+	 */
+	for (i = 0; i < count; i++) {
+		page = pvec->pages[i];
+		lock_page_memcg(page);
+		if (TestSetPageDirty(page)) {
+			unlock_page_memcg(page);
+			continue;
+		}
+		LASSERTF(page->mapping == mapping,
+			 "all pages must have the same mapping.  page %p, mapping %p, first mapping %p\n",
+			 page, page->mapping, mapping);
+		WARN_ON_ONCE(!PagePrivate(page) && !PageUptodate(page));
+		ll_account_page_dirtied(page, mapping);
+		ll_page_tag_dirty(page, mapping);
+		dirtied++;
+		unlock_page_memcg(page);
+	}
+	ll_xa_unlock_irqrestore(&mapping->i_pages, flags);
+
+	CDEBUG(D_VFSTRACE, "mapping %p, count %d, dirtied %d\n", mapping,
+	       count, dirtied);
+
+	if (mapping->host && dirtied) {
+		/* !PageAnon && !swapper_space */
+		__mark_inode_dirty(mapping->host, I_DIRTY_PAGES);
+	}
+
+	EXIT;
+}
+
 static void write_commit_callback(const struct lu_env *env, struct cl_io *io,
-				struct cl_page *page)
+				  struct pagevec *pvec)
 {
-	struct page *vmpage = page->cp_vmpage;
+	int count = 0;
+	int i = 0;
+
+	ENTRY;
 
-	SetPageUptodate(vmpage);
-	set_page_dirty(vmpage);
+	count = pagevec_count(pvec);
+	LASSERT(count > 0);
 
-	cl_page_disown(env, io, page);
+	for (i = 0; i < count; i++) {
+		struct page *vmpage = pvec->pages[i];
+		SetPageUptodate(vmpage);
+	}
+
+	vvp_set_pagevec_dirty(pvec);
+
+	for (i = 0; i < count; i++) {
+		struct page *vmpage = pvec->pages[i];
+		struct cl_page *page = (struct cl_page *) vmpage->private;
+		cl_page_disown(env, io, page);
+		lu_ref_del(&page->cp_reference, "cl_io", cl_io_top(io));
+		cl_page_put(env, page);
+	}
 
-	/* held in ll_cl_init() */
-	lu_ref_del(&page->cp_reference, "cl_io", cl_io_top(io));
-	cl_page_put(env, page);
+	EXIT;
 }
 
 /* make sure the page list is contiguous */
@@ -1031,11 +1193,14 @@ static int vvp_io_write_start(const struct lu_env *env,
 	loff_t			 pos = io->u.ci_wr.wr.crw_pos;
 	size_t			 cnt = io->u.ci_wr.wr.crw_count;
 	bool			 lock_inode = !IS_NOSEC(inode);
+	size_t nob = io->ci_nob;
+	struct iov_iter iter;
+	size_t written = 0;
 
 	ENTRY;
 
 	if (vio->vui_io_subtype == IO_NORMAL)
-		down_read(&lli->lli_trunc_sem);
+		trunc_sem_down_read(&lli->lli_trunc_sem);
 
 	if (!can_populate_pages(env, io, inode))
 		RETURN(0);
@@ -1092,6 +1257,7 @@ static int vvp_io_write_start(const struct lu_env *env,
 		 * trucates, etc. is handled in the higher layers of lustre.
 		 */
 		lock_inode = !IS_NOSEC(inode);
+		iter = *vio->vui_iter;
 
 		if (unlikely(lock_inode))
 			inode_lock(inode);
@@ -1100,7 +1266,8 @@ static int vvp_io_write_start(const struct lu_env *env,
 		if (unlikely(lock_inode))
 			inode_unlock(inode);
 
-		if (result > 0 || result == -EIOCBQUEUED)
+		written = result;
+		if (result > 0)
 #ifdef HAVE_GENERIC_WRITE_SYNC_2ARGS
 			result = generic_write_sync(vio->vui_iocb, result);
 #else
@@ -1117,22 +1284,47 @@ static int vvp_io_write_start(const struct lu_env *env,
 
 	if (result > 0) {
 		result = vvp_io_write_commit(env, io);
+		/* Simulate short commit */
+		if (CFS_FAULT_CHECK(OBD_FAIL_LLITE_SHORT_COMMIT)) {
+			vio->u.write.vui_written >>= 1;
+			if (vio->u.write.vui_written > 0)
+				io->ci_need_restart = 1;
+		}
 		if (vio->u.write.vui_written > 0) {
 			result = vio->u.write.vui_written;
 			CDEBUG(D_VFSTRACE, "%s: write nob %zd, result: %zd\n",
 				file_dentry(file)->d_name.name,
 				io->ci_nob, result);
 			io->ci_nob += result;
+		} else {
+			io->ci_continue = 0;
 		}
 	}
-	if (result > 0) {
+	if (vio->vui_iocb->ki_pos != (pos + io->ci_nob - nob)) {
+		CDEBUG(D_VFSTRACE, "%s: write position mismatch: "
+		       "ki_pos %lld vs. pos %lld, written %ld, commit %ld "
+		       "rc %ld\n",
+		       file_dentry(file)->d_name.name,
+		       vio->vui_iocb->ki_pos, pos + io->ci_nob - nob,
+		       written, io->ci_nob - nob, result);
+		/*
+		 * Rewind ki_pos and vui_iter to where it has
+		 * successfully committed.
+		 */
+		vio->vui_iocb->ki_pos = pos + io->ci_nob - nob;
+		iov_iter_advance(&iter, io->ci_nob - nob);
+		vio->vui_iter->iov = iter.iov;
+		vio->vui_iter->nr_segs = iter.nr_segs;
+		vio->vui_iter->iov_offset = iter.iov_offset;
+		vio->vui_iter->count = iter.count;
+	}
+	if (result > 0 || result == -EIOCBQUEUED) {
 		ll_file_set_flag(ll_i2info(inode), LLIF_DATA_MODIFIED);
 
 		if (result < cnt)
 			io->ci_continue = 0;
-		ll_rw_stats_tally(ll_i2sbi(inode), current->pid,
-				  vio->vui_fd, pos, result, WRITE);
-		result = 0;
+		if (result > 0)
+			result = 0;
 	}
 
 	RETURN(result);
@@ -1146,7 +1338,7 @@ static void vvp_io_rw_end(const struct lu_env *env,
 	struct ll_inode_info	*lli = ll_i2info(inode);
 
 	if (vio->vui_io_subtype == IO_NORMAL)
-		up_read(&lli->lli_trunc_sem);
+		trunc_sem_up_read(&lli->lli_trunc_sem);
 }
 
 static int vvp_io_kernel_fault(struct vvp_fault_io *cfio)
@@ -1188,9 +1380,9 @@ static int vvp_io_kernel_fault(struct vvp_fault_io *cfio)
 }
 
 static void mkwrite_commit_callback(const struct lu_env *env, struct cl_io *io,
-				    struct cl_page *page)
+				    struct pagevec *pvec)
 {
-	set_page_dirty(page->cp_vmpage);
+	vvp_set_pagevec_dirty(pvec);
 }
 
 static int vvp_io_fault_start(const struct lu_env *env,
@@ -1211,7 +1403,7 @@ static int vvp_io_fault_start(const struct lu_env *env,
 	pgoff_t			 last_index;
 	ENTRY;
 
-	down_read(&lli->lli_trunc_sem);
+	trunc_sem_down_read_nowait(&lli->lli_trunc_sem);
 
         /* offset of the last byte on the page */
         offset = cl_offset(obj, fio->ft_index + 1) - 1;
@@ -1234,7 +1426,7 @@ static int vvp_io_fault_start(const struct lu_env *env,
 	LASSERT(PageLocked(vmpage));
 
 	if (OBD_FAIL_CHECK(OBD_FAIL_LLITE_FAULT_TRUNC_RACE))
-		ll_invalidate_page(vmpage);
+		generic_error_remove_page(vmpage->mapping, vmpage);
 
 	size = i_size_read(inode);
         /* Though we have already held a cl_lock upon this page, but
@@ -1286,7 +1478,7 @@ static int vvp_io_fault_start(const struct lu_env *env,
 	if (fio->ft_mkwrite) {
 		wait_on_page_writeback(vmpage);
 		if (!PageDirty(vmpage)) {
-			struct cl_page_list *plist = &io->ci_queue.c2_qin;
+			struct cl_page_list *plist = &vio->u.fault.ft_queue;
 			struct vvp_page *vpg = cl_object_page_slice(obj, page);
 			int to = PAGE_SIZE;
 
@@ -1298,13 +1490,34 @@ static int vvp_io_fault_start(const struct lu_env *env,
 
 			/* size fixup */
 			if (last_index == vvp_index(vpg))
-				to = size & ~PAGE_MASK;
+				to = ((size - 1) & ~PAGE_MASK) + 1;
 
 			/* Do not set Dirty bit here so that in case IO is
 			 * started before the page is really made dirty, we
 			 * still have chance to detect it. */
 			result = cl_io_commit_async(env, io, plist, 0, to,
 						    mkwrite_commit_callback);
+			/* Have overquota flag, trying sync write to check
+			 * whether indeed out of quota */
+			if (result == -EDQUOT) {
+				cl_page_get(page);
+				result = vvp_io_commit_sync(env, io,
+							    plist, 0, to);
+				if (result >= 0) {
+					io->ci_noquota = 1;
+					cl_page_own(env, io, page);
+					cl_page_list_add(plist, page);
+					lu_ref_add(&page->cp_reference,
+						   "cl_io", io);
+					result = cl_io_commit_async(env, io,
+						plist, 0, to,
+						mkwrite_commit_callback);
+					io->ci_noquota = 0;
+				} else {
+					cl_page_put(env, page);
+				}
+			}
+
 			LASSERT(cl_page_is_owned(page, io));
 			cl_page_list_fini(env, plist);
 
@@ -1319,8 +1532,9 @@ static int vvp_io_fault_start(const struct lu_env *env,
 				if (result == -EDQUOT)
 					result = -ENOSPC;
 				GOTO(out, result);
-			} else
+			} else {
 				cl_page_disown(env, io, page);
+			}
 		}
 	}
 
@@ -1361,7 +1575,7 @@ static void vvp_io_fault_end(const struct lu_env *env,
 
 	CLOBINVRNT(env, ios->cis_io->ci_obj,
 		   vvp_object_invariant(ios->cis_io->ci_obj));
-	up_read(&lli->lli_trunc_sem);
+	trunc_sem_up_read(&lli->lli_trunc_sem);
 }
 
 static int vvp_io_fsync_start(const struct lu_env *env,
@@ -1385,7 +1599,7 @@ static int vvp_io_read_ahead(const struct lu_env *env,
 		struct vvp_io *vio = cl2vvp_io(env, ios);
 
 		if (unlikely(vio->vui_fd->fd_flags & LL_FILE_GROUP_LOCKED)) {
-			ra->cra_end = CL_PAGE_EOF;
+			ra->cra_end_idx = CL_PAGE_EOF;
 			result = +1; /* no need to call down */
 		}
 	}