1 /* -*- mode: c; c-basic-offset: 8; indent-tabs-mode: nil; -*-
2 * vim:expandtab:shiftwidth=8:tabstop=8:
4 * Copyright (c) 2001-2003 Cluster File Systems, Inc.
6 * This file is part of Lustre, http://www.lustre.org.
8 * Lustre is free software; you can redistribute it and/or
9 * modify it under the terms of version 2 of the GNU General Public
10 * License as published by the Free Software Foundation.
12 * Lustre is distributed in the hope that it will be useful,
13 * but WITHOUT ANY WARRANTY; without even the implied warranty of
14 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
15 * GNU General Public License for more details.
17 * You should have received a copy of the GNU General Public License
18 * along with Lustre; if not, write to the Free Software
19 * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
21 #ifndef AUTOCONF_INCLUDED
22 #include <linux/config.h>
24 #include <linux/kernel.h>
26 #include <linux/string.h>
27 #include <linux/stat.h>
28 #include <linux/errno.h>
29 #include <linux/smp_lock.h>
30 #include <linux/unistd.h>
31 #include <linux/version.h>
32 #include <asm/system.h>
33 #include <asm/uaccess.h>
36 #include <linux/stat.h>
37 #include <asm/uaccess.h>
39 #include <linux/pagemap.h>
40 #include <linux/smp_lock.h>
41 #if (LINUX_VERSION_CODE < KERNEL_VERSION(2,5,0))
42 #include <linux/iobuf.h>
45 #define DEBUG_SUBSYSTEM S_LLITE
47 #include <lustre_lite.h>
48 #include "llite_internal.h"
49 #include <linux/lustre_compat25.h>
51 #define VMA_DEBUG(vma, fmt, arg...) \
52 CDEBUG(D_MMAP, "vma(%p) start(%ld) end(%ld) pgoff(%ld) inode(%p) " \
53 "ino(%lu) iname(%s): " fmt, vma, vma->vm_start, vma->vm_end, \
54 vma->vm_pgoff, vma->vm_file->f_dentry->d_inode, \
55 vma->vm_file->f_dentry->d_inode->i_ino, \
56 vma->vm_file->f_dentry->d_iname, ## arg); \
59 struct ll_lock_tree_node {
61 struct list_head lt_locked_item;
63 ldlm_policy_data_t lt_policy;
64 struct lustre_handle lt_lockh;
66 struct inode *lt_inode;
69 int lt_get_mmap_locks(struct ll_lock_tree *tree,
70 unsigned long addr, size_t count);
72 struct page *ll_nopage(struct vm_area_struct *vma, unsigned long address,
75 struct ll_lock_tree_node * ll_node_from_inode(struct inode *inode, __u64 start,
76 __u64 end, ldlm_mode_t mode)
78 struct ll_lock_tree_node *node;
80 OBD_ALLOC(node, sizeof(*node));
82 RETURN(ERR_PTR(-ENOMEM));
84 node->lt_inode = inode;
85 node->lt_oid = ll_i2info(inode)->lli_smd->lsm_object_id;
86 node->lt_policy.l_extent.start = start;
87 node->lt_policy.l_extent.end = end;
88 memset(&node->lt_lockh, 0, sizeof(node->lt_lockh));
89 INIT_LIST_HEAD(&node->lt_locked_item);
95 int lt_compare(struct ll_lock_tree_node *one, struct ll_lock_tree_node *two)
97 /* To avoid multiple fs deadlock */
98 if (one->lt_inode->i_sb->s_dev < two->lt_inode->i_sb->s_dev)
100 if (one->lt_inode->i_sb->s_dev > two->lt_inode->i_sb->s_dev)
103 if (one->lt_oid < two->lt_oid)
105 if (one->lt_oid > two->lt_oid)
108 if (one->lt_policy.l_extent.end < two->lt_policy.l_extent.start)
110 if (one->lt_policy.l_extent.start > two->lt_policy.l_extent.end)
113 return 0; /* they are the same object and overlap */
116 static void lt_merge(struct ll_lock_tree_node *dst,
117 struct ll_lock_tree_node *src)
119 dst->lt_policy.l_extent.start = min(dst->lt_policy.l_extent.start,
120 src->lt_policy.l_extent.start);
121 dst->lt_policy.l_extent.end = max(dst->lt_policy.l_extent.end,
122 src->lt_policy.l_extent.end);
124 /* XXX could be a real call to the dlm to find superset modes */
125 if (src->lt_mode == LCK_PW && dst->lt_mode != LCK_PW)
126 dst->lt_mode = LCK_PW;
129 static void lt_insert(struct ll_lock_tree *tree,
130 struct ll_lock_tree_node *node)
132 struct ll_lock_tree_node *walk;
133 rb_node_t **p, *parent;
137 p = &tree->lt_root.rb_node;
141 walk = rb_entry(parent, struct ll_lock_tree_node, lt_node);
142 switch (lt_compare(node, walk)) {
150 lt_merge(node, walk);
151 rb_erase(&walk->lt_node, &tree->lt_root);
152 OBD_FREE(walk, sizeof(*walk));
160 rb_link_node(&node->lt_node, parent, p);
161 rb_insert_color(&node->lt_node, &tree->lt_root);
165 static struct ll_lock_tree_node *lt_least_node(struct ll_lock_tree *tree)
168 struct ll_lock_tree_node *node = NULL;
170 for ( rbnode = tree->lt_root.rb_node; rbnode != NULL;
171 rbnode = rbnode->rb_left) {
172 if (rbnode->rb_left == NULL) {
173 node = rb_entry(rbnode, struct ll_lock_tree_node,
181 int ll_tree_unlock(struct ll_lock_tree *tree)
183 struct ll_lock_tree_node *node;
184 struct list_head *pos, *n;
189 list_for_each_safe(pos, n, &tree->lt_locked_list) {
190 node = list_entry(pos, struct ll_lock_tree_node,
193 inode = node->lt_inode;
194 rc = ll_extent_unlock(tree->lt_fd, inode,
195 ll_i2info(inode)->lli_smd, node->lt_mode,
198 /* XXX better message */
199 CERROR("couldn't unlock %d\n", rc);
201 list_del(&node->lt_locked_item);
202 OBD_FREE(node, sizeof(*node));
205 while ((node = lt_least_node(tree))) {
206 rb_erase(&node->lt_node, &tree->lt_root);
207 OBD_FREE(node, sizeof(*node));
213 int ll_tree_lock_iov(struct ll_lock_tree *tree,
214 struct ll_lock_tree_node *first_node,
215 const struct iovec *iov, unsigned long nr_segs, int ast_flags)
217 struct ll_lock_tree_node *node;
222 tree->lt_root.rb_node = NULL;
223 INIT_LIST_HEAD(&tree->lt_locked_list);
224 if (first_node != NULL)
225 lt_insert(tree, first_node);
227 /* To avoid such subtle deadlock case: client1 try to read file1 to
228 * mmapped file2, on the same time, client2 try to read file2 to
230 for (seg = 0; seg < nr_segs; seg++) {
231 const struct iovec *iv = &iov[seg];
232 rc = lt_get_mmap_locks(tree, (unsigned long)iv->iov_base,
238 while ((node = lt_least_node(tree))) {
239 struct inode *inode = node->lt_inode;
240 rc = ll_extent_lock(tree->lt_fd, inode,
241 ll_i2info(inode)->lli_smd, node->lt_mode,
242 &node->lt_policy, &node->lt_lockh,
247 rb_erase(&node->lt_node, &tree->lt_root);
248 list_add_tail(&node->lt_locked_item, &tree->lt_locked_list);
252 ll_tree_unlock(tree);
256 int ll_tree_lock(struct ll_lock_tree *tree,
257 struct ll_lock_tree_node *first_node,
258 const char *buf, size_t count, int ast_flags)
260 struct iovec local_iov = { .iov_base = (void __user *)buf,
263 return ll_tree_lock_iov(tree, first_node, &local_iov, 1, ast_flags);
266 static ldlm_mode_t mode_from_vma(struct vm_area_struct *vma)
268 /* we only want to hold PW locks if the mmap() can generate
269 * writes back to the file and that only happens in shared
271 if ((vma->vm_flags & VM_SHARED) && (vma->vm_flags & VM_WRITE))
276 static void policy_from_vma(ldlm_policy_data_t *policy,
277 struct vm_area_struct *vma, unsigned long addr,
280 policy->l_extent.start = ((addr - vma->vm_start) & CFS_PAGE_MASK) +
281 ((__u64)vma->vm_pgoff << CFS_PAGE_SHIFT);
282 policy->l_extent.end = (policy->l_extent.start + count - 1) |
286 static struct vm_area_struct * our_vma(unsigned long addr, size_t count)
288 struct mm_struct *mm = current->mm;
289 struct vm_area_struct *vma, *ret = NULL;
292 /* No MM (e.g. NFS)? No vmas too. */
296 spin_lock(&mm->page_table_lock);
297 for(vma = find_vma(mm, addr);
298 vma != NULL && vma->vm_start < (addr + count); vma = vma->vm_next) {
299 if (vma->vm_ops && vma->vm_ops->nopage == ll_nopage &&
300 vma->vm_flags & VM_SHARED) {
305 spin_unlock(&mm->page_table_lock);
309 int ll_region_mapped(unsigned long addr, size_t count)
311 return !!our_vma(addr, count);
314 int lt_get_mmap_locks(struct ll_lock_tree *tree,
315 unsigned long addr, size_t count)
317 struct vm_area_struct *vma;
318 struct ll_lock_tree_node *node;
319 ldlm_policy_data_t policy;
326 /* we need to look up vmas on page aligned addresses */
327 count += addr & (~CFS_PAGE_MASK);
328 addr &= CFS_PAGE_MASK;
330 while ((vma = our_vma(addr, count)) != NULL) {
331 LASSERT(vma->vm_file);
333 inode = vma->vm_file->f_dentry->d_inode;
334 policy_from_vma(&policy, vma, addr, count);
335 node = ll_node_from_inode(inode, policy.l_extent.start,
339 CERROR("not enough mem for lock_tree_node!\n");
342 lt_insert(tree, node);
344 if (vma->vm_end - addr >= count)
346 count -= vma->vm_end - addr;
352 * Page fault handler.
354 * \param vma - is virtiual area struct related to page fault
355 * \param address - address when hit fault
356 * \param type - of fault
358 * \return allocated and filled page for address
359 * \retval NOPAGE_SIGBUS if page not exist on this address
360 * \retval NOPAGE_OOM not have memory for allocate new page
362 struct page *ll_nopage(struct vm_area_struct *vma, unsigned long address,
365 struct file *filp = vma->vm_file;
366 struct ll_file_data *fd = LUSTRE_FPRIVATE(filp);
367 struct inode *inode = filp->f_dentry->d_inode;
368 struct lustre_handle lockh = { 0 };
369 ldlm_policy_data_t policy;
371 struct page *page = NULL;
372 struct ll_inode_info *lli = ll_i2info(inode);
373 struct lov_stripe_md *lsm;
375 __u64 kms, old_mtime;
376 unsigned long pgoff, size, rand_read, seq_read;
380 if (lli->lli_smd == NULL) {
381 CERROR("No lsm on fault?\n");
382 RETURN(NOPAGE_SIGBUS);
385 ll_clear_file_contended(inode);
387 /* start and end the lock on the first and last bytes in the page */
388 policy_from_vma(&policy, vma, address, CFS_PAGE_SIZE);
390 CDEBUG(D_MMAP, "nopage vma %p inode %lu, locking ["LPU64", "LPU64"]\n",
391 vma, inode->i_ino, policy.l_extent.start, policy.l_extent.end);
393 mode = mode_from_vma(vma);
394 old_mtime = LTIME_S(inode->i_mtime);
397 rc = ll_extent_lock(fd, inode, lsm, mode, &policy,
398 &lockh, LDLM_FL_CBPENDING | LDLM_FL_NO_LRU);
400 RETURN(NOPAGE_SIGBUS);
402 if (vma->vm_flags & VM_EXEC && LTIME_S(inode->i_mtime) != old_mtime)
403 CWARN("binary changed. inode %lu\n", inode->i_ino);
405 lov_stripe_lock(lsm);
406 inode_init_lvb(inode, &lvb);
407 obd_merge_lvb(ll_i2obdexp(inode), lsm, &lvb, 1);
410 pgoff = ((address - vma->vm_start) >> CFS_PAGE_SHIFT) + vma->vm_pgoff;
411 size = (kms + CFS_PAGE_SIZE - 1) >> CFS_PAGE_SHIFT;
414 lov_stripe_unlock(lsm);
415 ll_glimpse_size(inode, LDLM_FL_BLOCK_GRANTED);
417 /* XXX change inode size without ll_inode_size_lock() held!
418 * there is a race condition with truncate path. (see
420 /* XXX i_size_write() is not used because it is not safe to
421 * take the ll_inode_size_lock() due to a potential lock
422 * inversion (bug 6077). And since it's not safe to use
423 * i_size_write() without a covering mutex we do the
424 * assignment directly. It is not critical that the
425 * size be correct. */
426 /* NOTE: region is within kms and, hence, within real file size (A).
427 * We need to increase i_size to cover the read region so that
428 * generic_file_read() will do its job, but that doesn't mean
429 * the kms size is _correct_, it is only the _minimum_ size.
430 * If someone does a stat they will get the correct size which
431 * will always be >= the kms value here. b=11081 */
432 if (i_size_read(inode) < kms) {
434 CDEBUG(D_INODE, "ino=%lu, updating i_size %llu\n",
435 inode->i_ino, i_size_read(inode));
437 lov_stripe_unlock(lsm);
440 /* If mapping is writeable, adjust kms to cover this page,
441 * but do not extend kms beyond actual file size.
442 * policy.l_extent.end is set to the end of the page by policy_from_vma
444 lov_stripe_lock(lsm);
446 obd_adjust_kms(ll_i2obdexp(inode), lsm,
447 min_t(loff_t, policy.l_extent.end + 1,
448 i_size_read(inode)), 0);
449 lov_stripe_unlock(lsm);
451 /* disable VM_SEQ_READ and use VM_RAND_READ to make sure that
452 * the kernel will not read other pages not covered by ldlm in
453 * filemap_nopage. we do our readahead in ll_readpage.
455 rand_read = vma->vm_flags & VM_RAND_READ;
456 seq_read = vma->vm_flags & VM_SEQ_READ;
457 vma->vm_flags &= ~ VM_SEQ_READ;
458 vma->vm_flags |= VM_RAND_READ;
460 page = filemap_nopage(vma, address, type);
461 if (page != NOPAGE_SIGBUS && page != NOPAGE_OOM)
462 LL_CDEBUG_PAGE(D_PAGE, page, "got addr %lu type %lx\n", address,
465 CDEBUG(D_PAGE, "got addr %lu type %lx - SIGBUS\n", address,
468 vma->vm_flags &= ~VM_RAND_READ;
469 vma->vm_flags |= (rand_read | seq_read);
471 ll_extent_unlock(fd, inode, ll_i2info(inode)->lli_smd, mode, &lockh);
475 /* To avoid cancel the locks covering mmapped region for lock cache pressure,
476 * we track the mapped vma count by lli_mmap_cnt.
477 * ll_vm_open(): when first vma is linked, split locks from lru.
478 * ll_vm_close(): when last vma is unlinked, join all this file's locks to lru.
480 * XXX we don't check the if the region of vma/lock for performance.
482 static void ll_vm_open(struct vm_area_struct * vma)
484 struct inode *inode = vma->vm_file->f_dentry->d_inode;
485 struct ll_inode_info *lli = ll_i2info(inode);
488 LASSERT(vma->vm_file);
490 spin_lock(&lli->lli_lock);
491 LASSERT(atomic_read(&lli->lli_mmap_cnt) >= 0);
493 atomic_inc(&lli->lli_mmap_cnt);
494 if (atomic_read(&lli->lli_mmap_cnt) == 1) {
495 struct lov_stripe_md *lsm = lli->lli_smd;
496 struct ll_sb_info *sbi = ll_i2sbi(inode);
499 spin_unlock(&lli->lli_lock);
503 count = obd_join_lru(sbi->ll_osc_exp, lsm, 0);
504 VMA_DEBUG(vma, "split %d unused locks from lru\n", count);
506 spin_unlock(&lli->lli_lock);
511 static void ll_vm_close(struct vm_area_struct *vma)
513 struct inode *inode = vma->vm_file->f_dentry->d_inode;
514 struct ll_inode_info *lli = ll_i2info(inode);
517 LASSERT(vma->vm_file);
519 spin_lock(&lli->lli_lock);
520 LASSERT(atomic_read(&lli->lli_mmap_cnt) > 0);
522 atomic_dec(&lli->lli_mmap_cnt);
523 if (atomic_read(&lli->lli_mmap_cnt) == 0) {
524 struct lov_stripe_md *lsm = lli->lli_smd;
525 struct ll_sb_info *sbi = ll_i2sbi(inode);
528 spin_unlock(&lli->lli_lock);
532 count = obd_join_lru(sbi->ll_osc_exp, lsm, 1);
533 VMA_DEBUG(vma, "join %d unused locks to lru\n", count);
535 spin_unlock(&lli->lli_lock);
539 #ifndef HAVE_FILEMAP_POPULATE
540 static int (*filemap_populate)(struct vm_area_struct * area, unsigned long address, unsigned long len, pgprot_t prot, unsigned long pgoff, int nonblock);
542 static int ll_populate(struct vm_area_struct *area, unsigned long address,
543 unsigned long len, pgprot_t prot, unsigned long pgoff,
549 /* always set nonblock as true to avoid page read ahead */
550 rc = filemap_populate(area, address, len, prot, pgoff, 1);
554 /* return the user space pointer that maps to a file offset via a vma */
555 static inline unsigned long file_to_user(struct vm_area_struct *vma, __u64 byte)
557 return vma->vm_start + (byte - ((__u64)vma->vm_pgoff << CFS_PAGE_SHIFT));
561 /* XXX put nice comment here. talk about __free_pte -> dirty pages and
562 * nopage's reference passing to the pte */
563 int ll_teardown_mmaps(struct address_space *mapping, __u64 first, __u64 last)
568 LASSERTF(last > first, "last "LPU64" first "LPU64"\n", last, first);
569 if (mapping_mapped(mapping)) {
571 unmap_mapping_range(mapping, first + CFS_PAGE_SIZE - 1,
572 last - first + 1, 0);
578 static struct vm_operations_struct ll_file_vm_ops = {
581 .close = ll_vm_close,
582 .populate = ll_populate,
585 int ll_file_mmap(struct file * file, struct vm_area_struct * vma)
590 ll_stats_ops_tally(ll_i2sbi(file->f_dentry->d_inode), LPROC_LL_MAP, 1);
591 rc = generic_file_mmap(file, vma);
593 #if !defined(HAVE_FILEMAP_POPULATE) && \
594 (LINUX_VERSION_CODE >= KERNEL_VERSION(2,5,0))
595 if (!filemap_populate)
596 filemap_populate = vma->vm_ops->populate;
598 vma->vm_ops = &ll_file_vm_ops;
599 vma->vm_ops->open(vma);
600 /* update the inode's size and mtime */
601 rc = ll_glimpse_size(file->f_dentry->d_inode, 0);