Whamcloud - gitweb
Branch: b1_4
[fs/lustre-release.git] / lustre / llite / llite_mmap.c
1 /* -*- mode: c; c-basic-offset: 8; indent-tabs-mode: nil; -*-
2  * vim:expandtab:shiftwidth=8:tabstop=8:
3  *
4  *  Copyright (c) 2001-2003 Cluster File Systems, Inc.
5  *
6  *   This file is part of Lustre, http://www.lustre.org.
7  *
8  *   Lustre is free software; you can redistribute it and/or
9  *   modify it under the terms of version 2 of the GNU General Public
10  *   License as published by the Free Software Foundation.
11  *
12  *   Lustre is distributed in the hope that it will be useful,
13  *   but WITHOUT ANY WARRANTY; without even the implied warranty of
14  *   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
15  *   GNU General Public License for more details.
16  *
17  *   You should have received a copy of the GNU General Public License
18  *   along with Lustre; if not, write to the Free Software
19  *   Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
20  */
21
22 #include <linux/config.h>
23 #include <linux/kernel.h>
24 #include <linux/mm.h>
25 #include <linux/string.h>
26 #include <linux/stat.h>
27 #include <linux/errno.h>
28 #include <linux/smp_lock.h>
29 #include <linux/unistd.h>
30 #include <linux/version.h>
31 #include <asm/system.h>
32 #include <asm/uaccess.h>
33
34 #include <linux/fs.h>
35 #include <linux/stat.h>
36 #include <asm/uaccess.h>
37 #include <asm/segment.h>
38 #include <linux/mm.h>
39 #include <linux/pagemap.h>
40 #include <linux/smp_lock.h>
41 #if (LINUX_VERSION_CODE < KERNEL_VERSION(2,5,0))
42 #include <linux/iobuf.h>
43 #endif
44
45
46 #define DEBUG_SUBSYSTEM S_LLITE
47
48 #include <linux/lustre_mds.h>
49 #include <linux/lustre_lite.h>
50 #include "llite_internal.h"
51 #include <linux/lustre_compat25.h>
52
53 #define VMA_DEBUG(vma, fmt, arg...)                                     \
54         CDEBUG(D_MMAP, "vma(%p) start(%ld) end(%ld) pgoff(%ld) inode(%p) "   \
55                "ino(%lu) iname(%s): " fmt, vma, vma->vm_start, vma->vm_end,  \
56                vma->vm_pgoff, vma->vm_file->f_dentry->d_inode,               \
57                vma->vm_file->f_dentry->d_inode->i_ino,                       \
58                vma->vm_file->f_dentry->d_iname, ## arg);                     \
59
60
61 struct ll_lock_tree_node {
62         rb_node_t               lt_node;
63         struct list_head        lt_locked_item;
64         __u64                   lt_oid;
65         ldlm_policy_data_t      lt_policy;
66         struct lustre_handle    lt_lockh;
67         ldlm_mode_t             lt_mode;
68         struct inode           *lt_inode;
69 };
70
71 __u64 lov_merge_size(struct lov_stripe_md *lsm, int kms);
72 int lt_get_mmap_locks(struct ll_lock_tree *tree,
73                       unsigned long addr, size_t count);
74
75 #if (LINUX_VERSION_CODE >= KERNEL_VERSION(2,5,0))
76 struct page *ll_nopage(struct vm_area_struct *vma, unsigned long address,
77                        int *type);
78 #else
79
80 struct page *ll_nopage(struct vm_area_struct *vma, unsigned long address,
81                        int unused);
82 #endif
83
84 struct ll_lock_tree_node * ll_node_from_inode(struct inode *inode, __u64 start,
85                                               __u64 end, ldlm_mode_t mode)
86 {
87         struct ll_lock_tree_node *node;
88
89         OBD_ALLOC(node, sizeof(*node));
90         if (node == NULL)
91                 RETURN(ERR_PTR(-ENOMEM));
92
93         node->lt_inode = inode;
94         node->lt_oid = ll_i2info(inode)->lli_smd->lsm_object_id;
95         node->lt_policy.l_extent.start = start;
96         node->lt_policy.l_extent.end = end;
97         memset(&node->lt_lockh, 0, sizeof(node->lt_lockh));
98         INIT_LIST_HEAD(&node->lt_locked_item);
99         node->lt_mode = mode;
100
101         return node;
102 }
103
104 int lt_compare(struct ll_lock_tree_node *one, struct ll_lock_tree_node *two)
105 {
106         /* To avoid multiple fs deadlock */
107         if (one->lt_inode->i_sb->s_dev < two->lt_inode->i_sb->s_dev)
108                 return -1;
109         if (one->lt_inode->i_sb->s_dev > two->lt_inode->i_sb->s_dev)
110                 return 1;
111
112         if (one->lt_oid < two->lt_oid)
113                 return -1;
114         if (one->lt_oid > two->lt_oid)
115                 return 1;
116
117         if (one->lt_policy.l_extent.end < two->lt_policy.l_extent.start)
118                 return -1;
119         if (one->lt_policy.l_extent.start > two->lt_policy.l_extent.end)
120                 return 1;
121
122         return 0; /* they are the same object and overlap */
123 }
124
125 static void lt_merge(struct ll_lock_tree_node *dst,
126                      struct ll_lock_tree_node *src)
127 {
128         dst->lt_policy.l_extent.start = min(dst->lt_policy.l_extent.start,
129                                             src->lt_policy.l_extent.start);
130         dst->lt_policy.l_extent.end = max(dst->lt_policy.l_extent.end,
131                                           src->lt_policy.l_extent.end);
132
133         /* XXX could be a real call to the dlm to find superset modes */
134         if (src->lt_mode == LCK_PW && dst->lt_mode != LCK_PW)
135                 dst->lt_mode = LCK_PW;
136 }
137
138 static void lt_insert(struct ll_lock_tree *tree,
139                       struct ll_lock_tree_node *node)
140 {
141         struct ll_lock_tree_node *walk;
142         rb_node_t **p, *parent;
143         ENTRY;
144
145 restart:
146         p = &tree->lt_root.rb_node;
147         parent = NULL;
148         while (*p) {
149                 parent = *p;
150                 walk = rb_entry(parent, struct ll_lock_tree_node, lt_node);
151                 switch (lt_compare(node, walk)) {
152                 case -1:
153                         p = &(*p)->rb_left;
154                         break;
155                 case 1:
156                         p = &(*p)->rb_right;
157                         break;
158                 case 0:
159                         lt_merge(node, walk);
160                         rb_erase(&walk->lt_node, &tree->lt_root);
161                         OBD_FREE(walk, sizeof(*walk));
162                         goto restart;
163                         break;
164                 default:
165                         LBUG();
166                         break;
167                 }
168         }
169         rb_link_node(&node->lt_node, parent, p);
170         rb_insert_color(&node->lt_node, &tree->lt_root);
171         EXIT;
172 }
173
174 static struct ll_lock_tree_node *lt_least_node(struct ll_lock_tree *tree)
175 {
176         rb_node_t *rbnode;
177         struct ll_lock_tree_node *node = NULL;
178
179         for ( rbnode = tree->lt_root.rb_node; rbnode != NULL;
180               rbnode = rbnode->rb_left) {
181                 if (rbnode->rb_left == NULL) {
182                         node = rb_entry(rbnode, struct ll_lock_tree_node,
183                                         lt_node);
184                         break;
185                 }
186         }
187         RETURN(node);
188 }
189
190 int ll_tree_unlock(struct ll_lock_tree *tree)
191 {
192         struct ll_lock_tree_node *node;
193         struct list_head *pos, *n;
194         struct inode *inode;
195         int rc = 0;
196         ENTRY;
197
198         list_for_each_safe(pos, n, &tree->lt_locked_list) {
199                 node = list_entry(pos, struct ll_lock_tree_node,
200                                   lt_locked_item);
201
202                 inode = node->lt_inode;
203                 rc = ll_extent_unlock(tree->lt_fd, inode,
204                                       ll_i2info(inode)->lli_smd, node->lt_mode,
205                                       &node->lt_lockh);
206                 if (rc != 0) {
207                         /* XXX better message */
208                         CERROR("couldn't unlock %d\n", rc);
209                 }
210                 list_del(&node->lt_locked_item);
211                 OBD_FREE(node, sizeof(*node));
212         }
213
214         while ((node = lt_least_node(tree))) {
215                 rb_erase(&node->lt_node, &tree->lt_root);
216                 OBD_FREE(node, sizeof(*node));
217         }
218
219         RETURN(rc);
220 }
221
222 int ll_tree_lock(struct ll_lock_tree *tree,
223                  struct ll_lock_tree_node *first_node,
224                  const char *buf, size_t count, int ast_flags)
225 {
226         struct ll_lock_tree_node *node;
227         int rc = 0;
228         ENTRY;
229
230         tree->lt_root.rb_node = NULL;
231         INIT_LIST_HEAD(&tree->lt_locked_list);
232         if (first_node != NULL)
233                 lt_insert(tree, first_node);
234
235         /* To avoid such subtle deadlock case: client1 try to read file1 to
236          * mmapped file2, on the same time, client2 try to read file2 to
237          * mmapped file1.*/
238         rc = lt_get_mmap_locks(tree, (unsigned long)buf, count);
239         if (rc)
240                 GOTO(out, rc);
241
242         while ((node = lt_least_node(tree))) {
243                 struct inode *inode = node->lt_inode;
244                 rc = ll_extent_lock(tree->lt_fd, inode,
245                                     ll_i2info(inode)->lli_smd, node->lt_mode,
246                                     &node->lt_policy, &node->lt_lockh,
247                                     ast_flags);
248                 if (rc != 0)
249                         GOTO(out, rc);
250
251                 rb_erase(&node->lt_node, &tree->lt_root);
252                 list_add_tail(&node->lt_locked_item, &tree->lt_locked_list);
253         }
254         RETURN(rc);
255 out:
256         ll_tree_unlock(tree);
257         RETURN(rc);
258 }
259
260 static ldlm_mode_t mode_from_vma(struct vm_area_struct *vma)
261 {
262         /* we only want to hold PW locks if the mmap() can generate
263          * writes back to the file and that only happens in shared
264          * writable vmas */
265         if ((vma->vm_flags & VM_SHARED) && (vma->vm_flags & VM_WRITE))
266                 return LCK_PW;
267         return LCK_PR;
268 }
269
270 static void policy_from_vma(ldlm_policy_data_t *policy,
271                             struct vm_area_struct *vma, unsigned long addr,
272                             size_t count)
273 {
274         policy->l_extent.start = ((addr - vma->vm_start) & PAGE_CACHE_MASK) +
275                                  (vma->vm_pgoff << PAGE_CACHE_SHIFT);
276         policy->l_extent.end = (policy->l_extent.start + count - 1) |
277                                (PAGE_CACHE_SIZE - 1);
278 }
279
280 static struct vm_area_struct * our_vma(unsigned long addr, size_t count)
281 {
282         struct mm_struct *mm = current->mm;
283         struct vm_area_struct *vma, *ret = NULL;
284         ENTRY;
285
286         /* No MM (e.g. NFS)? No vmas too. */
287         if (!mm)
288                 RETURN(NULL);
289
290         spin_lock(&mm->page_table_lock);
291         for(vma = find_vma(mm, addr);
292             vma != NULL && vma->vm_start < (addr + count); vma = vma->vm_next) {
293                 if (vma->vm_ops && vma->vm_ops->nopage == ll_nopage &&
294                     vma->vm_flags & VM_SHARED) {
295                         ret = vma;
296                         break;
297                 }
298         }
299         spin_unlock(&mm->page_table_lock);
300         RETURN(ret);
301 }
302
303 int lt_get_mmap_locks(struct ll_lock_tree *tree,
304                       unsigned long addr, size_t count)
305 {
306         struct vm_area_struct *vma;
307         struct ll_lock_tree_node *node;
308         ldlm_policy_data_t policy;
309         struct inode *inode;
310         ENTRY;
311
312         if (count == 0)
313                 RETURN(0);
314
315         /* we need to look up vmas on page aligned addresses */
316         count += addr & (PAGE_SIZE - 1);
317         addr &= PAGE_MASK;
318
319         while ((vma = our_vma(addr, count)) != NULL) {
320                 LASSERT(vma->vm_file);
321
322                 inode = vma->vm_file->f_dentry->d_inode;
323                 policy_from_vma(&policy, vma, addr, count);
324                 node = ll_node_from_inode(inode, policy.l_extent.start,
325                                           policy.l_extent.end,
326                                           mode_from_vma(vma));
327                 if (IS_ERR(node)) {
328                         CERROR("not enough mem for lock_tree_node!\n");
329                         RETURN(-ENOMEM);
330                 }
331                 lt_insert(tree, node);
332
333                 if (vma->vm_end - addr >= count)
334                         break;
335                 count -= vma->vm_end - addr;
336                 addr = vma->vm_end;
337         }
338         RETURN(0);
339 }
340
341 /* FIXME: there is a pagefault race goes as follow (only 2.4):
342  * 1. A user process on node A accesses a portion of a mapped file,
343  *    resulting in a page fault.  The pagefault handler invokes the
344  *    ll_nopage function, which reads the page into memory.
345  * 2. A user process on node B writes to the same portion of the file
346  *    (either via mmap or write()), that cause node A to cancel the
347  *    lock and truncate the page.
348  * 3. Node A then executes the rest of do_no_page(), entering the
349  *    now-invalid page into the PTEs.
350  *
351  * Make the whole do_no_page as a hook to cover both the page cache
352  * and page mapping installing with dlm lock would eliminate this race.
353  *
354  * In 2.6, the truncate_count of address_space can cover this race.
355  */
356 #if (LINUX_VERSION_CODE >= KERNEL_VERSION(2,5,0))
357 struct page *ll_nopage(struct vm_area_struct *vma, unsigned long address,
358                        int *type)
359 #else
360 struct page *ll_nopage(struct vm_area_struct *vma, unsigned long address,
361                        int type /* unused */)
362 #endif
363 {
364         struct file *filp = vma->vm_file;
365         struct ll_file_data *fd = LUSTRE_FPRIVATE(filp);
366         struct inode *inode = filp->f_dentry->d_inode;
367         struct lustre_handle lockh = { 0 };
368         ldlm_policy_data_t policy;
369         ldlm_mode_t mode;
370         struct page *page = NULL;
371         struct ll_inode_info *lli = ll_i2info(inode);
372         __u64 kms, old_mtime;
373         unsigned long pgoff, size, rand_read, seq_read;
374         int rc = 0;
375         ENTRY;
376
377         if (lli->lli_smd == NULL) {
378                 CERROR("No lsm on fault?\n");
379                 RETURN(NULL);
380         }
381
382         /* start and end the lock on the first and last bytes in the page */
383         policy_from_vma(&policy, vma, address, PAGE_CACHE_SIZE);
384
385         CDEBUG(D_MMAP, "nopage vma %p inode %lu, locking ["LPU64", "LPU64"]\n",
386                vma, inode->i_ino, policy.l_extent.start, policy.l_extent.end);
387
388         mode = mode_from_vma(vma);
389         old_mtime = LTIME_S(inode->i_mtime);
390
391         rc = ll_extent_lock(fd, inode, lli->lli_smd, mode, &policy,
392                             &lockh, LDLM_FL_CBPENDING | LDLM_FL_NO_LRU);
393         if (rc != 0)
394                 RETURN(NULL);
395
396         if (vma->vm_flags & VM_EXEC && LTIME_S(inode->i_mtime) != old_mtime)
397                 CWARN("binary changed. inode %lu\n", inode->i_ino);
398
399         /* XXX change inode size without i_sem hold! there is a race condition
400          *     with truncate path. (see ll_extent_lock) */
401         //down(&lli->lli_size_sem);
402         kms = lov_merge_size(lli->lli_smd, 1);
403         pgoff = ((address - vma->vm_start) >> PAGE_CACHE_SHIFT) + vma->vm_pgoff;
404         size = (kms + PAGE_CACHE_SIZE - 1) >> PAGE_CACHE_SHIFT;
405
406         if (pgoff >= size) {
407                 //up(&lli->lli_size_sem);
408                 ll_glimpse_size(inode);
409         } else {
410                 inode->i_size = kms;
411                 //up(&lli->lli_size_sem);
412         }
413
414         /* disable VM_SEQ_READ and use VM_RAND_READ to make sure that
415          * the kernel will not read other pages not covered by ldlm in
416          * filemap_nopage. we do our readahead in ll_readpage.
417          */
418         rand_read = vma->vm_flags & VM_RAND_READ;
419         seq_read = vma->vm_flags & VM_SEQ_READ;
420         vma->vm_flags &= ~ VM_SEQ_READ;
421         vma->vm_flags |= VM_RAND_READ;
422
423         page = filemap_nopage(vma, address, type);
424         LL_CDEBUG_PAGE(D_PAGE, page, "got addr %lu type %lx\n", address,
425                        (long)type);
426         vma->vm_flags &= ~VM_RAND_READ;
427         vma->vm_flags |= (rand_read | seq_read);
428
429         ll_extent_unlock(fd, inode, ll_i2info(inode)->lli_smd, mode, &lockh);
430         RETURN(page);
431 }
432
433 /* To avoid cancel the locks covering mmapped region for lock cache pressure,
434  * we track the mapped vma count by lli_mmap_cnt.
435  * ll_vm_open():  when first vma is linked, split locks from lru.
436  * ll_vm_close(): when last vma is unlinked, join all this file's locks to lru.
437  *
438  * XXX we don't check the if the region of vma/lock for performance.
439  */
440 static void ll_vm_open(struct vm_area_struct * vma)
441 {
442         struct inode *inode = vma->vm_file->f_dentry->d_inode;
443         struct ll_inode_info *lli = ll_i2info(inode);
444         ENTRY;
445
446         LASSERT(vma->vm_file);
447
448         spin_lock(&lli->lli_lock);
449         LASSERT(atomic_read(&lli->lli_mmap_cnt) >= 0);
450
451         atomic_inc(&lli->lli_mmap_cnt);
452         if (atomic_read(&lli->lli_mmap_cnt) == 1) {
453                 struct lov_stripe_md *lsm = lli->lli_smd;
454                 struct ll_sb_info *sbi = ll_i2sbi(inode);
455                 int count;
456
457                 spin_unlock(&lli->lli_lock);
458                 count = obd_join_lru(sbi->ll_osc_exp, lsm, 0);
459                 VMA_DEBUG(vma, "split %d unused locks from lru\n", count);
460         } else {
461                 spin_unlock(&lli->lli_lock);
462         }
463
464 }
465
466 static void ll_vm_close(struct vm_area_struct *vma)
467 {
468         struct inode *inode = vma->vm_file->f_dentry->d_inode;
469         struct ll_inode_info *lli = ll_i2info(inode);
470         ENTRY;
471
472         LASSERT(vma->vm_file);
473
474         spin_lock(&lli->lli_lock);
475         LASSERT(atomic_read(&lli->lli_mmap_cnt) > 0);
476
477         atomic_dec(&lli->lli_mmap_cnt);
478         if (atomic_read(&lli->lli_mmap_cnt) == 0) {
479                 struct lov_stripe_md *lsm = lli->lli_smd;
480                 struct ll_sb_info *sbi = ll_i2sbi(inode);
481                 int count;
482
483                 spin_unlock(&lli->lli_lock);
484                 count = obd_join_lru(sbi->ll_osc_exp, lsm, 1);
485                 VMA_DEBUG(vma, "join %d unused locks to lru\n", count);
486         } else {
487                 spin_unlock(&lli->lli_lock);
488         }
489 }
490
491 #if (LINUX_VERSION_CODE >= KERNEL_VERSION(2,5,0))
492 static int ll_populate(struct vm_area_struct *area, unsigned long address,
493                        unsigned long len, pgprot_t prot, unsigned long pgoff,
494                        int nonblock)
495 {
496         int rc = 0;
497         ENTRY;
498
499         /* always set nonblock as true to avoid page read ahead */
500         rc = filemap_populate(area, address, len, prot, pgoff, 1);
501         RETURN(rc);
502 }
503 #endif
504
505 /* return the user space pointer that maps to a file offset via a vma */
506 static inline unsigned long file_to_user(struct vm_area_struct *vma, __u64 byte)
507 {
508         return vma->vm_start + (byte - ((__u64)vma->vm_pgoff << PAGE_SHIFT));
509
510 }
511
512 #if (LINUX_VERSION_CODE < KERNEL_VERSION(2,5,0))
513 /* [first, last] are the byte offsets affected.
514  * vm_{start, end} are user addresses of the first byte of the mapping and
515  *      the next byte beyond it
516  * vm_pgoff is the page index of the first byte in the mapping */
517 static void teardown_vmas(struct vm_area_struct *vma, __u64 first,
518                           __u64 last)
519 {
520         unsigned long address, len;
521         for (; vma ; vma = vma->vm_next_share) {
522                 if (last >> PAGE_SHIFT < vma->vm_pgoff)
523                         continue;
524                 if (first >> PAGE_SHIFT >= (vma->vm_pgoff +
525                     ((vma->vm_end - vma->vm_start) >> PAGE_SHIFT)))
526                         continue;
527
528                 /* XXX in case of unmap the cow pages of a running file,
529                  * don't unmap these private writeable mapping here!
530                  * though that will break private mappping a little.
531                  *
532                  * the clean way is to check the mapping of every page
533                  * and just unmap the non-cow pages, just like
534                  * unmap_mapping_range() with even_cow=0 in kernel 2.6.
535                  */
536                 if (!(vma->vm_flags & VM_SHARED) &&
537                     (vma->vm_flags & VM_WRITE))
538                         continue;
539
540                 address = max((unsigned long)vma->vm_start,
541                               file_to_user(vma, first));
542                 len = min((unsigned long)vma->vm_end,
543                           file_to_user(vma, last) + 1) - address;
544
545                 VMA_DEBUG(vma, "zapping vma [first="LPU64" last="LPU64" "
546                           "address=%ld len=%ld]\n", first, last, address, len);
547                 LASSERT(len > 0);
548                 ll_zap_page_range(vma, address, len);
549         }
550 }
551 #endif
552
553 /* XXX put nice comment here.  talk about __free_pte -> dirty pages and
554  * nopage's reference passing to the pte */
555 int ll_teardown_mmaps(struct address_space *mapping, __u64 first, __u64 last)
556 {
557         int rc = -ENOENT;
558         ENTRY;
559
560         LASSERTF(last > first, "last "LPU64" first "LPU64"\n", last, first);
561 #if (LINUX_VERSION_CODE >= KERNEL_VERSION(2,5,0))
562         if (mapping_mapped(mapping)) {
563                 rc = 0;
564                 unmap_mapping_range(mapping, first + PAGE_SIZE - 1,
565                                     last - first + 1, 0);
566         }
567 #else
568         spin_lock(&mapping->i_shared_lock);
569         if (mapping->i_mmap != NULL) {
570                 rc = 0;
571                 teardown_vmas(mapping->i_mmap, first, last);
572         }
573         if (mapping->i_mmap_shared != NULL) {
574                 rc = 0;
575                 teardown_vmas(mapping->i_mmap_shared, first, last);
576         }
577         spin_unlock(&mapping->i_shared_lock);
578 #endif
579         RETURN(rc);
580 }
581
582 static struct vm_operations_struct ll_file_vm_ops = {
583         .nopage         = ll_nopage,
584         .open           = ll_vm_open,
585         .close          = ll_vm_close,
586 #if (LINUX_VERSION_CODE >= KERNEL_VERSION(2,5,0))
587         .populate       = ll_populate,
588 #endif
589 };
590
591 int ll_file_mmap(struct file * file, struct vm_area_struct * vma)
592 {
593         int rc;
594         ENTRY;
595
596         rc = generic_file_mmap(file, vma);
597         if (rc == 0) {
598                 vma->vm_ops = &ll_file_vm_ops;
599                 vma->vm_ops->open(vma);
600                 /* update the inode's size and mtime */
601                 rc = ll_glimpse_size(file->f_dentry->d_inode);
602         }
603
604         RETURN(rc);
605 }