Whamcloud - gitweb
Branch HEAD
[fs/lustre-release.git] / lustre / llite / llite_mmap.c
1 /* -*- mode: c; c-basic-offset: 8; indent-tabs-mode: nil; -*-
2  * vim:expandtab:shiftwidth=8:tabstop=8:
3  *
4  *  Copyright (c) 2001-2003 Cluster File Systems, Inc.
5  *
6  *   This file is part of Lustre, http://www.lustre.org.
7  *
8  *   Lustre is free software; you can redistribute it and/or
9  *   modify it under the terms of version 2 of the GNU General Public
10  *   License as published by the Free Software Foundation.
11  *
12  *   Lustre is distributed in the hope that it will be useful,
13  *   but WITHOUT ANY WARRANTY; without even the implied warranty of
14  *   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
15  *   GNU General Public License for more details.
16  *
17  *   You should have received a copy of the GNU General Public License
18  *   along with Lustre; if not, write to the Free Software
19  *   Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
20  */
21
22 #ifndef AUTOCONF_INCLUDED
23 #include <linux/config.h>
24 #endif
25 #include <linux/kernel.h>
26 #include <linux/mm.h>
27 #include <linux/string.h>
28 #include <linux/stat.h>
29 #include <linux/errno.h>
30 #include <linux/smp_lock.h>
31 #include <linux/unistd.h>
32 #include <linux/version.h>
33 #include <asm/system.h>
34 #include <asm/uaccess.h>
35
36 #include <linux/fs.h>
37 #include <linux/stat.h>
38 #include <asm/uaccess.h>
39 #ifdef HAVE_SEGMENT_H
40 # include <asm/segment.h>
41 #endif
42 #include <linux/mm.h>
43 #include <linux/pagemap.h>
44 #include <linux/smp_lock.h>
45
46 #define DEBUG_SUBSYSTEM S_LLITE
47
48 //#include <lustre_mdc.h>
49 #include <lustre_lite.h>
50 #include "llite_internal.h"
51 #include <linux/lustre_compat25.h>
52
53 #define VMA_DEBUG(vma, fmt, arg...)                                     \
54         CDEBUG(D_MMAP, "vma(%p) start(%ld) end(%ld) pgoff(%ld) inode(%p) "   \
55                "ino(%lu) iname(%s): " fmt, vma, vma->vm_start, vma->vm_end,  \
56                vma->vm_pgoff, vma->vm_file->f_dentry->d_inode,               \
57                vma->vm_file->f_dentry->d_inode->i_ino,                       \
58                vma->vm_file->f_dentry->d_iname, ## arg);                     \
59
60
61 struct ll_lock_tree_node {
62         rb_node_t               lt_node;
63         struct list_head        lt_locked_item;
64         __u64                   lt_oid;
65         ldlm_policy_data_t      lt_policy;
66         struct lustre_handle    lt_lockh;
67         ldlm_mode_t             lt_mode;
68         struct inode           *lt_inode;
69 };
70
71 int lt_get_mmap_locks(struct ll_lock_tree *tree,
72                       unsigned long addr, size_t count);
73
74 struct page *ll_nopage(struct vm_area_struct *vma, unsigned long address,
75                        int *type);
76
77 struct ll_lock_tree_node * ll_node_from_inode(struct inode *inode, __u64 start,
78                                               __u64 end, ldlm_mode_t mode)
79 {
80         struct ll_lock_tree_node *node;
81
82         OBD_ALLOC(node, sizeof(*node));
83         if (node == NULL)
84                 RETURN(ERR_PTR(-ENOMEM));
85
86         node->lt_inode = inode;
87         node->lt_oid = ll_i2info(inode)->lli_smd->lsm_object_id;
88         node->lt_policy.l_extent.start = start;
89         node->lt_policy.l_extent.end = end;
90         memset(&node->lt_lockh, 0, sizeof(node->lt_lockh));
91         INIT_LIST_HEAD(&node->lt_locked_item);
92         node->lt_mode = mode;
93
94         return node;
95 }
96
97 int lt_compare(struct ll_lock_tree_node *one, struct ll_lock_tree_node *two)
98 {
99         /* To avoid multiple fs deadlock */
100         if (one->lt_inode->i_sb->s_dev < two->lt_inode->i_sb->s_dev)
101                 return -1;
102         if (one->lt_inode->i_sb->s_dev > two->lt_inode->i_sb->s_dev)
103                 return 1;
104
105         if (one->lt_oid < two->lt_oid)
106                 return -1;
107         if (one->lt_oid > two->lt_oid)
108                 return 1;
109
110         if (one->lt_policy.l_extent.end < two->lt_policy.l_extent.start)
111                 return -1;
112         if (one->lt_policy.l_extent.start > two->lt_policy.l_extent.end)
113                 return 1;
114
115         return 0; /* they are the same object and overlap */
116 }
117
118 static void lt_merge(struct ll_lock_tree_node *dst,
119                      struct ll_lock_tree_node *src)
120 {
121         dst->lt_policy.l_extent.start = min(dst->lt_policy.l_extent.start,
122                                             src->lt_policy.l_extent.start);
123         dst->lt_policy.l_extent.end = max(dst->lt_policy.l_extent.end,
124                                           src->lt_policy.l_extent.end);
125
126         /* XXX could be a real call to the dlm to find superset modes */
127         if (src->lt_mode == LCK_PW && dst->lt_mode != LCK_PW)
128                 dst->lt_mode = LCK_PW;
129 }
130
131 static void lt_insert(struct ll_lock_tree *tree,
132                       struct ll_lock_tree_node *node)
133 {
134         struct ll_lock_tree_node *walk;
135         rb_node_t **p, *parent;
136         ENTRY;
137
138 restart:
139         p = &tree->lt_root.rb_node;
140         parent = NULL;
141         while (*p) {
142                 parent = *p;
143                 walk = rb_entry(parent, struct ll_lock_tree_node, lt_node);
144                 switch (lt_compare(node, walk)) {
145                 case -1:
146                         p = &(*p)->rb_left;
147                         break;
148                 case 1:
149                         p = &(*p)->rb_right;
150                         break;
151                 case 0:
152                         lt_merge(node, walk);
153                         rb_erase(&walk->lt_node, &tree->lt_root);
154                         OBD_FREE(walk, sizeof(*walk));
155                         goto restart;
156                         break;
157                 default:
158                         LBUG();
159                         break;
160                 }
161         }
162         rb_link_node(&node->lt_node, parent, p);
163         rb_insert_color(&node->lt_node, &tree->lt_root);
164         EXIT;
165 }
166
167 static struct ll_lock_tree_node *lt_least_node(struct ll_lock_tree *tree)
168 {
169         rb_node_t *rbnode;
170         struct ll_lock_tree_node *node = NULL;
171
172         for ( rbnode = tree->lt_root.rb_node; rbnode != NULL;
173               rbnode = rbnode->rb_left) {
174                 if (rbnode->rb_left == NULL) {
175                         node = rb_entry(rbnode, struct ll_lock_tree_node,
176                                         lt_node);
177                         break;
178                 }
179         }
180         RETURN(node);
181 }
182
183 int ll_tree_unlock(struct ll_lock_tree *tree)
184 {
185         struct ll_lock_tree_node *node;
186         struct list_head *pos, *n;
187         struct inode *inode;
188         int rc = 0;
189         ENTRY;
190
191         list_for_each_safe(pos, n, &tree->lt_locked_list) {
192                 node = list_entry(pos, struct ll_lock_tree_node,
193                                   lt_locked_item);
194
195                 inode = node->lt_inode;
196                 rc = ll_extent_unlock(tree->lt_fd, inode,
197                                       ll_i2info(inode)->lli_smd, node->lt_mode,
198                                       &node->lt_lockh);
199                 if (rc != 0) {
200                         /* XXX better message */
201                         CERROR("couldn't unlock %d\n", rc);
202                 }
203                 list_del(&node->lt_locked_item);
204                 OBD_FREE(node, sizeof(*node));
205         }
206
207         while ((node = lt_least_node(tree))) {
208                 rb_erase(&node->lt_node, &tree->lt_root);
209                 OBD_FREE(node, sizeof(*node));
210         }
211
212         RETURN(rc);
213 }
214
215 int ll_tree_lock(struct ll_lock_tree *tree,
216                  struct ll_lock_tree_node *first_node,
217                  const char *buf, size_t count, int ast_flags)
218 {
219         struct ll_lock_tree_node *node;
220         int rc = 0;
221         ENTRY;
222
223         tree->lt_root.rb_node = NULL;
224         INIT_LIST_HEAD(&tree->lt_locked_list);
225         if (first_node != NULL)
226                 lt_insert(tree, first_node);
227
228         /* To avoid such subtle deadlock case: client1 try to read file1 to
229          * mmapped file2, on the same time, client2 try to read file2 to
230          * mmapped file1.*/
231         rc = lt_get_mmap_locks(tree, (unsigned long)buf, count);
232         if (rc)
233                 GOTO(out, rc);
234
235         while ((node = lt_least_node(tree))) {
236                 struct inode *inode = node->lt_inode;
237                 rc = ll_extent_lock(tree->lt_fd, inode,
238                                     ll_i2info(inode)->lli_smd, node->lt_mode,
239                                     &node->lt_policy, &node->lt_lockh,
240                                     ast_flags);
241                 if (rc != 0)
242                         GOTO(out, rc);
243
244                 rb_erase(&node->lt_node, &tree->lt_root);
245                 list_add_tail(&node->lt_locked_item, &tree->lt_locked_list);
246         }
247         RETURN(rc);
248 out:
249         ll_tree_unlock(tree);
250         RETURN(rc);
251 }
252
253 static ldlm_mode_t mode_from_vma(struct vm_area_struct *vma)
254 {
255         /* we only want to hold PW locks if the mmap() can generate
256          * writes back to the file and that only happens in shared
257          * writable vmas */
258         if ((vma->vm_flags & VM_SHARED) && (vma->vm_flags & VM_WRITE))
259                 return LCK_PW;
260         return LCK_PR;
261 }
262
263 static void policy_from_vma(ldlm_policy_data_t *policy,
264                             struct vm_area_struct *vma, unsigned long addr,
265                             size_t count)
266 {
267         policy->l_extent.start = ((addr - vma->vm_start) & CFS_PAGE_MASK) +
268                                  (vma->vm_pgoff << CFS_PAGE_SHIFT);
269         policy->l_extent.end = (policy->l_extent.start + count - 1) |
270                                ~CFS_PAGE_MASK;
271 }
272
273 static struct vm_area_struct * our_vma(unsigned long addr, size_t count)
274 {
275         struct mm_struct *mm = current->mm;
276         struct vm_area_struct *vma, *ret = NULL;
277         ENTRY;
278
279         /* No MM (e.g. NFS)? No vmas too. */
280         if (!mm)
281                 RETURN(NULL);
282
283         spin_lock(&mm->page_table_lock);
284         for(vma = find_vma(mm, addr);
285             vma != NULL && vma->vm_start < (addr + count); vma = vma->vm_next) {
286                 if (vma->vm_ops && vma->vm_ops->nopage == ll_nopage &&
287                     vma->vm_flags & VM_SHARED) {
288                         ret = vma;
289                         break;
290                 }
291         }
292         spin_unlock(&mm->page_table_lock);
293         RETURN(ret);
294 }
295
296 int lt_get_mmap_locks(struct ll_lock_tree *tree,
297                       unsigned long addr, size_t count)
298 {
299         struct vm_area_struct *vma;
300         struct ll_lock_tree_node *node;
301         ldlm_policy_data_t policy;
302         struct inode *inode;
303         ENTRY;
304
305         if (count == 0)
306                 RETURN(0);
307
308         /* we need to look up vmas on page aligned addresses */
309         count += addr & (~CFS_PAGE_MASK);
310         addr &= CFS_PAGE_MASK;
311
312         while ((vma = our_vma(addr, count)) != NULL) {
313                 LASSERT(vma->vm_file);
314
315                 inode = vma->vm_file->f_dentry->d_inode;
316                 policy_from_vma(&policy, vma, addr, count);
317                 node = ll_node_from_inode(inode, policy.l_extent.start,
318                                           policy.l_extent.end,
319                                           mode_from_vma(vma));
320                 if (IS_ERR(node)) {
321                         CERROR("not enough mem for lock_tree_node!\n");
322                         RETURN(-ENOMEM);
323                 }
324                 lt_insert(tree, node);
325
326                 if (vma->vm_end - addr >= count)
327                         break;
328                 count -= vma->vm_end - addr;
329                 addr = vma->vm_end;
330         }
331         RETURN(0);
332 }
333
334 /* FIXME: there is a pagefault race goes as follow (only 2.4):
335  * 1. A user process on node A accesses a portion of a mapped file,
336  *    resulting in a page fault.  The pagefault handler invokes the
337  *    ll_nopage function, which reads the page into memory.
338  * 2. A user process on node B writes to the same portion of the file
339  *    (either via mmap or write()), that cause node A to cancel the
340  *    lock and truncate the page.
341  * 3. Node A then executes the rest of do_no_page(), entering the
342  *    now-invalid page into the PTEs.
343  *
344  * Make the whole do_no_page as a hook to cover both the page cache
345  * and page mapping installing with dlm lock would eliminate this race.
346  *
347  * In 2.6, the truncate_count of address_space can cover this race.
348  */
349 struct page *ll_nopage(struct vm_area_struct *vma, unsigned long address,
350                        int *type)
351 {
352         struct file *filp = vma->vm_file;
353         struct ll_file_data *fd = LUSTRE_FPRIVATE(filp);
354         struct inode *inode = filp->f_dentry->d_inode;
355         struct lustre_handle lockh = { 0 };
356         ldlm_policy_data_t policy;
357         ldlm_mode_t mode;
358         struct page *page = NULL;
359         struct ll_inode_info *lli = ll_i2info(inode);
360         struct lov_stripe_md *lsm;
361         struct ost_lvb lvb;
362         __u64 kms, old_mtime;
363         unsigned long pgoff, size, rand_read, seq_read;
364         int rc = 0;
365         ENTRY;
366
367         if (lli->lli_smd == NULL) {
368                 CERROR("No lsm on fault?\n");
369                 RETURN(NULL);
370         }
371
372         /* start and end the lock on the first and last bytes in the page */
373         policy_from_vma(&policy, vma, address, CFS_PAGE_SIZE);
374
375         CDEBUG(D_MMAP, "nopage vma %p inode %lu, locking ["LPU64", "LPU64"]\n",
376                vma, inode->i_ino, policy.l_extent.start, policy.l_extent.end);
377
378         mode = mode_from_vma(vma);
379         old_mtime = LTIME_S(inode->i_mtime);
380
381         lsm = lli->lli_smd;
382         rc = ll_extent_lock(fd, inode, lsm, mode, &policy,
383                             &lockh, LDLM_FL_CBPENDING | LDLM_FL_NO_LRU);
384         if (rc != 0)
385                 RETURN(NULL);
386
387         if (vma->vm_flags & VM_EXEC && LTIME_S(inode->i_mtime) != old_mtime)
388                 CWARN("binary changed. inode %lu\n", inode->i_ino);
389
390         lov_stripe_lock(lsm);
391         inode_init_lvb(inode, &lvb);
392         obd_merge_lvb(ll_i2dtexp(inode), lsm, &lvb, 1);
393         kms = lvb.lvb_size;
394
395         pgoff = ((address - vma->vm_start) >> CFS_PAGE_SHIFT) + vma->vm_pgoff;
396         size = (kms + CFS_PAGE_SIZE - 1) >> CFS_PAGE_SHIFT;
397
398         if (pgoff >= size) {
399                 lov_stripe_unlock(lsm);
400                 ll_glimpse_size(inode, LDLM_FL_BLOCK_GRANTED);
401         } else {
402                 /* XXX change inode size without ll_inode_size_lock() held!
403                  *     there is a race condition with truncate path. (see
404                  *     ll_extent_lock) */
405                 /* XXX i_size_write() is not used because it is not safe to
406                  *     take the ll_inode_size_lock() due to a potential lock
407                  *     inversion (bug 6077).  And since it's not safe to use
408                  *     i_size_write() without a covering mutex we do the
409                  *     assignment directly.  It is not critical that the
410                  *     size be correct. */
411                 /* region is within kms and, hence, within real file size (A).
412                  * We need to increase i_size to cover the read region so that
413                  * generic_file_read() will do its job, but that doesn't mean
414                  * the kms size is _correct_, it is only the _minimum_ size.
415                  * If someone does a stat they will get the correct size which
416                  * will always be >= the kms value here.  b=11081 */
417                 if (i_size_read(inode) < kms) {
418                         inode->i_size = kms;
419                         CDEBUG(D_INODE, "ino=%lu, updating i_size %llu\n",
420                                inode->i_ino, i_size_read(inode));
421                 }
422                 lov_stripe_unlock(lsm);
423         }
424
425         /* If mapping is writeable, adjust kms to cover this page,
426          * but do not extend kms beyond actual file size.
427          * policy.l_extent.end is set to the end of the page by policy_from_vma
428          * bug 10919 */
429         lov_stripe_lock(lsm);
430         if (mode == LCK_PW)
431                 obd_adjust_kms(ll_i2dtexp(inode), lsm,
432                                min_t(loff_t, policy.l_extent.end + 1,
433                                i_size_read(inode)), 0);
434         lov_stripe_unlock(lsm);
435
436         /* disable VM_SEQ_READ and use VM_RAND_READ to make sure that
437          * the kernel will not read other pages not covered by ldlm in
438          * filemap_nopage. we do our readahead in ll_readpage.
439          */
440         rand_read = vma->vm_flags & VM_RAND_READ;
441         seq_read = vma->vm_flags & VM_SEQ_READ;
442         vma->vm_flags &= ~ VM_SEQ_READ;
443         vma->vm_flags |= VM_RAND_READ;
444
445         page = filemap_nopage(vma, address, type);
446         LL_CDEBUG_PAGE(D_PAGE, page, "got addr %lu type %lx\n", address,
447                        (long)type);
448         vma->vm_flags &= ~VM_RAND_READ;
449         vma->vm_flags |= (rand_read | seq_read);
450
451         ll_extent_unlock(fd, inode, ll_i2info(inode)->lli_smd, mode, &lockh);
452         RETURN(page);
453 }
454
455 /* To avoid cancel the locks covering mmapped region for lock cache pressure,
456  * we track the mapped vma count by lli_mmap_cnt.
457  * ll_vm_open():  when first vma is linked, split locks from lru.
458  * ll_vm_close(): when last vma is unlinked, join all this file's locks to lru.
459  *
460  * XXX we don't check the if the region of vma/lock for performance.
461  */
462 static void ll_vm_open(struct vm_area_struct * vma)
463 {
464         struct inode *inode = vma->vm_file->f_dentry->d_inode;
465         struct ll_inode_info *lli = ll_i2info(inode);
466         ENTRY;
467
468         LASSERT(vma->vm_file);
469
470         spin_lock(&lli->lli_lock);
471         LASSERT(atomic_read(&lli->lli_mmap_cnt) >= 0);
472
473         atomic_inc(&lli->lli_mmap_cnt);
474         if (atomic_read(&lli->lli_mmap_cnt) == 1) {
475                 struct lov_stripe_md *lsm = lli->lli_smd;
476                 struct ll_sb_info *sbi = ll_i2sbi(inode);
477                 int count;
478
479                 spin_unlock(&lli->lli_lock);
480
481                 if (!lsm)
482                         return;
483                 count = obd_join_lru(sbi->ll_dt_exp, lsm, 0);
484                 VMA_DEBUG(vma, "split %d unused locks from lru\n", count);
485         } else {
486                 spin_unlock(&lli->lli_lock);
487         }
488
489 }
490
491 static void ll_vm_close(struct vm_area_struct *vma)
492 {
493         struct inode *inode = vma->vm_file->f_dentry->d_inode;
494         struct ll_inode_info *lli = ll_i2info(inode);
495         ENTRY;
496
497         LASSERT(vma->vm_file);
498
499         spin_lock(&lli->lli_lock);
500         LASSERT(atomic_read(&lli->lli_mmap_cnt) > 0);
501
502         atomic_dec(&lli->lli_mmap_cnt);
503         if (atomic_read(&lli->lli_mmap_cnt) == 0) {
504                 struct lov_stripe_md *lsm = lli->lli_smd;
505                 struct ll_sb_info *sbi = ll_i2sbi(inode);
506                 int count;
507
508                 spin_unlock(&lli->lli_lock);
509
510                 if (!lsm)
511                         return;
512                 count = obd_join_lru(sbi->ll_dt_exp, lsm, 1);
513                 VMA_DEBUG(vma, "join %d unused locks to lru\n", count);
514         } else {
515                 spin_unlock(&lli->lli_lock);
516         }
517 }
518
519 #ifndef HAVE_FILEMAP_POPULATE
520 static int (*filemap_populate)(struct vm_area_struct * area, unsigned long address, unsigned long len, pgprot_t prot, unsigned long pgoff, int nonblock);
521 #endif
522 static int ll_populate(struct vm_area_struct *area, unsigned long address,
523                        unsigned long len, pgprot_t prot, unsigned long pgoff,
524                        int nonblock)
525 {
526         int rc = 0;
527         ENTRY;
528
529         /* always set nonblock as true to avoid page read ahead */
530         rc = filemap_populate(area, address, len, prot, pgoff, 1);
531         RETURN(rc);
532 }
533
534 /* return the user space pointer that maps to a file offset via a vma */
535 static inline unsigned long file_to_user(struct vm_area_struct *vma, __u64 byte)
536 {
537         return vma->vm_start + (byte - ((__u64)vma->vm_pgoff << CFS_PAGE_SHIFT));
538
539 }
540
541 /* XXX put nice comment here.  talk about __free_pte -> dirty pages and
542  * nopage's reference passing to the pte */
543 int ll_teardown_mmaps(struct address_space *mapping, __u64 first, __u64 last)
544 {
545         int rc = -ENOENT;
546         ENTRY;
547
548         LASSERTF(last > first, "last "LPU64" first "LPU64"\n", last, first);
549         if (mapping_mapped(mapping)) {
550                 rc = 0;
551                 unmap_mapping_range(mapping, first + CFS_PAGE_SIZE - 1,
552                                     last - first + 1, 0);
553         }
554
555         RETURN(rc);
556 }
557
558 static struct vm_operations_struct ll_file_vm_ops = {
559         .nopage         = ll_nopage,
560         .open           = ll_vm_open,
561         .close          = ll_vm_close,
562         .populate       = ll_populate,
563 };
564
565 int ll_file_mmap(struct file * file, struct vm_area_struct * vma)
566 {
567         int rc;
568         ENTRY;
569
570         ll_stats_ops_tally(ll_i2sbi(file->f_dentry->d_inode), LPROC_LL_MAP, 1);
571         rc = generic_file_mmap(file, vma);
572         if (rc == 0) {
573 #if !defined(HAVE_FILEMAP_POPULATE)
574                 if (!filemap_populate)
575                         filemap_populate = vma->vm_ops->populate;
576 #endif
577                 vma->vm_ops = &ll_file_vm_ops;
578                 vma->vm_ops->open(vma);
579                 /* update the inode's size and mtime */
580                 rc = ll_glimpse_size(file->f_dentry->d_inode, 0);
581         }
582
583         RETURN(rc);
584 }