Whamcloud - gitweb
b=16098
[fs/lustre-release.git] / lustre / llite / llite_mmap.c
1 /* -*- mode: c; c-basic-offset: 8; indent-tabs-mode: nil; -*-
2  * vim:expandtab:shiftwidth=8:tabstop=8:
3  *
4  * GPL HEADER START
5  *
6  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
7  *
8  * This program is free software; you can redistribute it and/or modify
9  * it under the terms of the GNU General Public License version 2 only,
10  * as published by the Free Software Foundation.
11  *
12  * This program is distributed in the hope that it will be useful, but
13  * WITHOUT ANY WARRANTY; without even the implied warranty of
14  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
15  * General Public License version 2 for more details (a copy is included
16  * in the LICENSE file that accompanied this code).
17  *
18  * You should have received a copy of the GNU General Public License
19  * version 2 along with this program; If not, see [sun.com URL with a
20  * copy of GPLv2].
21  *
22  * Please contact Sun Microsystems, Inc., 4150 Network Circle, Santa Clara,
23  * CA 95054 USA or visit www.sun.com if you need additional information or
24  * have any questions.
25  *
26  * GPL HEADER END
27  */
28 /*
29  * Copyright  2008 Sun Microsystems, Inc. All rights reserved
30  * Use is subject to license terms.
31  */
32 /*
33  * This file is part of Lustre, http://www.lustre.org/
34  * Lustre is a trademark of Sun Microsystems, Inc.
35  */
36
37 #ifndef AUTOCONF_INCLUDED
38 #include <linux/config.h>
39 #endif
40 #include <linux/kernel.h>
41 #include <linux/mm.h>
42 #include <linux/string.h>
43 #include <linux/stat.h>
44 #include <linux/errno.h>
45 #include <linux/smp_lock.h>
46 #include <linux/unistd.h>
47 #include <linux/version.h>
48 #include <asm/system.h>
49 #include <asm/uaccess.h>
50
51 #include <linux/fs.h>
52 #include <linux/stat.h>
53 #include <asm/uaccess.h>
54 #include <linux/mm.h>
55 #include <linux/pagemap.h>
56 #include <linux/smp_lock.h>
57
58 #define DEBUG_SUBSYSTEM S_LLITE
59
60 //#include <lustre_mdc.h>
61 #include <lustre_lite.h>
62 #include "llite_internal.h"
63 #include <linux/lustre_compat25.h>
64
65 #define VMA_DEBUG(vma, fmt, arg...)                                     \
66         CDEBUG(D_MMAP, "vma(%p) start(%ld) end(%ld) pgoff(%ld) inode(%p) "   \
67                "ino(%lu) iname(%s): " fmt, vma, vma->vm_start, vma->vm_end,  \
68                vma->vm_pgoff, vma->vm_file->f_dentry->d_inode,               \
69                vma->vm_file->f_dentry->d_inode->i_ino,                       \
70                vma->vm_file->f_dentry->d_iname, ## arg);                     \
71
72
73 struct ll_lock_tree_node {
74         rb_node_t               lt_node;
75         struct list_head        lt_locked_item;
76         __u64                   lt_oid;
77         ldlm_policy_data_t      lt_policy;
78         struct lustre_handle    lt_lockh;
79         ldlm_mode_t             lt_mode;
80         struct inode           *lt_inode;
81 };
82
83 int lt_get_mmap_locks(struct ll_lock_tree *tree,
84                       unsigned long addr, size_t count);
85
86 struct page *ll_nopage(struct vm_area_struct *vma, unsigned long address,
87                        int *type);
88
89 struct ll_lock_tree_node * ll_node_from_inode(struct inode *inode, __u64 start,
90                                               __u64 end, ldlm_mode_t mode)
91 {
92         struct ll_lock_tree_node *node;
93
94         OBD_ALLOC(node, sizeof(*node));
95         if (node == NULL)
96                 RETURN(ERR_PTR(-ENOMEM));
97
98         node->lt_inode = inode;
99         node->lt_oid = ll_i2info(inode)->lli_smd->lsm_object_id;
100         node->lt_policy.l_extent.start = start;
101         node->lt_policy.l_extent.end = end;
102         memset(&node->lt_lockh, 0, sizeof(node->lt_lockh));
103         INIT_LIST_HEAD(&node->lt_locked_item);
104         node->lt_mode = mode;
105
106         return node;
107 }
108
109 int lt_compare(struct ll_lock_tree_node *one, struct ll_lock_tree_node *two)
110 {
111         /* To avoid multiple fs deadlock */
112         if (one->lt_inode->i_sb->s_dev < two->lt_inode->i_sb->s_dev)
113                 return -1;
114         if (one->lt_inode->i_sb->s_dev > two->lt_inode->i_sb->s_dev)
115                 return 1;
116
117         if (one->lt_oid < two->lt_oid)
118                 return -1;
119         if (one->lt_oid > two->lt_oid)
120                 return 1;
121
122         if (one->lt_policy.l_extent.end < two->lt_policy.l_extent.start)
123                 return -1;
124         if (one->lt_policy.l_extent.start > two->lt_policy.l_extent.end)
125                 return 1;
126
127         return 0; /* they are the same object and overlap */
128 }
129
130 static void lt_merge(struct ll_lock_tree_node *dst,
131                      struct ll_lock_tree_node *src)
132 {
133         dst->lt_policy.l_extent.start = min(dst->lt_policy.l_extent.start,
134                                             src->lt_policy.l_extent.start);
135         dst->lt_policy.l_extent.end = max(dst->lt_policy.l_extent.end,
136                                           src->lt_policy.l_extent.end);
137
138         /* XXX could be a real call to the dlm to find superset modes */
139         if (src->lt_mode == LCK_PW && dst->lt_mode != LCK_PW)
140                 dst->lt_mode = LCK_PW;
141 }
142
143 static void lt_insert(struct ll_lock_tree *tree,
144                       struct ll_lock_tree_node *node)
145 {
146         struct ll_lock_tree_node *walk;
147         rb_node_t **p, *parent;
148         ENTRY;
149
150 restart:
151         p = &tree->lt_root.rb_node;
152         parent = NULL;
153         while (*p) {
154                 parent = *p;
155                 walk = rb_entry(parent, struct ll_lock_tree_node, lt_node);
156                 switch (lt_compare(node, walk)) {
157                 case -1:
158                         p = &(*p)->rb_left;
159                         break;
160                 case 1:
161                         p = &(*p)->rb_right;
162                         break;
163                 case 0:
164                         lt_merge(node, walk);
165                         rb_erase(&walk->lt_node, &tree->lt_root);
166                         OBD_FREE(walk, sizeof(*walk));
167                         goto restart;
168                         break;
169                 default:
170                         LBUG();
171                         break;
172                 }
173         }
174         rb_link_node(&node->lt_node, parent, p);
175         rb_insert_color(&node->lt_node, &tree->lt_root);
176         EXIT;
177 }
178
179 static struct ll_lock_tree_node *lt_least_node(struct ll_lock_tree *tree)
180 {
181         rb_node_t *rbnode;
182         struct ll_lock_tree_node *node = NULL;
183
184         for ( rbnode = tree->lt_root.rb_node; rbnode != NULL;
185               rbnode = rbnode->rb_left) {
186                 if (rbnode->rb_left == NULL) {
187                         node = rb_entry(rbnode, struct ll_lock_tree_node,
188                                         lt_node);
189                         break;
190                 }
191         }
192         RETURN(node);
193 }
194
195 int ll_tree_unlock(struct ll_lock_tree *tree)
196 {
197         struct ll_lock_tree_node *node;
198         struct list_head *pos, *n;
199         struct inode *inode;
200         int rc = 0;
201         ENTRY;
202
203         list_for_each_safe(pos, n, &tree->lt_locked_list) {
204                 node = list_entry(pos, struct ll_lock_tree_node,
205                                   lt_locked_item);
206
207                 inode = node->lt_inode;
208                 rc = ll_extent_unlock(tree->lt_fd, inode,
209                                       ll_i2info(inode)->lli_smd, node->lt_mode,
210                                       &node->lt_lockh);
211                 if (rc != 0) {
212                         /* XXX better message */
213                         CERROR("couldn't unlock %d\n", rc);
214                 }
215                 list_del(&node->lt_locked_item);
216                 OBD_FREE(node, sizeof(*node));
217         }
218
219         while ((node = lt_least_node(tree))) {
220                 rb_erase(&node->lt_node, &tree->lt_root);
221                 OBD_FREE(node, sizeof(*node));
222         }
223
224         RETURN(rc);
225 }
226
227 int ll_tree_lock(struct ll_lock_tree *tree,
228                  struct ll_lock_tree_node *first_node,
229                  const char *buf, size_t count, int ast_flags)
230 {
231         struct ll_lock_tree_node *node;
232         int rc = 0;
233         ENTRY;
234
235         tree->lt_root.rb_node = NULL;
236         INIT_LIST_HEAD(&tree->lt_locked_list);
237         if (first_node != NULL)
238                 lt_insert(tree, first_node);
239
240         /* To avoid such subtle deadlock case: client1 try to read file1 to
241          * mmapped file2, on the same time, client2 try to read file2 to
242          * mmapped file1.*/
243         rc = lt_get_mmap_locks(tree, (unsigned long)buf, count);
244         if (rc)
245                 GOTO(out, rc);
246
247         while ((node = lt_least_node(tree))) {
248                 struct inode *inode = node->lt_inode;
249                 rc = ll_extent_lock(tree->lt_fd, inode,
250                                     ll_i2info(inode)->lli_smd, node->lt_mode,
251                                     &node->lt_policy, &node->lt_lockh,
252                                     ast_flags);
253                 if (rc != 0)
254                         GOTO(out, rc);
255
256                 rb_erase(&node->lt_node, &tree->lt_root);
257                 list_add_tail(&node->lt_locked_item, &tree->lt_locked_list);
258         }
259         RETURN(rc);
260 out:
261         ll_tree_unlock(tree);
262         RETURN(rc);
263 }
264
265 static ldlm_mode_t mode_from_vma(struct vm_area_struct *vma)
266 {
267         /* we only want to hold PW locks if the mmap() can generate
268          * writes back to the file and that only happens in shared
269          * writable vmas */
270         if ((vma->vm_flags & VM_SHARED) && (vma->vm_flags & VM_WRITE))
271                 return LCK_PW;
272         return LCK_PR;
273 }
274
275 static void policy_from_vma(ldlm_policy_data_t *policy,
276                             struct vm_area_struct *vma, unsigned long addr,
277                             size_t count)
278 {
279         policy->l_extent.start = ((addr - vma->vm_start) & CFS_PAGE_MASK) +
280                                  (vma->vm_pgoff << CFS_PAGE_SHIFT);
281         policy->l_extent.end = (policy->l_extent.start + count - 1) |
282                                ~CFS_PAGE_MASK;
283 }
284
285 static struct vm_area_struct * our_vma(unsigned long addr, size_t count)
286 {
287         struct mm_struct *mm = current->mm;
288         struct vm_area_struct *vma, *ret = NULL;
289         ENTRY;
290
291         /* No MM (e.g. NFS)? No vmas too. */
292         if (!mm)
293                 RETURN(NULL);
294
295         spin_lock(&mm->page_table_lock);
296         for(vma = find_vma(mm, addr);
297             vma != NULL && vma->vm_start < (addr + count); vma = vma->vm_next) {
298                 if (vma->vm_ops && vma->vm_ops->nopage == ll_nopage &&
299                     vma->vm_flags & VM_SHARED) {
300                         ret = vma;
301                         break;
302                 }
303         }
304         spin_unlock(&mm->page_table_lock);
305         RETURN(ret);
306 }
307
308 int ll_region_mapped(unsigned long addr, size_t count)
309 {
310         return !!our_vma(addr, count);
311 }
312
313 int lt_get_mmap_locks(struct ll_lock_tree *tree,
314                       unsigned long addr, size_t count)
315 {
316         struct vm_area_struct *vma;
317         struct ll_lock_tree_node *node;
318         ldlm_policy_data_t policy;
319         struct inode *inode;
320         ENTRY;
321
322         if (count == 0)
323                 RETURN(0);
324
325         /* we need to look up vmas on page aligned addresses */
326         count += addr & (~CFS_PAGE_MASK);
327         addr &= CFS_PAGE_MASK;
328
329         while ((vma = our_vma(addr, count)) != NULL) {
330                 LASSERT(vma->vm_file);
331
332                 inode = vma->vm_file->f_dentry->d_inode;
333                 policy_from_vma(&policy, vma, addr, count);
334                 node = ll_node_from_inode(inode, policy.l_extent.start,
335                                           policy.l_extent.end,
336                                           mode_from_vma(vma));
337                 if (IS_ERR(node)) {
338                         CERROR("not enough mem for lock_tree_node!\n");
339                         RETURN(-ENOMEM);
340                 }
341                 lt_insert(tree, node);
342
343                 if (vma->vm_end - addr >= count)
344                         break;
345                 count -= vma->vm_end - addr;
346                 addr = vma->vm_end;
347         }
348         RETURN(0);
349 }
350
351 /**
352  * Page fault handler.
353  *
354  * \param vma - is virtiual area struct related to page fault
355  * \param address - address when hit fault
356  * \param type - of fault
357  *
358  * \return allocated and filled page for address
359  * \retval NOPAGE_SIGBUS if page not exist on this address
360  * \retval NOPAGE_OOM not have memory for allocate new page
361  */
362 struct page *ll_nopage(struct vm_area_struct *vma, unsigned long address,
363                        int *type)
364 {
365         struct file *filp = vma->vm_file;
366         struct ll_file_data *fd = LUSTRE_FPRIVATE(filp);
367         struct inode *inode = filp->f_dentry->d_inode;
368         struct lustre_handle lockh = { 0 };
369         ldlm_policy_data_t policy;
370         ldlm_mode_t mode;
371         struct page *page = NULL;
372         struct ll_inode_info *lli = ll_i2info(inode);
373         struct lov_stripe_md *lsm;
374         struct ost_lvb lvb;
375         __u64 kms, old_mtime;
376         unsigned long pgoff, size, rand_read, seq_read;
377         int rc = 0;
378         ENTRY;
379
380         if (lli->lli_smd == NULL) {
381                 CERROR("No lsm on fault?\n");
382                 RETURN(NULL);
383         }
384
385         ll_clear_file_contended(inode);
386
387         /* start and end the lock on the first and last bytes in the page */
388         policy_from_vma(&policy, vma, address, CFS_PAGE_SIZE);
389
390         CDEBUG(D_MMAP, "nopage vma %p inode %lu, locking ["LPU64", "LPU64"]\n",
391                vma, inode->i_ino, policy.l_extent.start, policy.l_extent.end);
392
393         mode = mode_from_vma(vma);
394         old_mtime = LTIME_S(inode->i_mtime);
395
396         lsm = lli->lli_smd;
397         rc = ll_extent_lock(fd, inode, lsm, mode, &policy,
398                             &lockh, LDLM_FL_CBPENDING | LDLM_FL_NO_LRU);
399         if (rc != 0)
400                 RETURN(NULL);
401
402         if (vma->vm_flags & VM_EXEC && LTIME_S(inode->i_mtime) != old_mtime)
403                 CWARN("binary changed. inode %lu\n", inode->i_ino);
404
405         lov_stripe_lock(lsm);
406         inode_init_lvb(inode, &lvb);
407         obd_merge_lvb(ll_i2dtexp(inode), lsm, &lvb, 1);
408         kms = lvb.lvb_size;
409
410         pgoff = ((address - vma->vm_start) >> CFS_PAGE_SHIFT) + vma->vm_pgoff;
411         size = (kms + CFS_PAGE_SIZE - 1) >> CFS_PAGE_SHIFT;
412
413         if (pgoff >= size) {
414                 lov_stripe_unlock(lsm);
415                 ll_glimpse_size(inode, LDLM_FL_BLOCK_GRANTED);
416         } else {
417                 /* XXX change inode size without ll_inode_size_lock() held!
418                  *     there is a race condition with truncate path. (see
419                  *     ll_extent_lock) */
420                 /* XXX i_size_write() is not used because it is not safe to
421                  *     take the ll_inode_size_lock() due to a potential lock
422                  *     inversion (bug 6077).  And since it's not safe to use
423                  *     i_size_write() without a covering mutex we do the
424                  *     assignment directly.  It is not critical that the
425                  *     size be correct. */
426                 /* region is within kms and, hence, within real file size (A).
427                  * We need to increase i_size to cover the read region so that
428                  * generic_file_read() will do its job, but that doesn't mean
429                  * the kms size is _correct_, it is only the _minimum_ size.
430                  * If someone does a stat they will get the correct size which
431                  * will always be >= the kms value here.  b=11081 */
432                 if (i_size_read(inode) < kms) {
433                         inode->i_size = kms;
434                         CDEBUG(D_INODE, "ino=%lu, updating i_size %llu\n",
435                                inode->i_ino, i_size_read(inode));
436                 }
437                 lov_stripe_unlock(lsm);
438         }
439
440         /* If mapping is writeable, adjust kms to cover this page,
441          * but do not extend kms beyond actual file size.
442          * policy.l_extent.end is set to the end of the page by policy_from_vma
443          * bug 10919 */
444         lov_stripe_lock(lsm);
445         if (mode == LCK_PW)
446                 obd_adjust_kms(ll_i2dtexp(inode), lsm,
447                                min_t(loff_t, policy.l_extent.end + 1,
448                                i_size_read(inode)), 0);
449         lov_stripe_unlock(lsm);
450
451         /* disable VM_SEQ_READ and use VM_RAND_READ to make sure that
452          * the kernel will not read other pages not covered by ldlm in
453          * filemap_nopage. we do our readahead in ll_readpage.
454          */
455         rand_read = vma->vm_flags & VM_RAND_READ;
456         seq_read = vma->vm_flags & VM_SEQ_READ;
457         vma->vm_flags &= ~ VM_SEQ_READ;
458         vma->vm_flags |= VM_RAND_READ;
459
460         page = filemap_nopage(vma, address, type);
461         if (page != NOPAGE_SIGBUS && page != NOPAGE_OOM)
462                 LL_CDEBUG_PAGE(D_PAGE, page, "got addr %lu type %lx\n", address,
463                                (long)type);
464         else
465                 CDEBUG(D_PAGE, "got addr %lu type %lx - SIGBUS\n",  address,
466                                (long)type);
467
468         vma->vm_flags &= ~VM_RAND_READ;
469         vma->vm_flags |= (rand_read | seq_read);
470
471         ll_extent_unlock(fd, inode, ll_i2info(inode)->lli_smd, mode, &lockh);
472         RETURN(page);
473 }
474
475 /* To avoid cancel the locks covering mmapped region for lock cache pressure,
476  * we track the mapped vma count by lli_mmap_cnt.
477  * ll_vm_open():  when first vma is linked, split locks from lru.
478  * ll_vm_close(): when last vma is unlinked, join all this file's locks to lru.
479  *
480  * XXX we don't check the if the region of vma/lock for performance.
481  */
482 static void ll_vm_open(struct vm_area_struct * vma)
483 {
484         struct inode *inode = vma->vm_file->f_dentry->d_inode;
485         struct ll_inode_info *lli = ll_i2info(inode);
486         ENTRY;
487
488         LASSERT(vma->vm_file);
489
490         spin_lock(&lli->lli_lock);
491         LASSERT(atomic_read(&lli->lli_mmap_cnt) >= 0);
492
493         atomic_inc(&lli->lli_mmap_cnt);
494         if (atomic_read(&lli->lli_mmap_cnt) == 1) {
495                 struct lov_stripe_md *lsm = lli->lli_smd;
496                 struct ll_sb_info *sbi = ll_i2sbi(inode);
497                 int count;
498
499                 spin_unlock(&lli->lli_lock);
500
501                 if (!lsm)
502                         return;
503                 count = obd_join_lru(sbi->ll_dt_exp, lsm, 0);
504                 VMA_DEBUG(vma, "split %d unused locks from lru\n", count);
505         } else {
506                 spin_unlock(&lli->lli_lock);
507         }
508
509 }
510
511 static void ll_vm_close(struct vm_area_struct *vma)
512 {
513         struct inode *inode = vma->vm_file->f_dentry->d_inode;
514         struct ll_inode_info *lli = ll_i2info(inode);
515         ENTRY;
516
517         LASSERT(vma->vm_file);
518
519         spin_lock(&lli->lli_lock);
520         LASSERT(atomic_read(&lli->lli_mmap_cnt) > 0);
521
522         atomic_dec(&lli->lli_mmap_cnt);
523         if (atomic_read(&lli->lli_mmap_cnt) == 0) {
524                 struct lov_stripe_md *lsm = lli->lli_smd;
525                 struct ll_sb_info *sbi = ll_i2sbi(inode);
526                 int count;
527
528                 spin_unlock(&lli->lli_lock);
529
530                 if (!lsm)
531                         return;
532                 count = obd_join_lru(sbi->ll_dt_exp, lsm, 1);
533                 VMA_DEBUG(vma, "join %d unused locks to lru\n", count);
534         } else {
535                 spin_unlock(&lli->lli_lock);
536         }
537 }
538
539 #ifndef HAVE_FILEMAP_POPULATE
540 static int (*filemap_populate)(struct vm_area_struct * area, unsigned long address, unsigned long len, pgprot_t prot, unsigned long pgoff, int nonblock);
541 #endif
542 static int ll_populate(struct vm_area_struct *area, unsigned long address,
543                        unsigned long len, pgprot_t prot, unsigned long pgoff,
544                        int nonblock)
545 {
546         int rc = 0;
547         ENTRY;
548
549         /* always set nonblock as true to avoid page read ahead */
550         rc = filemap_populate(area, address, len, prot, pgoff, 1);
551         RETURN(rc);
552 }
553
554 /* return the user space pointer that maps to a file offset via a vma */
555 static inline unsigned long file_to_user(struct vm_area_struct *vma, __u64 byte)
556 {
557         return vma->vm_start + (byte - ((__u64)vma->vm_pgoff << CFS_PAGE_SHIFT));
558
559 }
560
561 /* XXX put nice comment here.  talk about __free_pte -> dirty pages and
562  * nopage's reference passing to the pte */
563 int ll_teardown_mmaps(struct address_space *mapping, __u64 first, __u64 last)
564 {
565         int rc = -ENOENT;
566         ENTRY;
567
568         LASSERTF(last > first, "last "LPU64" first "LPU64"\n", last, first);
569         if (mapping_mapped(mapping)) {
570                 rc = 0;
571                 unmap_mapping_range(mapping, first + CFS_PAGE_SIZE - 1,
572                                     last - first + 1, 0);
573         }
574
575         RETURN(rc);
576 }
577
578 static struct vm_operations_struct ll_file_vm_ops = {
579         .nopage         = ll_nopage,
580         .open           = ll_vm_open,
581         .close          = ll_vm_close,
582         .populate       = ll_populate,
583 };
584
585 int ll_file_mmap(struct file * file, struct vm_area_struct * vma)
586 {
587         int rc;
588         ENTRY;
589
590         ll_stats_ops_tally(ll_i2sbi(file->f_dentry->d_inode), LPROC_LL_MAP, 1);
591         rc = generic_file_mmap(file, vma);
592         if (rc == 0) {
593 #if !defined(HAVE_FILEMAP_POPULATE)
594                 if (!filemap_populate)
595                         filemap_populate = vma->vm_ops->populate;
596 #endif
597                 vma->vm_ops = &ll_file_vm_ops;
598                 vma->vm_ops->open(vma);
599                 /* update the inode's size and mtime */
600                 rc = ll_glimpse_size(file->f_dentry->d_inode, 0);
601         }
602
603         RETURN(rc);
604 }