lustre/llite/llite_mmap.c

   1 /* -*- mode: c; c-basic-offset: 8; indent-tabs-mode: nil; -*-
   2  * vim:expandtab:shiftwidth=8:tabstop=8:
   3  *
   4  *  Copyright (c) 2001-2003 Cluster File Systems, Inc.
   5  *
   6  *   This file is part of Lustre, http://www.lustre.org.
   7  *
   8  *   Lustre is free software; you can redistribute it and/or
   9  *   modify it under the terms of version 2 of the GNU General Public
  10  *   License as published by the Free Software Foundation.
  11  *
  12  *   Lustre is distributed in the hope that it will be useful,
  13  *   but WITHOUT ANY WARRANTY; without even the implied warranty of
  14  *   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  15  *   GNU General Public License for more details.
  16  *
  17  *   You should have received a copy of the GNU General Public License
  18  *   along with Lustre; if not, write to the Free Software
  19  *   Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
  20  */
  21
  22 #include <linux/config.h>
  23 #include <linux/kernel.h>
  24 #include <linux/mm.h>
  25 #include <linux/string.h>
  26 #include <linux/stat.h>
  27 #include <linux/errno.h>
  28 #include <linux/smp_lock.h>
  29 #include <linux/unistd.h>
  30 #include <linux/version.h>
  31 #include <asm/system.h>
  32 #include <asm/uaccess.h>
  33
  34 #include <linux/fs.h>
  35 #include <linux/stat.h>
  36 #include <asm/uaccess.h>
  37 #include <asm/segment.h>
  38 #include <linux/mm.h>
  39 #include <linux/pagemap.h>
  40 #include <linux/smp_lock.h>
  41 #if (LINUX_VERSION_CODE < KERNEL_VERSION(2,5,0))
  42 #include <linux/iobuf.h>
  43 #endif
  44
  45 #include <linux/pagevec.h>
  46
  47 #define DEBUG_SUBSYSTEM S_LLITE
  48
  49 #include <linux/lustre_mds.h>
  50 #include <linux/lustre_lite.h>
  51 #include <linux/lustre_audit.h>
  52 #include "llite_internal.h"
  53 #include <linux/lustre_compat25.h>
  54
  55 __u64 lov_merge_size(struct lov_stripe_md *lsm, int kms);
  56 int lt_get_mmap_locks(struct ll_lock_tree *tree, struct inode *inode,
  57                       unsigned long addr, size_t count);
  58 #if (LINUX_VERSION_CODE >= KERNEL_VERSION(2,5,0))
  59 struct page *ll_nopage(struct vm_area_struct *vma, unsigned long address,
  60                        int *type);
  61 #else
  62 struct page *ll_nopage(struct vm_area_struct *vma, unsigned long address,
  63                        int unused);
  64 #endif
  65
  66 struct ll_lock_tree_node * ll_node_from_inode(struct inode *inode, __u64 start,
  67                                               __u64 end, ldlm_mode_t mode)
  68 {
  69         struct ll_lock_tree_node *node;
  70
  71         OBD_ALLOC(node, sizeof(*node));
  72         if (node == NULL)
  73                 RETURN(ERR_PTR(-ENOMEM));
  74
  75         node->lt_oid = ll_i2info(inode)->lli_smd->lsm_object_id;
  76         node->lt_policy.l_extent.start = start;
  77         node->lt_policy.l_extent.end = end;
  78         memset(&node->lt_lockh, 0, sizeof(node->lt_lockh));
  79         INIT_LIST_HEAD(&node->lt_locked_item);
  80         node->lt_mode = mode;
  81
  82         return node;
  83 }
  84
  85 int lt_compare(struct ll_lock_tree_node *one, struct ll_lock_tree_node *two)
  86 {
  87         /* XXX remove this assert when we really want to use this function
  88          * to compare different file's region */
  89         LASSERT(one->lt_oid == two->lt_oid);
  90
  91         if ( one->lt_oid < two->lt_oid)
  92                 return -1;
  93         if ( one->lt_oid > two->lt_oid)
  94                 return 1;
  95
  96         if ( one->lt_policy.l_extent.end < two->lt_policy.l_extent.start )
  97                 return -1;
  98         if ( one->lt_policy.l_extent.start > two->lt_policy.l_extent.end )
  99                 return 1;
 100
 101         return 0; /* they are the same object and overlap */
 102 }
 103
 104 static void lt_merge(struct ll_lock_tree_node *dst,
 105                      struct ll_lock_tree_node *src)
 106 {
 107         dst->lt_policy.l_extent.start = min(dst->lt_policy.l_extent.start,
 108                                             src->lt_policy.l_extent.start);
 109         dst->lt_policy.l_extent.end = max(dst->lt_policy.l_extent.end,
 110                                           src->lt_policy.l_extent.end);
 111
 112         /* XXX could be a real call to the dlm to find superset modes */
 113         if (src->lt_mode == LCK_PW && dst->lt_mode != LCK_PW)
 114                 dst->lt_mode = LCK_PW;
 115 }
 116
 117 static void lt_insert(struct ll_lock_tree *tree,
 118                       struct ll_lock_tree_node *node)
 119 {
 120         struct ll_lock_tree_node *walk;
 121         rb_node_t **p, *parent;
 122         ENTRY;
 123
 124 restart:
 125         p = &tree->lt_root.rb_node;
 126         parent = NULL;
 127         while (*p) {
 128                 parent = *p;
 129                 walk = rb_entry(parent, struct ll_lock_tree_node, lt_node);
 130                 switch (lt_compare(node, walk)) {
 131                 case -1:
 132                         p = &(*p)->rb_left;
 133                         break;
 134                 case 1:
 135                         p = &(*p)->rb_right;
 136                         break;
 137                 case 0:
 138                         lt_merge(node, walk);
 139                         rb_erase(&walk->lt_node, &tree->lt_root);
 140                         OBD_FREE(walk, sizeof(*walk));
 141                         goto restart;
 142                         break;
 143                 default:
 144                         LBUG();
 145                         break;
 146                 }
 147         }
 148         rb_link_node(&node->lt_node, parent, p);
 149         rb_insert_color(&node->lt_node, &tree->lt_root);
 150         EXIT;
 151 }
 152
 153 static struct ll_lock_tree_node *lt_least_node(struct ll_lock_tree *tree)
 154 {
 155         rb_node_t *rbnode;
 156         struct ll_lock_tree_node *node = NULL;
 157
 158         for ( rbnode = tree->lt_root.rb_node; rbnode != NULL;
 159               rbnode = rbnode->rb_left) {
 160                 if (rbnode->rb_left == NULL) {
 161                         node = rb_entry(rbnode, struct ll_lock_tree_node,
 162                                         lt_node);
 163                         break;
 164                 }
 165         }
 166         RETURN(node);
 167 }
 168
 169 int ll_tree_unlock(struct ll_lock_tree *tree, struct inode *inode)
 170 {
 171         struct ll_lock_tree_node *node;
 172         struct list_head *pos, *n;
 173         int rc = 0;
 174         ENTRY;
 175
 176         list_for_each_safe(pos, n, &tree->lt_locked_list) {
 177                 node = list_entry(pos, struct ll_lock_tree_node,
 178                                   lt_locked_item);
 179
 180                 rc = ll_extent_unlock(tree->lt_fd, inode,
 181                                       ll_i2info(inode)->lli_smd, node->lt_mode,
 182                                       &node->lt_lockh);
 183                 if (rc != 0) {
 184                         /* XXX better message */
 185                         CERROR("couldn't unlock %d\n", rc);
 186                 }
 187                 list_del(&node->lt_locked_item);
 188                 OBD_FREE(node, sizeof(*node));
 189         }
 190
 191         while ((node = lt_least_node(tree))) {
 192                 rb_erase(&node->lt_node, &tree->lt_root);
 193                 OBD_FREE(node, sizeof(*node));
 194         }
 195
 196         RETURN(rc);
 197 }
 198
 199 int ll_tree_lock(struct ll_lock_tree *tree,
 200                  struct ll_lock_tree_node *first_node, struct inode *inode,
 201                  const char *buf, size_t count, int ast_flags)
 202 {
 203         struct ll_lock_tree_node *node;
 204         int rc = 0;
 205         ENTRY;
 206
 207         tree->lt_root.rb_node = NULL;
 208         INIT_LIST_HEAD(&tree->lt_locked_list);
 209         if (first_node != NULL)
 210                 lt_insert(tree, first_node);
 211
 212         /* order locking. what we have to concern about is ONLY double lock:
 213          * the buffer is mapped to exactly this file. */
 214         if (mapping_mapped(inode->i_mapping)) {
 215                 rc = lt_get_mmap_locks(tree, inode, (unsigned long)buf, count);
 216                 if (rc)
 217                         GOTO(out, rc);
 218         }
 219
 220         while ((node = lt_least_node(tree))) {
 221                 struct obd_service_time *stime;
 222                 stime = (node->lt_mode & LCK_PW) ?
 223                         &ll_i2sbi(inode)->ll_write_stime :
 224                         &ll_i2sbi(inode)->ll_read_stime;
 225
 226                 rc = ll_extent_lock(tree->lt_fd, inode,
 227                                     ll_i2info(inode)->lli_smd, node->lt_mode,
 228                                     &node->lt_policy, &node->lt_lockh,
 229                                     ast_flags, stime);
 230                 if (rc != 0)
 231                         GOTO(out, rc);
 232
 233                 rb_erase(&node->lt_node, &tree->lt_root);
 234                 list_add_tail(&node->lt_locked_item, &tree->lt_locked_list);
 235         }
 236         RETURN(rc);
 237 out:
 238         ll_tree_unlock(tree, inode);
 239         return rc;
 240 }
 241
 242 static ldlm_mode_t mode_from_vma(struct vm_area_struct *vma)
 243 {
 244         /* we only want to hold PW locks if the mmap() can generate
 245          * writes back to the file and that only happens in shared
 246          * writable vmas */
 247         if ((vma->vm_flags & VM_SHARED) && (vma->vm_flags & VM_WRITE))
 248                 return LCK_PW;
 249         return LCK_PR;
 250 }
 251
 252 static void policy_from_vma(ldlm_policy_data_t *policy,
 253                             struct vm_area_struct *vma, unsigned long addr,
 254                             size_t count)
 255 {
 256         policy->l_extent.start = ((addr - vma->vm_start) & PAGE_CACHE_MASK) +
 257                                  (vma->vm_pgoff << PAGE_CACHE_SHIFT);
 258         policy->l_extent.end = (policy->l_extent.start + count - 1) |
 259                                (PAGE_CACHE_SIZE - 1);
 260 }
 261
 262 static struct vm_area_struct *our_vma(unsigned long addr, size_t count,
 263                                        struct inode *inode)
 264 {
 265         struct mm_struct *mm = current->mm;
 266         struct vm_area_struct *vma, *ret = NULL;
 267         ENTRY;
 268
 269         spin_lock(&mm->page_table_lock);
 270         for(vma = find_vma(mm, addr);
 271             vma != NULL && vma->vm_start < (addr + count); vma = vma->vm_next) {
 272                 if (vma->vm_ops && vma->vm_ops->nopage == ll_nopage &&
 273                     vma->vm_file && vma->vm_file->f_dentry->d_inode == inode) {
 274                         ret = vma;
 275                         break;
 276                 }
 277         }
 278         spin_unlock(&mm->page_table_lock);
 279         RETURN(ret);
 280 }
 281
 282 int lt_get_mmap_locks(struct ll_lock_tree *tree, struct inode *inode,
 283                       unsigned long addr, size_t count)
 284 {
 285         struct vm_area_struct *vma;
 286         struct ll_lock_tree_node *node;
 287         ldlm_policy_data_t policy;
 288         ENTRY;
 289
 290         if (count == 0)
 291                 RETURN(0);
 292
 293         /* we need to look up vmas on page aligned addresses */
 294         count += addr & (PAGE_SIZE - 1);
 295         addr -= addr & (PAGE_SIZE - 1);
 296
 297         while ((vma = our_vma(addr, count, inode)) != NULL) {
 298
 299                 policy_from_vma(&policy, vma, addr, count);
 300                 node = ll_node_from_inode(inode, policy.l_extent.start,
 301                                           policy.l_extent.end,
 302                                           mode_from_vma(vma));
 303                 if (IS_ERR(node)) {
 304                         CERROR("not enough mem for lock_tree_node!\n");
 305                         RETURN(-ENOMEM);
 306                 }
 307                 lt_insert(tree, node);
 308
 309                 if (vma->vm_end - addr >= count)
 310                         break;
 311                 count -= vma->vm_end - addr;
 312                 addr = vma->vm_end;
 313         }
 314         RETURN(0);
 315 }
 316 /* FIXME: there is a pagefault race goes as follow:
 317  * 1. A user process on node A accesses a portion of a mapped file,
 318  *    resulting in a page fault.  The pagefault handler invokes the
 319  *    ll_nopage function, which reads the page into memory.
 320  * 2. A user process on node B writes to the same portion of the file
 321  *    (either via mmap or write()), that cause node A to cancel the
 322  *    lock and truncate the page.
 323  * 3. Node A then executes the rest of do_no_page(), entering the
 324  *    now-invalid page into the PTEs.
 325  *
 326  * Make the whole do_no_page as a hook to cover both the page cache
 327  * and page mapping installing with dlm lock would eliminate this race.
 328  */
 329 #if (LINUX_VERSION_CODE >= KERNEL_VERSION(2,5,0))
 330 struct page *ll_nopage(struct vm_area_struct *vma, unsigned long address,
 331                        int *type)
 332 #else
 333 struct page *ll_nopage(struct vm_area_struct *vma, unsigned long address,
 334                        int unused)
 335 #endif
 336 {
 337         struct file *filp = vma->vm_file;
 338         struct ll_file_data *fd = filp->private_data;
 339         struct inode *inode = filp->f_dentry->d_inode;
 340         struct lustre_handle lockh = { 0 };
 341         ldlm_policy_data_t policy;
 342         ldlm_mode_t mode;
 343         struct page *page = NULL;
 344         struct ll_inode_info *lli = ll_i2info(inode);
 345         struct obd_service_time *stime;
 346         __u64 kms;
 347         unsigned long pgoff, size, rand_read, seq_read;
 348         int rc = 0;
 349         ENTRY;
 350
 351         if (lli->lli_smd == NULL) {
 352                 CERROR("No lsm on fault?\n");
 353                 RETURN(NULL);
 354         }
 355
 356         /* start and end the lock on the first and last bytes in the page */
 357         policy_from_vma(&policy, vma, address, PAGE_CACHE_SIZE);
 358
 359         CDEBUG(D_MMAP, "nopage vma %p inode %lu, locking ["LPU64", "LPU64"]\n",
 360                vma, inode->i_ino, policy.l_extent.start, policy.l_extent.end);
 361
 362         mode = mode_from_vma(vma);
 363         stime = (mode & LCK_PW) ? &ll_i2sbi(inode)->ll_write_stime :
 364                                   &ll_i2sbi(inode)->ll_read_stime;
 365
 366         rc = ll_extent_lock(fd, inode, lli->lli_smd, mode, &policy,
 367                             &lockh, LDLM_FL_CBPENDING, stime);
 368         if (rc != 0)
 369                 RETURN(NULL);
 370
 371         /* XXX change inode size without i_sem hold! there is a race condition
 372          *     with truncate path. (see ll_extent_lock) */
 373         down(&lli->lli_size_sem);
 374         kms = lov_merge_size(lli->lli_smd, 1);
 375         pgoff = ((address - vma->vm_start) >> PAGE_CACHE_SHIFT) + vma->vm_pgoff;
 376         size = (kms + PAGE_CACHE_SIZE - 1) >> PAGE_CACHE_SHIFT;
 377
 378         if (pgoff >= size) {
 379                 up(&lli->lli_size_sem);
 380                 ll_glimpse_size(inode);
 381         } else {
 382                 inode->i_size = kms;
 383                 up(&lli->lli_size_sem);
 384         }
 385
 386         /* disable VM_SEQ_READ and use VM_RAND_READ to make sure that
 387          * the kernel will not read other pages not covered by ldlm in
 388          * filemap_nopage. we do our readahead in ll_readpage.
 389          */
 390         rand_read = vma->vm_flags & VM_RAND_READ;
 391         seq_read = vma->vm_flags & VM_SEQ_READ;
 392         vma->vm_flags &= ~ VM_SEQ_READ;
 393         vma->vm_flags |= VM_RAND_READ;
 394
 395 #if (LINUX_VERSION_CODE >= KERNEL_VERSION(2,5,0))
 396         page = filemap_nopage(vma, address, type);
 397 #else
 398         page = filemap_nopage(vma, address, unused);
 399 #endif
 400         vma->vm_flags &= ~VM_RAND_READ;
 401         vma->vm_flags |= (rand_read | seq_read);
 402
 403         ll_extent_unlock(fd, inode, ll_i2info(inode)->lli_smd, mode, &lockh);
 404         RETURN(page);
 405 }
 406
 407 /* return the user space pointer that maps to a file offset via a vma */
 408 static inline unsigned long file_to_user(struct vm_area_struct *vma,
 409                                          __u64 byte)
 410 {
 411         return vma->vm_start +
 412                (byte - ((__u64)vma->vm_pgoff << PAGE_CACHE_SHIFT));
 413 }
 414
 415 #define VMA_DEBUG(vma, fmt, arg...)                                          \
 416         CDEBUG(D_MMAP, "vma(%p) start(%ld) end(%ld) pgoff(%ld) inode(%p) "   \
 417                "ino(%lu) iname(%s): " fmt, vma, vma->vm_start, vma->vm_end,  \
 418                vma->vm_pgoff, vma->vm_file->f_dentry->d_inode,               \
 419                vma->vm_file->f_dentry->d_inode->i_ino,                       \
 420                vma->vm_file->f_dentry->d_iname, ## arg);                     \
 421
 422 #if (LINUX_VERSION_CODE < KERNEL_VERSION(2,5,0))
 423 /* [first, last] are the byte offsets affected.
 424  * vm_{start, end} are user addresses of the first byte of the mapping and
 425  *      the next byte beyond it
 426  * vm_pgoff is the page index of the first byte in the mapping */
 427 static void teardown_vmas(struct vm_area_struct *vma, __u64 first,
 428                           __u64 last)
 429 {
 430         unsigned long address, len;
 431         for (; vma ; vma = vma->vm_next_share) {
 432                 if (last >> PAGE_SHIFT < vma->vm_pgoff)
 433                         continue;
 434                 if (first >> PAGE_CACHE_SHIFT > (vma->vm_pgoff +
 435                     ((vma->vm_end - vma->vm_start) >> PAGE_CACHE_SHIFT)))
 436                         continue;
 437
 438                 /* XXX in case of unmap the cow pages of a running file,
 439                  * don't unmap these private writeable mapping here!
 440                  * though that will break private mappping a little.
 441                  *
 442                  * the clean way is to check the mapping of every page
 443                  * and just unmap the non-cow pages, just like
 444                  * unmap_mapping_range() with even_cow=0 in kernel 2.6.
 445                  */
 446                 if (!(vma->vm_flags & VM_SHARED) &&
 447                     (vma->vm_flags & VM_WRITE))
 448                         continue;
 449
 450                 address = max((unsigned long)vma->vm_start,
 451                               file_to_user(vma, first));
 452                 len = min((unsigned long)vma->vm_end,
 453                           file_to_user(vma, last) + 1) - address;
 454
 455                 VMA_DEBUG(vma, "zapping vma [first="LPU64" last="LPU64" "
 456                           "address=%ld len=%ld]\n", first, last, address, len);
 457                 LASSERT(len > 0);
 458                 ll_zap_page_range(vma, address, len);
 459         }
 460 }
 461 #endif
 462
 463 /* XXX put nice comment here.  talk about __free_pte -> dirty pages and
 464  * nopage's reference passing to the pte */
 465 int ll_teardown_mmaps(struct address_space *mapping, __u64 first,
 466                        __u64 last)
 467 {
 468         int rc = -ENOENT;
 469         ENTRY;
 470
 471         LASSERTF(last > first, "last "LPU64" first "LPU64"\n", last, first);
 472 #if (LINUX_VERSION_CODE >= KERNEL_VERSION(2,5,0))
 473         if (mapping_mapped(mapping)) {
 474                 rc = 0;
 475                 unmap_mapping_range(mapping, first + PAGE_SIZE - 1,
 476                                     last - first + 1, 0);
 477         }
 478 #else
 479         spin_lock(&mapping->i_shared_lock);
 480         if (mapping->i_mmap != NULL) {
 481                 rc = 0;
 482                 teardown_vmas(mapping->i_mmap, first, last);
 483         }
 484         if (mapping->i_mmap_shared != NULL) {
 485                 rc = 0;
 486                 teardown_vmas(mapping->i_mmap_shared, first, last);
 487         }
 488         spin_unlock(&mapping->i_shared_lock);
 489 #endif
 490
 491         RETURN(rc);
 492 }
 493
 494
 495 static void ll_close_vma(struct vm_area_struct *vma)
 496 {
 497         struct inode *inode = vma->vm_file->f_dentry->d_inode;
 498         struct address_space *mapping = inode->i_mapping;
 499         unsigned long next, size, end;
 500         struct ll_async_page *llap;
 501         struct obd_export *exp;
 502         struct pagevec pvec;
 503         int i;
 504
 505         if (!(vma->vm_flags & VM_SHARED))
 506                 return;
 507
 508         /* all pte's are synced to mem_map by the moment
 509          * we scan backing store and put all dirty pages
 510          * onto pending list to track flushing */
 511
 512         LASSERT(LLI_DIRTY_HANDLE(inode));
 513         exp = ll_i2dtexp(inode);
 514         if (exp == NULL) {
 515                 CERROR("can't get export for the inode\n");
 516                 return;
 517         }
 518
 519         pagevec_init(&pvec, 0);
 520         next = vma->vm_pgoff;
 521         size = (vma->vm_end - vma->vm_start) / PAGE_SIZE;
 522         end = next + size - 1;
 523
 524         CDEBUG(D_INODE, "close vma 0x%p[%lu/%lu/%lu from %lu/%u]\n", vma,
 525                next, size, end, inode->i_ino, inode->i_generation);
 526
 527         while (next <= end && pagevec_lookup(&pvec, mapping, next, PAGEVEC_SIZE)) {
 528                 for (i = 0; i < pagevec_count(&pvec); i++) {
 529                         struct page *page = pvec.pages[i];
 530
 531                         if (page->index > next)
 532                                 next = page->index;
 533                         if (next > end)
 534                                 continue;
 535                         next++;
 536
 537                         lock_page(page);
 538                         if (page->mapping != mapping || !PageDirty(page)) {
 539                                 unlock_page(page);
 540                                 continue;
 541                         }
 542
 543                         llap = llap_from_page(page, LLAP_ORIGIN_COMMIT_WRITE);
 544                         if (IS_ERR(llap)) {
 545                                 CERROR("can't get llap\n");
 546                                 unlock_page(page);
 547                                 continue;
 548                         }
 549
 550                         llap_write_pending(inode, llap);
 551                         unlock_page(page);
 552                 }
 553                 pagevec_release(&pvec);
 554         }
 555 }
 556
 557 static struct vm_operations_struct ll_file_vm_ops = {
 558         .nopage         = ll_nopage,
 559         .close          = ll_close_vma,
 560 };
 561
 562 /* Audit functions */
 563 extern int ll_audit_log (struct inode *, audit_op, int);
 564
 565 int ll_file_mmap(struct file * file, struct vm_area_struct * vma)
 566 {
 567         int rc;
 568         ENTRY;
 569
 570         rc = generic_file_mmap(file, vma);
 571         if (rc == 0) {
 572                 struct ll_inode_info *lli = ll_i2info(file->f_dentry->d_inode);
 573                 vma->vm_ops = &ll_file_vm_ops;
 574
 575                 /* mark i/o epoch dirty */
 576                 if (vma->vm_flags & VM_SHARED)
 577                         set_bit(LLI_F_DIRTY_HANDLE, &lli->lli_flags);
 578         }
 579
 580         ll_audit_log(file->f_dentry->d_inode, AUDIT_MMAP, rc);
 581
 582         RETURN(rc);
 583 }
 584