lustre/llite/llite_mmap.c

   1 /* -*- mode: c; c-basic-offset: 8; indent-tabs-mode: nil; -*-
   2  * vim:expandtab:shiftwidth=8:tabstop=8:
   3  *
   4  *  Copyright (c) 2001-2003 Cluster File Systems, Inc.
   5  *
   6  *   This file is part of Lustre, http://www.lustre.org.
   7  *
   8  *   Lustre is free software; you can redistribute it and/or
   9  *   modify it under the terms of version 2 of the GNU General Public
  10  *   License as published by the Free Software Foundation.
  11  *
  12  *   Lustre is distributed in the hope that it will be useful,
  13  *   but WITHOUT ANY WARRANTY; without even the implied warranty of
  14  *   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  15  *   GNU General Public License for more details.
  16  *
  17  *   You should have received a copy of the GNU General Public License
  18  *   along with Lustre; if not, write to the Free Software
  19  *   Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
  20  */
  21
  22 #include <linux/config.h>
  23 #include <linux/kernel.h>
  24 #include <linux/mm.h>
  25 #include <linux/string.h>
  26 #include <linux/stat.h>
  27 #include <linux/errno.h>
  28 #include <linux/smp_lock.h>
  29 #include <linux/unistd.h>
  30 #include <linux/version.h>
  31 #include <asm/system.h>
  32 #include <asm/uaccess.h>
  33
  34 #include <linux/fs.h>
  35 #include <linux/stat.h>
  36 #include <asm/uaccess.h>
  37 #include <asm/segment.h>
  38 #include <linux/mm.h>
  39 #include <linux/pagemap.h>
  40 #include <linux/smp_lock.h>
  41 #if (LINUX_VERSION_CODE < KERNEL_VERSION(2,5,0))
  42 #include <linux/iobuf.h>
  43 #endif
  44
  45 #define DEBUG_SUBSYSTEM S_LLITE
  46
  47 #include <linux/lustre_mds.h>
  48 #include <linux/lustre_lite.h>
  49 #include "llite_internal.h"
  50 #include <linux/lustre_compat25.h>
  51
  52
  53 struct ll_lock_tree_node {
  54         rb_node_t               lt_node;
  55         struct list_head        lt_locked_item;
  56         __u64                   lt_oid;
  57         ldlm_policy_data_t      lt_policy;
  58         struct lustre_handle    lt_lockh;
  59         ldlm_mode_t             lt_mode;
  60 };
  61
  62 __u64 lov_merge_size(struct lov_stripe_md *lsm, int kms);
  63 int lt_get_mmap_locks(struct ll_lock_tree *tree, struct inode *inode,
  64                       unsigned long addr, size_t count);
  65 #if (LINUX_VERSION_CODE >= KERNEL_VERSION(2,5,0))
  66 struct page *ll_nopage(struct vm_area_struct *vma, unsigned long address,
  67                        int *type);
  68 #else
  69 struct page *ll_nopage(struct vm_area_struct *vma, unsigned long address,
  70                        int unused);
  71 #endif
  72
  73 struct ll_lock_tree_node * ll_node_from_inode(struct inode *inode, __u64 start,
  74                                               __u64 end, ldlm_mode_t mode)
  75 {
  76         struct ll_lock_tree_node *node;
  77
  78         OBD_ALLOC(node, sizeof(*node));
  79         if (node == NULL)
  80                 RETURN(ERR_PTR(-ENOMEM));
  81
  82         node->lt_oid = ll_i2info(inode)->lli_smd->lsm_object_id;
  83         node->lt_policy.l_extent.start = start;
  84         node->lt_policy.l_extent.end = end;
  85         memset(&node->lt_lockh, 0, sizeof(node->lt_lockh));
  86         INIT_LIST_HEAD(&node->lt_locked_item);
  87         node->lt_mode = mode;
  88
  89         return node;
  90 }
  91
  92 int lt_compare(struct ll_lock_tree_node *one, struct ll_lock_tree_node *two)
  93 {
  94         if ( one->lt_oid < two->lt_oid)
  95                 return -1;
  96         if ( one->lt_oid > two->lt_oid)
  97                 return 1;
  98
  99         if ( one->lt_policy.l_extent.end < two->lt_policy.l_extent.start )
 100                 return -1;
 101         if ( one->lt_policy.l_extent.start > two->lt_policy.l_extent.end )
 102                 return 1;
 103
 104         return 0; /* they are the same object and overlap */
 105 }
 106
 107 static void lt_merge(struct ll_lock_tree_node *dst,
 108                      struct ll_lock_tree_node *src)
 109 {
 110         dst->lt_policy.l_extent.start = min(dst->lt_policy.l_extent.start,
 111                                             src->lt_policy.l_extent.start);
 112         dst->lt_policy.l_extent.end = max(dst->lt_policy.l_extent.end,
 113                                           src->lt_policy.l_extent.end);
 114
 115         /* XXX could be a real call to the dlm to find superset modes */
 116         if (src->lt_mode == LCK_PW && dst->lt_mode != LCK_PW)
 117                 dst->lt_mode = LCK_PW;
 118 }
 119
 120 static void lt_insert(struct ll_lock_tree *tree,
 121                       struct ll_lock_tree_node *node)
 122 {
 123         struct ll_lock_tree_node *walk;
 124         rb_node_t **p, *parent;
 125         ENTRY;
 126
 127 restart:
 128         p = &tree->lt_root.rb_node;
 129         parent = NULL;
 130         while (*p) {
 131                 parent = *p;
 132                 walk = rb_entry(parent, struct ll_lock_tree_node, lt_node);
 133                 switch (lt_compare(node, walk)) {
 134                 case -1:
 135                         p = &(*p)->rb_left;
 136                         break;
 137                 case 1:
 138                         p = &(*p)->rb_right;
 139                         break;
 140                 case 0:
 141                         lt_merge(node, walk);
 142                         rb_erase(&walk->lt_node, &tree->lt_root);
 143                         OBD_FREE(walk, sizeof(*walk));
 144                         goto restart;
 145                         break;
 146                 default:
 147                         LBUG();
 148                         break;
 149                 }
 150         }
 151         rb_link_node(&node->lt_node, parent, p);
 152         rb_insert_color(&node->lt_node, &tree->lt_root);
 153         EXIT;
 154 }
 155
 156 static struct ll_lock_tree_node *lt_least_node(struct ll_lock_tree *tree)
 157 {
 158         rb_node_t *rbnode;
 159         struct ll_lock_tree_node *node = NULL;
 160
 161         for ( rbnode = tree->lt_root.rb_node; rbnode != NULL;
 162               rbnode = rbnode->rb_left) {
 163                 if (rbnode->rb_left == NULL) {
 164                         node = rb_entry(rbnode, struct ll_lock_tree_node,
 165                                         lt_node);
 166                         break;
 167                 }
 168         }
 169         RETURN(node);
 170 }
 171
 172 int ll_tree_unlock(struct ll_lock_tree *tree, struct inode *inode)
 173 {
 174         struct ll_lock_tree_node *node;
 175         struct list_head *pos, *n;
 176         int rc = 0;
 177         ENTRY;
 178
 179         list_for_each_safe(pos, n, &tree->lt_locked_list) {
 180                 node = list_entry(pos, struct ll_lock_tree_node,
 181                                   lt_locked_item);
 182
 183                 rc = ll_extent_unlock(tree->lt_fd, inode,
 184                                       ll_i2info(inode)->lli_smd, node->lt_mode,
 185                                       &node->lt_lockh);
 186                 if (rc != 0) {
 187                         /* XXX better message */
 188                         CERROR("couldn't unlock %d\n", rc);
 189                 }
 190                 list_del(&node->lt_locked_item);
 191                 OBD_FREE(node, sizeof(*node));
 192         }
 193
 194         while ((node = lt_least_node(tree))) {
 195                 rb_erase(&node->lt_node, &tree->lt_root);
 196                 OBD_FREE(node, sizeof(*node));
 197         }
 198
 199         RETURN(rc);
 200 }
 201
 202 int ll_tree_lock(struct ll_lock_tree *tree,
 203                  struct ll_lock_tree_node *first_node, struct inode *inode,
 204                  const char *buf, size_t count, int ast_flags)
 205 {
 206         struct ll_lock_tree_node *node;
 207         int rc = 0;
 208         ENTRY;
 209
 210         tree->lt_root.rb_node = NULL;
 211         INIT_LIST_HEAD(&tree->lt_locked_list);
 212         if (first_node != NULL)
 213                 lt_insert(tree, first_node);
 214
 215         if (mapping_mapped(inode->i_mapping)) {
 216                 rc = lt_get_mmap_locks(tree, inode, (unsigned long)buf, count);
 217                 if (rc)
 218                         GOTO(out, rc);
 219         }
 220
 221         while ((node = lt_least_node(tree))) {
 222                 struct obd_service_time *stime;
 223                 stime = (node->lt_mode & LCK_PW) ?
 224                         &ll_i2sbi(inode)->ll_write_stime :
 225                         &ll_i2sbi(inode)->ll_read_stime;
 226
 227                 rc = ll_extent_lock(tree->lt_fd, inode,
 228                                     ll_i2info(inode)->lli_smd, node->lt_mode,
 229                                     &node->lt_policy, &node->lt_lockh,
 230                                     ast_flags, stime);
 231                 if (rc != 0)
 232                         GOTO(out, rc);
 233
 234                 rb_erase(&node->lt_node, &tree->lt_root);
 235                 list_add_tail(&node->lt_locked_item, &tree->lt_locked_list);
 236         }
 237         RETURN(rc);
 238 out:
 239         ll_tree_unlock(tree, inode);
 240         return rc;
 241 }
 242
 243 static ldlm_mode_t mode_from_vma(struct vm_area_struct *vma)
 244 {
 245         /* we only want to hold PW locks if the mmap() can generate
 246          * writes back to the file and that only happens in shared
 247          * writable vmas */
 248         if ((vma->vm_flags & VM_SHARED) && (vma->vm_flags & VM_WRITE))
 249                 return LCK_PW;
 250         return LCK_PR;
 251 }
 252
 253 static void policy_from_vma(ldlm_policy_data_t *policy,
 254                             struct vm_area_struct *vma, unsigned long addr,
 255                             size_t count)
 256 {
 257         policy->l_extent.start = ((addr - vma->vm_start) & PAGE_CACHE_MASK) +
 258                                  (vma->vm_pgoff << PAGE_CACHE_SHIFT);
 259         policy->l_extent.end = (policy->l_extent.start + count - 1) |
 260                                (PAGE_CACHE_SIZE - 1);
 261 }
 262 static struct vm_area_struct * our_vma(unsigned long addr, size_t count)
 263 {
 264         struct mm_struct *mm = current->mm;
 265         struct vm_area_struct *vma, *ret = NULL;
 266         ENTRY;
 267
 268         spin_lock(&mm->page_table_lock);
 269         for(vma = find_vma(mm, addr);
 270             vma != NULL && vma->vm_start < (addr + count); vma = vma->vm_next) {
 271                 if (vma->vm_ops && vma->vm_ops->nopage == ll_nopage) {
 272                         ret = vma;
 273                         break;
 274                 }
 275         }
 276         spin_unlock(&mm->page_table_lock);
 277         RETURN(ret);
 278 }
 279
 280 int lt_get_mmap_locks(struct ll_lock_tree *tree, struct inode *inode,
 281                       unsigned long addr, size_t count)
 282 {
 283         struct vm_area_struct *vma;
 284         struct ll_lock_tree_node *node;
 285         ldlm_policy_data_t policy;
 286         ENTRY;
 287
 288         if (count == 0)
 289                 RETURN(0);
 290
 291         /* we need to look up vmas on page aligned addresses */
 292         count += addr & (PAGE_SIZE - 1);
 293         addr -= addr & (PAGE_SIZE - 1);
 294
 295         while ((vma = our_vma(addr, count)) != NULL) {
 296
 297                 policy_from_vma(&policy, vma, addr, count);
 298                 node = ll_node_from_inode(inode, policy.l_extent.start,
 299                                           policy.l_extent.end,
 300                                           mode_from_vma(vma));
 301                 if (IS_ERR(node)) {
 302                         CERROR("not enough mem for lock_tree_node!\n");
 303                         RETURN(-ENOMEM);
 304                 }
 305                 lt_insert(tree, node);
 306
 307                 if (vma->vm_end - addr >= count)
 308                         break;
 309                 count -= vma->vm_end - addr;
 310                 addr = vma->vm_end;
 311         }
 312         RETURN(0);
 313 }
 314 /* FIXME: there is a pagefault race goes as follow:
 315  * 1. A user process on node A accesses a portion of a mapped file,
 316  *    resulting in a page fault.  The pagefault handler invokes the
 317  *    ll_nopage function, which reads the page into memory.
 318  * 2. A user process on node B writes to the same portion of the file
 319  *    (either via mmap or write()), that cause node A to cancel the
 320  *    lock and truncate the page.
 321  * 3. Node A then executes the rest of do_no_page(), entering the
 322  *    now-invalid page into the PTEs.
 323  *
 324  * Make the whole do_no_page as a hook to cover both the page cache
 325  * and page mapping installing with dlm lock would eliminate this race.
 326  */
 327 #if (LINUX_VERSION_CODE >= KERNEL_VERSION(2,5,0))
 328 struct page *ll_nopage(struct vm_area_struct *vma, unsigned long address,
 329                        int *type)
 330 #else
 331 struct page *ll_nopage(struct vm_area_struct *vma, unsigned long address,
 332                        int unused)
 333 #endif
 334 {
 335         struct file *filp = vma->vm_file;
 336         struct ll_file_data *fd = filp->private_data;
 337         struct inode *inode = filp->f_dentry->d_inode;
 338         struct lustre_handle lockh = { 0 };
 339         ldlm_policy_data_t policy;
 340         ldlm_mode_t mode;
 341         struct page *page;
 342         struct obd_service_time *stime;
 343         __u64 kms;
 344         unsigned long pgoff, size, rand_read, seq_read;
 345         int rc = 0;
 346         ENTRY;
 347
 348         if (ll_i2info(inode)->lli_smd == NULL) {
 349                 CERROR("No lsm on fault?\n");
 350                 RETURN(NULL);
 351         }
 352
 353         /* start and end the lock on the first and last bytes in the page */
 354         policy_from_vma(&policy, vma, address, PAGE_CACHE_SIZE);
 355
 356         CDEBUG(D_MMAP, "nopage vma %p inode %lu, locking ["LPU64", "LPU64"]\n",
 357                vma, inode->i_ino, policy.l_extent.start,
 358                policy.l_extent.end);
 359
 360         mode = mode_from_vma(vma);
 361         stime = (mode & LCK_PW) ? &ll_i2sbi(inode)->ll_write_stime :
 362                                   &ll_i2sbi(inode)->ll_read_stime;
 363
 364         rc = ll_extent_lock(fd, inode, ll_i2info(inode)->lli_smd, mode, &policy,
 365                             &lockh, LDLM_FL_CBPENDING, stime);
 366         if (rc != 0)
 367                 RETURN(NULL);
 368
 369         /* XXX change inode size without i_sem hold! there is a race condition
 370          *     with truncate path. (see ll_extent_lock) */
 371         kms = lov_merge_size(ll_i2info(inode)->lli_smd, 1);
 372         pgoff = ((address - vma->vm_start) >> PAGE_CACHE_SHIFT) + vma->vm_pgoff;
 373         size = (kms + PAGE_CACHE_SIZE - 1) >> PAGE_CACHE_SHIFT;
 374
 375         if (pgoff >= size)
 376                 ll_glimpse_size(inode);
 377         else
 378                 inode->i_size = kms;
 379
 380         /* disable VM_SEQ_READ and use VM_RAND_READ to make sure that
 381          * the kernel will not read other pages not covered by ldlm in
 382          * filemap_nopage. we do our readahead in ll_readpage.
 383          */
 384         rand_read = vma->vm_flags & VM_RAND_READ;
 385         seq_read = vma->vm_flags & VM_SEQ_READ;
 386         vma->vm_flags &= ~ VM_SEQ_READ;
 387         vma->vm_flags |= VM_RAND_READ;
 388
 389 #if (LINUX_VERSION_CODE >= KERNEL_VERSION(2,5,0))
 390         page = filemap_nopage(vma, address, type);
 391 #else
 392         page = filemap_nopage(vma, address, unused);
 393 #endif
 394         vma->vm_flags &= ~VM_RAND_READ;
 395         vma->vm_flags |= (rand_read | seq_read);
 396
 397         ll_extent_unlock(fd, inode, ll_i2info(inode)->lli_smd, mode, &lockh);
 398         RETURN(page);
 399 }
 400
 401 /* return the user space pointer that maps to a file offset via a vma */
 402 static inline unsigned long file_to_user(struct vm_area_struct *vma,
 403                                          __u64 byte)
 404 {
 405         return vma->vm_start +
 406                (byte - ((__u64)vma->vm_pgoff << PAGE_CACHE_SHIFT));
 407
 408 }
 409
 410 #define VMA_DEBUG(vma, fmt, arg...)                                          \
 411         CDEBUG(D_MMAP, "vma(%p) start(%ld) end(%ld) pgoff(%ld) inode(%p): "  \
 412                fmt, vma, vma->vm_start, vma->vm_end, vma->vm_pgoff,          \
 413                vma->vm_file->f_dentry->d_inode, ## arg);
 414
 415 #if (LINUX_VERSION_CODE < KERNEL_VERSION(2,5,0))
 416 /* [first, last] are the byte offsets affected.
 417  * vm_{start, end} are user addresses of the first byte of the mapping and
 418  *      the next byte beyond it
 419  * vm_pgoff is the page index of the first byte in the mapping */
 420 static void teardown_vmas(struct vm_area_struct *vma, __u64 first,
 421                           __u64 last)
 422 {
 423         unsigned long address, len;
 424         for (; vma ; vma = vma->vm_next_share) {
 425                 if (last >> PAGE_CACHE_SHIFT < vma->vm_pgoff)
 426                         continue;
 427                 if (first >> PAGE_CACHE_SHIFT > (vma->vm_pgoff +
 428                     ((vma->vm_end - vma->vm_start) >> PAGE_CACHE_SHIFT)))
 429                         continue;
 430
 431                 address = max((unsigned long)vma->vm_start,
 432                               file_to_user(vma, first));
 433                 len = min((unsigned long)vma->vm_end,
 434                           file_to_user(vma, last) + 1) - address;
 435
 436                 VMA_DEBUG(vma, "zapping vma [address=%ld len=%ld]\n",
 437                           address, len);
 438                 LASSERT(vma->vm_mm);
 439                 ll_zap_page_range(vma, address, len);
 440         }
 441 }
 442 #endif
 443
 444 /* XXX put nice comment here.  talk about __free_pte -> dirty pages and
 445  * nopage's reference passing to the pte */
 446 int ll_teardown_mmaps(struct address_space *mapping, __u64 first,
 447                        __u64 last)
 448 {
 449         int rc = -ENOENT;
 450         ENTRY;
 451
 452 #if (LINUX_VERSION_CODE >= KERNEL_VERSION(2,5,0))
 453         if (mapping_mapped(mapping)) {
 454                 rc = 0;
 455                 unmap_mapping_range(mapping, first + PAGE_SIZE - 1,
 456                                     last - first + 1, 1);
 457         }
 458 #else
 459         spin_lock(&mapping->i_shared_lock);
 460         if (mapping->i_mmap != NULL) {
 461                 rc = 0;
 462                 teardown_vmas(mapping->i_mmap, first, last);
 463         }
 464         if (mapping->i_mmap_shared != NULL) {
 465                 rc = 0;
 466                 teardown_vmas(mapping->i_mmap_shared, first, last);
 467         }
 468         spin_unlock(&mapping->i_shared_lock);
 469 #endif
 470
 471         RETURN(rc);
 472 }
 473
 474 static struct vm_operations_struct ll_file_vm_ops = {
 475         .nopage         = ll_nopage,
 476 };
 477
 478 int ll_file_mmap(struct file * file, struct vm_area_struct * vma)
 479 {
 480         int rc;
 481         ENTRY;
 482
 483         rc = generic_file_mmap(file, vma);
 484         if (rc == 0)
 485                 vma->vm_ops = &ll_file_vm_ops;
 486
 487         RETURN(rc);
 488 }
 489