lustre/llite/llite_mmap.c

   1 /* -*- mode: c; c-basic-offset: 8; indent-tabs-mode: nil; -*-
   2  * vim:expandtab:shiftwidth=8:tabstop=8:
   3  *
   4  *  Copyright (c) 2001-2003 Cluster File Systems, Inc.
   5  *
   6  *   This file is part of Lustre, http://www.lustre.org.
   7  *
   8  *   Lustre is free software; you can redistribute it and/or
   9  *   modify it under the terms of version 2 of the GNU General Public
  10  *   License as published by the Free Software Foundation.
  11  *
  12  *   Lustre is distributed in the hope that it will be useful,
  13  *   but WITHOUT ANY WARRANTY; without even the implied warranty of
  14  *   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  15  *   GNU General Public License for more details.
  16  *
  17  *   You should have received a copy of the GNU General Public License
  18  *   along with Lustre; if not, write to the Free Software
  19  *   Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
  20  */
  21
  22 #include <linux/config.h>
  23 #include <linux/kernel.h>
  24 #include <linux/mm.h>
  25 #include <linux/string.h>
  26 #include <linux/stat.h>
  27 #include <linux/errno.h>
  28 #include <linux/smp_lock.h>
  29 #include <linux/unistd.h>
  30 #include <linux/version.h>
  31 #include <asm/system.h>
  32 #include <asm/uaccess.h>
  33
  34 #include <linux/fs.h>
  35 #include <linux/stat.h>
  36 #include <asm/uaccess.h>
  37 #include <asm/segment.h>
  38 #include <linux/mm.h>
  39 #include <linux/pagemap.h>
  40 #include <linux/smp_lock.h>
  41 #if (LINUX_VERSION_CODE < KERNEL_VERSION(2,5,0))
  42 #include <linux/iobuf.h>
  43 #endif
  44
  45 #define DEBUG_SUBSYSTEM S_LLITE
  46
  47 #include <linux/lustre_mds.h>
  48 #include <linux/lustre_lite.h>
  49 #include "llite_internal.h"
  50 #include <linux/lustre_compat25.h>
  51
  52
  53 struct ll_lock_tree_node {
  54         rb_node_t               lt_node;
  55         struct list_head        lt_locked_item;
  56         __u64                   lt_oid;
  57         ldlm_policy_data_t      lt_policy;
  58         struct lustre_handle    lt_lockh;
  59         ldlm_mode_t             lt_mode;
  60 };
  61
  62 __u64 lov_merge_size(struct lov_stripe_md *lsm, int kms);
  63 int lt_get_mmap_locks(struct ll_lock_tree *tree, struct inode *inode,
  64                       unsigned long addr, size_t count);
  65 #if (LINUX_VERSION_CODE >= KERNEL_VERSION(2,5,0))
  66 struct page *ll_nopage(struct vm_area_struct *vma, unsigned long address,
  67                        int *type);
  68 #else
  69 struct page *ll_nopage(struct vm_area_struct *vma, unsigned long address,
  70                        int unused);
  71 #endif
  72
  73 struct ll_lock_tree_node * ll_node_from_inode(struct inode *inode, __u64 start,
  74                                               __u64 end, ldlm_mode_t mode)
  75 {
  76         struct ll_lock_tree_node *node;
  77
  78         OBD_ALLOC(node, sizeof(*node));
  79         if (node == NULL)
  80                 RETURN(ERR_PTR(-ENOMEM));
  81
  82         node->lt_oid = ll_i2info(inode)->lli_smd->lsm_object_id;
  83         node->lt_policy.l_extent.start = start;
  84         node->lt_policy.l_extent.end = end;
  85         memset(&node->lt_lockh, 0, sizeof(node->lt_lockh));
  86         INIT_LIST_HEAD(&node->lt_locked_item);
  87         node->lt_mode = mode;
  88
  89         return node;
  90 }
  91
  92 int lt_compare(struct ll_lock_tree_node *one, struct ll_lock_tree_node *two)
  93 {
  94         if ( one->lt_oid < two->lt_oid)
  95                 return -1;
  96         if ( one->lt_oid > two->lt_oid)
  97                 return 1;
  98
  99         if ( one->lt_policy.l_extent.end < two->lt_policy.l_extent.start )
 100                 return -1;
 101         if ( one->lt_policy.l_extent.start > two->lt_policy.l_extent.end )
 102                 return 1;
 103
 104         return 0; /* they are the same object and overlap */
 105 }
 106
 107 static void lt_merge(struct ll_lock_tree_node *dst,
 108                      struct ll_lock_tree_node *src)
 109 {
 110         dst->lt_policy.l_extent.start = min(dst->lt_policy.l_extent.start,
 111                                             src->lt_policy.l_extent.start);
 112         dst->lt_policy.l_extent.end = max(dst->lt_policy.l_extent.end,
 113                                           src->lt_policy.l_extent.end);
 114
 115         /* XXX could be a real call to the dlm to find superset modes */
 116         if (src->lt_mode == LCK_PW && dst->lt_mode != LCK_PW)
 117                 dst->lt_mode = LCK_PW;
 118 }
 119
 120 static void lt_insert(struct ll_lock_tree *tree,
 121                       struct ll_lock_tree_node *node)
 122 {
 123         struct ll_lock_tree_node *walk;
 124         rb_node_t **p, *parent;
 125         ENTRY;
 126
 127 restart:
 128         p = &tree->lt_root.rb_node;
 129         parent = NULL;
 130         while (*p) {
 131                 parent = *p;
 132                 walk = rb_entry(parent, struct ll_lock_tree_node, lt_node);
 133                 switch (lt_compare(node, walk)) {
 134                 case -1:
 135                         p = &(*p)->rb_left;
 136                         break;
 137                 case 1:
 138                         p = &(*p)->rb_right;
 139                         break;
 140                 case 0:
 141                         lt_merge(node, walk);
 142                         rb_erase(&walk->lt_node, &tree->lt_root);
 143                         OBD_FREE(walk, sizeof(*walk));
 144                         goto restart;
 145                         break;
 146                 default:
 147                         LBUG();
 148                         break;
 149                 }
 150         }
 151         rb_link_node(&node->lt_node, parent, p);
 152         rb_insert_color(&node->lt_node, &tree->lt_root);
 153         EXIT;
 154 }
 155
 156 static struct ll_lock_tree_node *lt_least_node(struct ll_lock_tree *tree)
 157 {
 158         rb_node_t *rbnode;
 159         struct ll_lock_tree_node *node = NULL;
 160
 161         for ( rbnode = tree->lt_root.rb_node; rbnode != NULL;
 162               rbnode = rbnode->rb_left) {
 163                 if (rbnode->rb_left == NULL) {
 164                         node = rb_entry(rbnode, struct ll_lock_tree_node,
 165                                         lt_node);
 166                         break;
 167                 }
 168         }
 169         RETURN(node);
 170 }
 171
 172 int ll_tree_unlock(struct ll_lock_tree *tree, struct inode *inode)
 173 {
 174         struct ll_lock_tree_node *node;
 175         struct list_head *pos, *n;
 176         int rc = 0;
 177         ENTRY;
 178
 179         list_for_each_safe(pos, n, &tree->lt_locked_list) {
 180                 node = list_entry(pos, struct ll_lock_tree_node,
 181                                   lt_locked_item);
 182
 183                 rc = ll_extent_unlock(tree->lt_fd, inode,
 184                                       ll_i2info(inode)->lli_smd, node->lt_mode,
 185                                       &node->lt_lockh);
 186                 if (rc != 0) {
 187                         /* XXX better message */
 188                         CERROR("couldn't unlock %d\n", rc);
 189                 }
 190                 list_del(&node->lt_locked_item);
 191                 OBD_FREE(node, sizeof(*node));
 192         }
 193
 194         while ((node = lt_least_node(tree))) {
 195                 rb_erase(&node->lt_node, &tree->lt_root);
 196                 OBD_FREE(node, sizeof(*node));
 197         }
 198
 199         RETURN(rc);
 200 }
 201 int ll_tree_lock(struct ll_lock_tree *tree,
 202                  struct ll_lock_tree_node *first_node, struct inode *inode,
 203                  const char *buf, size_t count, int ast_flags)
 204 {
 205         struct ll_lock_tree_node *node;
 206         int rc = 0;
 207         ENTRY;
 208
 209         tree->lt_root.rb_node = NULL;
 210         INIT_LIST_HEAD(&tree->lt_locked_list);
 211         if (first_node != NULL)
 212                 lt_insert(tree, first_node);
 213
 214         if (mapping_mapped(inode->i_mapping)) {
 215                 rc = lt_get_mmap_locks(tree, inode, (unsigned long)buf, count);
 216                 if (rc)
 217                         GOTO(out, rc);
 218         }
 219
 220         while ((node = lt_least_node(tree))) {
 221                 struct obd_service_time *stime;
 222                 stime = (node->lt_mode & LCK_PW) ?
 223                         &ll_i2sbi(inode)->ll_write_stime :
 224                         &ll_i2sbi(inode)->ll_read_stime;
 225
 226                 rc = ll_extent_lock(tree->lt_fd, inode,
 227                                     ll_i2info(inode)->lli_smd, node->lt_mode,
 228                                     &node->lt_policy, &node->lt_lockh,
 229                                     ast_flags, stime);
 230                 if (rc != 0)
 231                         GOTO(out, rc);
 232
 233                 rb_erase(&node->lt_node, &tree->lt_root);
 234                 list_add_tail(&node->lt_locked_item, &tree->lt_locked_list);
 235         }
 236         RETURN(rc);
 237 out:
 238         ll_tree_unlock(tree, inode);
 239         RETURN(rc);
 240 }
 241
 242 static ldlm_mode_t mode_from_vma(struct vm_area_struct *vma)
 243 {
 244         /* we only want to hold PW locks if the mmap() can generate
 245          * writes back to the file and that only happens in shared
 246          * writable vmas */
 247         if ((vma->vm_flags & VM_SHARED) && (vma->vm_flags & VM_WRITE))
 248                 return LCK_PW;
 249         return LCK_PR;
 250 }
 251
 252 static void policy_from_vma(ldlm_policy_data_t *policy,
 253                             struct vm_area_struct *vma, unsigned long addr,
 254                             size_t count)
 255 {
 256         policy->l_extent.start = ((addr - vma->vm_start) & PAGE_CACHE_MASK) +
 257                                  (vma->vm_pgoff << PAGE_CACHE_SHIFT);
 258         policy->l_extent.end = (policy->l_extent.start + count - 1) |
 259                                (PAGE_CACHE_SIZE - 1);
 260 }
 261 static struct vm_area_struct * our_vma(unsigned long addr, size_t count)
 262 {
 263         struct mm_struct *mm = current->mm;
 264         struct vm_area_struct *vma, *ret = NULL;
 265         ENTRY;
 266
 267         spin_lock(&mm->page_table_lock);
 268         for(vma = find_vma(mm, addr);
 269             vma != NULL && vma->vm_start < (addr + count); vma = vma->vm_next) {
 270                 if (vma->vm_ops && vma->vm_ops->nopage == ll_nopage) {
 271                         ret = vma;
 272                         break;
 273                 }
 274         }
 275         spin_unlock(&mm->page_table_lock);
 276         RETURN(ret);
 277 }
 278
 279 int lt_get_mmap_locks(struct ll_lock_tree *tree, struct inode *inode,
 280                       unsigned long addr, size_t count)
 281 {
 282         struct vm_area_struct *vma;
 283         struct ll_lock_tree_node *node;
 284         ldlm_policy_data_t policy;
 285         ENTRY;
 286
 287         if (count == 0)
 288                 RETURN(0);
 289
 290         /* we need to look up vmas on page aligned addresses */
 291         count += addr & (PAGE_SIZE - 1);
 292         addr -= addr & (PAGE_SIZE - 1);
 293
 294         while ((vma = our_vma(addr, count)) != NULL) {
 295
 296                 policy_from_vma(&policy, vma, addr, count);
 297                 node = ll_node_from_inode(inode, policy.l_extent.start,
 298                                           policy.l_extent.end,
 299                                           mode_from_vma(vma));
 300                 if (IS_ERR(node)) {
 301                         CERROR("not enough mem for lock_tree_node!\n");
 302                         RETURN(-ENOMEM);
 303                 }
 304                 lt_insert(tree, node);
 305
 306                 if (vma->vm_end - addr >= count)
 307                         break;
 308                 count -= vma->vm_end - addr;
 309                 addr = vma->vm_end;
 310         }
 311         RETURN(0);
 312 }
 313 /* FIXME: there is a pagefault race goes as follow:
 314  * 1. A user process on node A accesses a portion of a mapped file,
 315  *    resulting in a page fault.  The pagefault handler invokes the
 316  *    ll_nopage function, which reads the page into memory.
 317  * 2. A user process on node B writes to the same portion of the file
 318  *    (either via mmap or write()), that cause node A to cancel the
 319  *    lock and truncate the page.
 320  * 3. Node A then executes the rest of do_no_page(), entering the
 321  *    now-invalid page into the PTEs.
 322  *
 323  * Make the whole do_no_page as a hook to cover both the page cache
 324  * and page mapping installing with dlm lock would eliminate this race.
 325  */
 326 #if (LINUX_VERSION_CODE >= KERNEL_VERSION(2,5,0))
 327 struct page *ll_nopage(struct vm_area_struct *vma, unsigned long address,
 328                        int *type)
 329 #else
 330 struct page *ll_nopage(struct vm_area_struct *vma, unsigned long address,
 331                        int unused)
 332 #endif
 333 {
 334         struct file *filp = vma->vm_file;
 335         struct ll_file_data *fd = filp->private_data;
 336         struct inode *inode = filp->f_dentry->d_inode;
 337         struct lustre_handle lockh = { 0 };
 338         ldlm_policy_data_t policy;
 339         ldlm_mode_t mode;
 340         struct page *page;
 341         struct obd_service_time *stime;
 342         __u64 kms;
 343         unsigned long pgoff, size, rand_read, seq_read;
 344         int rc = 0;
 345         ENTRY;
 346
 347         if (ll_i2info(inode)->lli_smd == NULL) {
 348                 CERROR("No lsm on fault?\n");
 349                 RETURN(NULL);
 350         }
 351
 352         /* start and end the lock on the first and last bytes in the page */
 353         policy_from_vma(&policy, vma, address, PAGE_CACHE_SIZE);
 354
 355         CDEBUG(D_MMAP, "nopage vma %p inode %lu, locking ["LPU64", "LPU64"]\n",
 356                vma, inode->i_ino, policy.l_extent.start,
 357                policy.l_extent.end);
 358
 359         mode = mode_from_vma(vma);
 360         stime = (mode & LCK_PW) ? &ll_i2sbi(inode)->ll_write_stime :
 361                                   &ll_i2sbi(inode)->ll_read_stime;
 362
 363         rc = ll_extent_lock(fd, inode, ll_i2info(inode)->lli_smd, mode, &policy,
 364                             &lockh, LDLM_FL_CBPENDING, stime);
 365         if (rc != 0)
 366                 RETURN(NULL);
 367
 368         /* XXX change inode size without i_sem hold! there is a race condition
 369          *     with truncate path. (see ll_extent_lock) */
 370         kms = lov_merge_size(ll_i2info(inode)->lli_smd, 1);
 371         pgoff = ((address - vma->vm_start) >> PAGE_CACHE_SHIFT) + vma->vm_pgoff;
 372         size = (kms + PAGE_CACHE_SIZE - 1) >> PAGE_CACHE_SHIFT;
 373
 374         if (pgoff >= size)
 375                 ll_glimpse_size(inode);
 376         else
 377                 inode->i_size = kms;
 378
 379         /* disable VM_SEQ_READ and use VM_RAND_READ to make sure that
 380          * the kernel will not read other pages not covered by ldlm in
 381          * filemap_nopage. we do our readahead in ll_readpage.
 382          */
 383         rand_read = vma->vm_flags & VM_RAND_READ;
 384         seq_read = vma->vm_flags & VM_SEQ_READ;
 385         vma->vm_flags &= ~ VM_SEQ_READ;
 386         vma->vm_flags |= VM_RAND_READ;
 387
 388 #if (LINUX_VERSION_CODE >= KERNEL_VERSION(2,5,0))
 389         page = filemap_nopage(vma, address, type);
 390 #else
 391         page = filemap_nopage(vma, address, unused);
 392 #endif
 393         vma->vm_flags &= ~VM_RAND_READ;
 394         vma->vm_flags |= (rand_read | seq_read);
 395
 396         ll_extent_unlock(fd, inode, ll_i2info(inode)->lli_smd, mode, &lockh);
 397         RETURN(page);
 398 }
 399
 400 /* return the user space pointer that maps to a file offset via a vma */
 401 static inline unsigned long file_to_user(struct vm_area_struct *vma,
 402                                          __u64 byte)
 403 {
 404         return vma->vm_start +
 405                (byte - ((__u64)vma->vm_pgoff << PAGE_CACHE_SHIFT));
 406
 407 }
 408
 409 #define VMA_DEBUG(vma, fmt, arg...)                                          \
 410         CDEBUG(D_MMAP, "vma(%p) start(%ld) end(%ld) pgoff(%ld) inode(%p): "  \
 411                fmt, vma, vma->vm_start, vma->vm_end, vma->vm_pgoff,          \
 412                vma->vm_file->f_dentry->d_inode, ## arg);
 413
 414 #if (LINUX_VERSION_CODE < KERNEL_VERSION(2,5,0))
 415 /* [first, last] are the byte offsets affected.
 416  * vm_{start, end} are user addresses of the first byte of the mapping and
 417  *      the next byte beyond it
 418  * vm_pgoff is the page index of the first byte in the mapping */
 419 static void teardown_vmas(struct vm_area_struct *vma, __u64 first,
 420                           __u64 last)
 421 {
 422         unsigned long address, len;
 423         for (; vma ; vma = vma->vm_next_share) {
 424                 if (last >> PAGE_CACHE_SHIFT < vma->vm_pgoff)
 425                         continue;
 426                 if (first >> PAGE_CACHE_SHIFT > (vma->vm_pgoff +
 427                     ((vma->vm_end - vma->vm_start) >> PAGE_CACHE_SHIFT)))
 428                         continue;
 429
 430                 address = max((unsigned long)vma->vm_start,
 431                               file_to_user(vma, first));
 432                 len = min((unsigned long)vma->vm_end,
 433                           file_to_user(vma, last) + 1) - address;
 434
 435                 VMA_DEBUG(vma, "zapping vma [address=%ld len=%ld]\n",
 436                           address, len);
 437                 LASSERT(vma->vm_mm);
 438                 ll_zap_page_range(vma, address, len);
 439         }
 440 }
 441 #endif
 442
 443 /* XXX put nice comment here.  talk about __free_pte -> dirty pages and
 444  * nopage's reference passing to the pte */
 445 int ll_teardown_mmaps(struct address_space *mapping, __u64 first,
 446                        __u64 last)
 447 {
 448         int rc = -ENOENT;
 449         ENTRY;
 450
 451 #if (LINUX_VERSION_CODE >= KERNEL_VERSION(2,5,0))
 452         if (mapping_mapped(mapping)) {
 453                 rc = 0;
 454                 unmap_mapping_range(mapping, first + PAGE_SIZE - 1,
 455                                     last - first + 1, 1);
 456         }
 457 #else
 458         spin_lock(&mapping->i_shared_lock);
 459         if (mapping->i_mmap != NULL) {
 460                 rc = 0;
 461                 teardown_vmas(mapping->i_mmap, first, last);
 462         }
 463         if (mapping->i_mmap_shared != NULL) {
 464                 rc = 0;
 465                 teardown_vmas(mapping->i_mmap_shared, first, last);
 466         }
 467         spin_unlock(&mapping->i_shared_lock);
 468 #endif
 469
 470         RETURN(rc);
 471 }
 472
 473 static struct vm_operations_struct ll_file_vm_ops = {
 474         .nopage         = ll_nopage,
 475 };
 476
 477 int ll_file_mmap(struct file * file, struct vm_area_struct * vma)
 478 {
 479         int rc;
 480         ENTRY;
 481
 482         rc = generic_file_mmap(file, vma);
 483         if (rc == 0)
 484                 vma->vm_ops = &ll_file_vm_ops;
 485
 486         RETURN(rc);
 487 }
 488