X-Git-Url: https://git.whamcloud.com/?p=fs%2Flustre-release.git;a=blobdiff_plain;f=lustre%2Fosd-ldiskfs%2Fosd_io.c;h=8c32812d8f06e8cd184834c9e335b5366523248a;hp=1de75ab98869624398a02cbccd3752195541d58d;hb=c65a2c2a41f4fb9b91f25d4ac073364258691d1b;hpb=25a36d277be36c5143a7583d4c5c46380afa0fdc diff --git a/lustre/osd-ldiskfs/osd_io.c b/lustre/osd-ldiskfs/osd_io.c index 1de75ab9..8c32812 100644 --- a/lustre/osd-ldiskfs/osd_io.c +++ b/lustre/osd-ldiskfs/osd_io.c @@ -27,7 +27,7 @@ * Copyright (c) 2010, Oracle and/or its affiliates. All rights reserved. * Use is subject to license terms. * - * Copyright (c) 2011, 2012, Intel Corporation. + * Copyright (c) 2012, 2014, Intel Corporation. */ /* * This file is part of Lustre, http://www.lustre.org/ @@ -60,37 +60,6 @@ /* ext_depth() */ #include -#ifndef HAVE_PAGE_CONSTANT -#define mapping_cap_page_constant_write(mapping) 0 -#define SetPageConstant(page) do {} while (0) -#define ClearPageConstant(page) do {} while (0) -#endif - -#ifndef HAS_GENERIC_ERROR_REMOVE_PAGE -int generic_error_remove_page(struct address_space *mapping, struct page *page) -{ - if (mapping == NULL) - return -EINVAL; - - if (mapping != page->mapping) - return -EIO; - /* - * Only punch for normal data pages for now. - * Handling other types like directories would need more auditing. - */ - if (!S_ISREG(mapping->host->i_mode)) - return -EIO; - - if (page_mapped(page)) { - unmap_mapping_range(mapping, - (loff_t)page->index << PAGE_CACHE_SHIFT, - PAGE_CACHE_SIZE, 0); - } - truncate_complete_page(mapping, page); - return 0; -} -#endif - static int __osd_init_iobuf(struct osd_device *d, struct osd_iobuf *iobuf, int rw, int line, int pages) { @@ -98,22 +67,22 @@ static int __osd_init_iobuf(struct osd_device *d, struct osd_iobuf *iobuf, LASSERTF(iobuf->dr_elapsed_valid == 0, "iobuf %p, reqs %d, rw %d, line %d\n", iobuf, - cfs_atomic_read(&iobuf->dr_numreqs), iobuf->dr_rw, + atomic_read(&iobuf->dr_numreqs), iobuf->dr_rw, iobuf->dr_init_at); LASSERT(pages <= PTLRPC_MAX_BRW_PAGES); - cfs_waitq_init(&iobuf->dr_wait); - cfs_atomic_set(&iobuf->dr_numreqs, 0); - iobuf->dr_npages = 0; - iobuf->dr_error = 0; - iobuf->dr_dev = d; - iobuf->dr_frags = 0; - iobuf->dr_elapsed = 0; - /* must be counted before, so assert */ - iobuf->dr_rw = rw; + init_waitqueue_head(&iobuf->dr_wait); + atomic_set(&iobuf->dr_numreqs, 0); + iobuf->dr_npages = 0; + iobuf->dr_error = 0; + iobuf->dr_dev = d; + iobuf->dr_frags = 0; + iobuf->dr_elapsed = 0; + /* must be counted before, so assert */ + iobuf->dr_rw = rw; iobuf->dr_init_at = line; - blocks = pages * (CFS_PAGE_SIZE >> osd_sb(d)->s_blocksize_bits); + blocks = pages * (PAGE_CACHE_SIZE >> osd_sb(d)->s_blocksize_bits); if (iobuf->dr_bl_buf.lb_len >= blocks * sizeof(iobuf->dr_blocks[0])) { LASSERT(iobuf->dr_pg_buf.lb_len >= pages * sizeof(iobuf->dr_pages[0])); @@ -128,7 +97,7 @@ static int __osd_init_iobuf(struct osd_device *d, struct osd_iobuf *iobuf, CDEBUG(D_OTHER, "realloc %u for %u (%u) pages\n", (unsigned)(pages * sizeof(iobuf->dr_pages[0])), i, pages); pages = i; - blocks = pages * (CFS_PAGE_SIZE >> osd_sb(d)->s_blocksize_bits); + blocks = pages * (PAGE_CACHE_SIZE >> osd_sb(d)->s_blocksize_bits); iobuf->dr_max_pages = 0; CDEBUG(D_OTHER, "realloc %u for %u blocks\n", (unsigned)(blocks * sizeof(iobuf->dr_blocks[0])), blocks); @@ -176,63 +145,53 @@ void osd_fini_iobuf(struct osd_device *d, struct osd_iobuf *iobuf) #define __REQ_WRITE BIO_RW #endif -#ifdef HAVE_BIO_ENDIO_2ARG -#define DIO_RETURN(a) static void dio_complete_routine(struct bio *bio, int error) +{ + struct osd_iobuf *iobuf = bio->bi_private; +#ifdef HAVE_BVEC_ITER + struct bvec_iter iter; + struct bio_vec bvl; #else -#define DIO_RETURN(a) return(a) -static int dio_complete_routine(struct bio *bio, unsigned int done, int error) + int iter; + struct bio_vec *bvl; #endif -{ - struct osd_iobuf *iobuf = bio->bi_private; - struct bio_vec *bvl; - int i; /* CAVEAT EMPTOR: possibly in IRQ context * DO NOT record procfs stats here!!! */ - if (unlikely(iobuf == NULL)) { - CERROR("***** bio->bi_private is NULL! This should never " - "happen. Normally, I would crash here, but instead I " - "will dump the bio contents to the console. Please " - "report this to , along " - "with any interesting messages leading up to this point " - "(like SCSI errors, perhaps). Because bi_private is " - "NULL, I can't wake up the thread that initiated this " - "IO - you will probably have to reboot this node.\n"); - CERROR("bi_next: %p, bi_flags: %lx, bi_rw: %lu, bi_vcnt: %d, " - "bi_idx: %d, bi->size: %d, bi_end_io: %p, bi_cnt: %d, " - "bi_private: %p\n", bio->bi_next, bio->bi_flags, - bio->bi_rw, bio->bi_vcnt, bio->bi_idx, bio->bi_size, - bio->bi_end_io, cfs_atomic_read(&bio->bi_cnt), - bio->bi_private); - DIO_RETURN(0); - } + if (unlikely(iobuf == NULL)) { + CERROR("***** bio->bi_private is NULL! This should never " + "happen. Normally, I would crash here, but instead I " + "will dump the bio contents to the console. Please " + "report this to , along " + "with any interesting messages leading up to this point " + "(like SCSI errors, perhaps). Because bi_private is " + "NULL, I can't wake up the thread that initiated this " + "IO - you will probably have to reboot this node.\n"); + CERROR("bi_next: %p, bi_flags: %lx, bi_rw: %lu, bi_vcnt: %d, " + "bi_idx: %d, bi->size: %d, bi_end_io: %p, bi_cnt: %d, " + "bi_private: %p\n", bio->bi_next, bio->bi_flags, + bio->bi_rw, bio->bi_vcnt, bio_idx(bio), + bio_sectors(bio) << 9, bio->bi_end_io, + atomic_read(&bio->bi_cnt), bio->bi_private); + return; + } - /* the check is outside of the cycle for performance reason -bzzz */ + /* the check is outside of the cycle for performance reason -bzzz */ if (!test_bit(__REQ_WRITE, &bio->bi_rw)) { - bio_for_each_segment(bvl, bio, i) { - if (likely(error == 0)) - SetPageUptodate(bvl->bv_page); - LASSERT(PageLocked(bvl->bv_page)); - ClearPageConstant(bvl->bv_page); - } - cfs_atomic_dec(&iobuf->dr_dev->od_r_in_flight); - } else { - struct page *p = iobuf->dr_pages[0]; - if (p->mapping) { - if (mapping_cap_page_constant_write(p->mapping)) { - bio_for_each_segment(bvl, bio, i) { - ClearPageConstant(bvl->bv_page); - } - } - } - cfs_atomic_dec(&iobuf->dr_dev->od_w_in_flight); - } + bio_for_each_segment(bvl, bio, iter) { + if (likely(error == 0)) + SetPageUptodate(bvec_iter_page(&bvl, iter)); + LASSERT(PageLocked(bvec_iter_page(&bvl, iter))); + } + atomic_dec(&iobuf->dr_dev->od_r_in_flight); + } else { + atomic_dec(&iobuf->dr_dev->od_w_in_flight); + } - /* any real error is good enough -bzzz */ - if (error != 0 && iobuf->dr_error == 0) - iobuf->dr_error = error; + /* any real error is good enough -bzzz */ + if (error != 0 && iobuf->dr_error == 0) + iobuf->dr_error = error; /* * set dr_elapsed before dr_numreqs turns to 0, otherwise @@ -241,43 +200,42 @@ static int dio_complete_routine(struct bio *bio, unsigned int done, int error) * data in this processing and an assertion in a subsequent * call to OSD. */ - if (cfs_atomic_read(&iobuf->dr_numreqs) == 1) { + if (atomic_read(&iobuf->dr_numreqs) == 1) { iobuf->dr_elapsed = jiffies - iobuf->dr_start_time; iobuf->dr_elapsed_valid = 1; } - if (cfs_atomic_dec_and_test(&iobuf->dr_numreqs)) - cfs_waitq_signal(&iobuf->dr_wait); - - /* Completed bios used to be chained off iobuf->dr_bios and freed in - * filter_clear_dreq(). It was then possible to exhaust the biovec-256 - * mempool when serious on-disk fragmentation was encountered, - * deadlocking the OST. The bios are now released as soon as complete - * so the pool cannot be exhausted while IOs are competing. bug 10076 */ - bio_put(bio); - DIO_RETURN(0); + if (atomic_dec_and_test(&iobuf->dr_numreqs)) + wake_up(&iobuf->dr_wait); + + /* Completed bios used to be chained off iobuf->dr_bios and freed in + * filter_clear_dreq(). It was then possible to exhaust the biovec-256 + * mempool when serious on-disk fragmentation was encountered, + * deadlocking the OST. The bios are now released as soon as complete + * so the pool cannot be exhausted while IOs are competing. bug 10076 */ + bio_put(bio); } static void record_start_io(struct osd_iobuf *iobuf, int size) { - struct osd_device *osd = iobuf->dr_dev; - struct obd_histogram *h = osd->od_brw_stats.hist; - - iobuf->dr_frags++; - cfs_atomic_inc(&iobuf->dr_numreqs); - - if (iobuf->dr_rw == 0) { - cfs_atomic_inc(&osd->od_r_in_flight); - lprocfs_oh_tally(&h[BRW_R_RPC_HIST], - cfs_atomic_read(&osd->od_r_in_flight)); - lprocfs_oh_tally_log2(&h[BRW_R_DISK_IOSIZE], size); - } else if (iobuf->dr_rw == 1) { - cfs_atomic_inc(&osd->od_w_in_flight); - lprocfs_oh_tally(&h[BRW_W_RPC_HIST], - cfs_atomic_read(&osd->od_w_in_flight)); - lprocfs_oh_tally_log2(&h[BRW_W_DISK_IOSIZE], size); - } else { - LBUG(); - } + struct osd_device *osd = iobuf->dr_dev; + struct obd_histogram *h = osd->od_brw_stats.hist; + + iobuf->dr_frags++; + atomic_inc(&iobuf->dr_numreqs); + + if (iobuf->dr_rw == 0) { + atomic_inc(&osd->od_r_in_flight); + lprocfs_oh_tally(&h[BRW_R_RPC_HIST], + atomic_read(&osd->od_r_in_flight)); + lprocfs_oh_tally_log2(&h[BRW_R_DISK_IOSIZE], size); + } else if (iobuf->dr_rw == 1) { + atomic_inc(&osd->od_w_in_flight); + lprocfs_oh_tally(&h[BRW_W_RPC_HIST], + atomic_read(&osd->od_w_in_flight)); + lprocfs_oh_tally_log2(&h[BRW_W_DISK_IOSIZE], size); + } else { + LBUG(); + } } static void osd_submit_bio(int rw, struct bio *bio) @@ -291,19 +249,16 @@ static void osd_submit_bio(int rw, struct bio *bio) static int can_be_merged(struct bio *bio, sector_t sector) { - unsigned int size; - - if (!bio) - return 0; + if (bio == NULL) + return 0; - size = bio->bi_size >> 9; - return bio->bi_sector + size == sector ? 1 : 0; + return bio_end_sector(bio) == sector ? 1 : 0; } static int osd_do_bio(struct osd_device *osd, struct inode *inode, struct osd_iobuf *iobuf) { - int blocks_per_page = CFS_PAGE_SIZE >> inode->i_blkbits; + int blocks_per_page = PAGE_CACHE_SIZE >> inode->i_blkbits; struct page **pages = iobuf->dr_pages; int npages = iobuf->dr_npages; unsigned long *blocks = iobuf->dr_blocks; @@ -357,39 +312,29 @@ static int osd_do_bio(struct osd_device *osd, struct inode *inode, sector_bits)) nblocks++; - /* I only set the page to be constant only if it - * is mapped to a contiguous underlying disk block(s). - * It will then make sure the corresponding device - * cache of raid5 will be overwritten by this page. - * - jay */ - if (iobuf->dr_rw && (nblocks == blocks_per_page) && - mapping_cap_page_constant_write(inode->i_mapping)) - SetPageConstant(page); - if (bio != NULL && can_be_merged(bio, sector) && bio_add_page(bio, page, blocksize * nblocks, page_offset) != 0) continue; /* added this frag OK */ - if (bio != NULL) { - struct request_queue *q = - bdev_get_queue(bio->bi_bdev); - - /* Dang! I have to fragment this I/O */ - CDEBUG(D_INODE, "bio++ sz %d vcnt %d(%d) " - "sectors %d(%d) psg %d(%d) hsg %d(%d)\n", - bio->bi_size, - bio->bi_vcnt, bio->bi_max_vecs, - bio->bi_size >> 9, queue_max_sectors(q), + if (bio != NULL) { + struct request_queue *q = + bdev_get_queue(bio->bi_bdev); + unsigned int bi_size = bio_sectors(bio) << 9; + + /* Dang! I have to fragment this I/O */ + CDEBUG(D_INODE, "bio++ sz %d vcnt %d(%d) " + "sectors %d(%d) psg %d(%d) hsg %d(%d)\n", + bi_size, bio->bi_vcnt, bio->bi_max_vecs, + bio_sectors(bio), + queue_max_sectors(q), bio_phys_segments(q, bio), queue_max_phys_segments(q), - bio_hw_segments(q, bio), - queue_max_hw_segments(q)); - - record_start_io(iobuf, bio->bi_size); - osd_submit_bio(iobuf->dr_rw, bio); - } + 0, queue_max_hw_segments(q)); + record_start_io(iobuf, bi_size); + osd_submit_bio(iobuf->dr_rw, bio); + } /* allocate new bio */ bio = bio_alloc(GFP_NOIO, min(BIO_MAX_PAGES, @@ -403,37 +348,38 @@ static int osd_do_bio(struct osd_device *osd, struct inode *inode, goto out; } - bio->bi_bdev = inode->i_sb->s_bdev; - bio->bi_sector = sector; + bio->bi_bdev = inode->i_sb->s_bdev; + bio_set_sector(bio, sector); bio->bi_rw = (iobuf->dr_rw == 0) ? READ : WRITE; - bio->bi_end_io = dio_complete_routine; - bio->bi_private = iobuf; + bio->bi_end_io = dio_complete_routine; + bio->bi_private = iobuf; - rc = bio_add_page(bio, page, - blocksize * nblocks, page_offset); - LASSERT(rc != 0); - } - } + rc = bio_add_page(bio, page, + blocksize * nblocks, page_offset); + LASSERT(rc != 0); + } + } - if (bio != NULL) { - record_start_io(iobuf, bio->bi_size); - osd_submit_bio(iobuf->dr_rw, bio); - rc = 0; - } + if (bio != NULL) { + record_start_io(iobuf, bio_sectors(bio) << 9); + osd_submit_bio(iobuf->dr_rw, bio); + rc = 0; + } - out: - /* in order to achieve better IO throughput, we don't wait for writes - * completion here. instead we proceed with transaction commit in - * parallel and wait for IO completion once transaction is stopped - * see osd_trans_stop() for more details -bzzz */ - if (iobuf->dr_rw == 0) { - cfs_wait_event(iobuf->dr_wait, - cfs_atomic_read(&iobuf->dr_numreqs) == 0); - } +out: + /* in order to achieve better IO throughput, we don't wait for writes + * completion here. instead we proceed with transaction commit in + * parallel and wait for IO completion once transaction is stopped + * see osd_trans_stop() for more details -bzzz */ + if (iobuf->dr_rw == 0) { + wait_event(iobuf->dr_wait, + atomic_read(&iobuf->dr_numreqs) == 0); + osd_fini_iobuf(osd, iobuf); + } - if (rc == 0) - rc = iobuf->dr_error; - RETURN(rc); + if (rc == 0) + rc = iobuf->dr_error; + RETURN(rc); } static int osd_map_remote_to_local(loff_t offset, ssize_t len, int *nrpages, @@ -444,18 +390,18 @@ static int osd_map_remote_to_local(loff_t offset, ssize_t len, int *nrpages, *nrpages = 0; while (len > 0) { - int poff = offset & (CFS_PAGE_SIZE - 1); - int plen = CFS_PAGE_SIZE - poff; + int poff = offset & (PAGE_CACHE_SIZE - 1); + int plen = PAGE_CACHE_SIZE - poff; if (plen > len) plen = len; lnb->lnb_file_offset = offset; lnb->lnb_page_offset = poff; - lnb->len = plen; - /* lb->flags = rnb->flags; */ - lnb->flags = 0; - lnb->page = NULL; - lnb->rc = 0; + lnb->lnb_len = plen; + /* lnb->lnb_flags = rnb->rnb_flags; */ + lnb->lnb_flags = 0; + lnb->lnb_page = NULL; + lnb->lnb_rc = 0; LASSERTF(plen <= len, "plen %u, len %lld\n", plen, (long long) len); @@ -468,7 +414,7 @@ static int osd_map_remote_to_local(loff_t offset, ssize_t len, int *nrpages, RETURN(0); } -struct page *osd_get_page(struct dt_object *dt, loff_t offset, int rw) +static struct page *osd_get_page(struct dt_object *dt, loff_t offset, int rw) { struct inode *inode = osd_dt_obj(dt)->oo_inode; struct osd_device *d = osd_obj2dev(osd_dt_obj(dt)); @@ -476,7 +422,7 @@ struct page *osd_get_page(struct dt_object *dt, loff_t offset, int rw) LASSERT(inode); - page = find_or_create_page(inode->i_mapping, offset >> CFS_PAGE_SHIFT, + page = find_or_create_page(inode->i_mapping, offset >> PAGE_CACHE_SHIFT, GFP_NOFS | __GFP_HIGHMEM); if (unlikely(page == NULL)) lprocfs_counter_add(d->od_stats, LPROC_OSD_NO_PAGE, 1); @@ -489,83 +435,538 @@ struct page *osd_get_page(struct dt_object *dt, loff_t offset, int rw) * journal_start * i_mutex * page lock + * + * osd write path: + * - lock page(s) + * - journal_start + * - truncate_sem + * + * ext4 vmtruncate: + * - lock pages, unlock + * - journal_start + * - lock partial page + * - i_data_sem + * + */ + +/** + * Unlock and release pages loaded by osd_bufs_get() + * + * Unlock \a npages pages from \a lnb and drop the refcount on them. + * + * \param env thread execution environment + * \param dt dt object undergoing IO (OSD object + methods) + * \param lnb array of pages undergoing IO + * \param npages number of pages in \a lnb + * + * \retval 0 always + */ +static int osd_bufs_put(const struct lu_env *env, struct dt_object *dt, + struct niobuf_local *lnb, int npages) +{ + int i; - * osd write path - * lock page(s) - * journal_start - * truncate_sem + for (i = 0; i < npages; i++) { + if (lnb[i].lnb_page == NULL) + continue; + LASSERT(PageLocked(lnb[i].lnb_page)); + unlock_page(lnb[i].lnb_page); + page_cache_release(lnb[i].lnb_page); + lu_object_put(env, &dt->do_lu); + lnb[i].lnb_page = NULL; + } - * ext4 vmtruncate: - * lock pages, unlock - * journal_start - * lock partial page - * i_data_sem - -*/ -int osd_bufs_get(const struct lu_env *env, struct dt_object *d, loff_t pos, - ssize_t len, struct niobuf_local *lnb, int rw, - struct lustre_capa *capa) + RETURN(0); +} + +/** + * Load and lock pages undergoing IO + * + * Pages as described in the \a lnb array are fetched (from disk or cache) + * and locked for IO by the caller. + * + * DLM locking protects us from write and truncate competing for same region, + * but partial-page truncate can leave dirty pages in the cache for ldiskfs. + * It's possible the writeout on a such a page is in progress when we access + * it. It's also possible that during this writeout we put new (partial) data + * into the page, but won't be able to proceed in filter_commitrw_write(). + * Therefore, just wait for writeout completion as it should be rare enough. + * + * \param env thread execution environment + * \param dt dt object undergoing IO (OSD object + methods) + * \param pos byte offset of IO start + * \param len number of bytes of IO + * \param lnb array of extents undergoing IO + * \param rw read or write operation? + * \param capa capabilities + * + * \retval pages (zero or more) loaded successfully + * \retval -ENOMEM on memory/page allocation error + */ +static int osd_bufs_get(const struct lu_env *env, struct dt_object *dt, + loff_t pos, ssize_t len, struct niobuf_local *lnb, + int rw) { - struct osd_object *obj = osd_dt_obj(d); - int npages, i, rc = 0; + struct osd_object *obj = osd_dt_obj(dt); + int npages, i, rc = 0; - LASSERT(obj->oo_inode); + LASSERT(obj->oo_inode); - osd_map_remote_to_local(pos, len, &npages, lnb); + osd_map_remote_to_local(pos, len, &npages, lnb); - for (i = 0; i < npages; i++, lnb++) { + for (i = 0; i < npages; i++, lnb++) { + lnb->lnb_page = osd_get_page(dt, lnb->lnb_file_offset, rw); + if (lnb->lnb_page == NULL) + GOTO(cleanup, rc = -ENOMEM); - /* We still set up for ungranted pages so that granted pages - * can be written to disk as they were promised, and portals - * needs to keep the pages all aligned properly. */ - lnb->dentry = (void *) obj; + wait_on_page_writeback(lnb->lnb_page); + BUG_ON(PageWriteback(lnb->lnb_page)); - lnb->page = osd_get_page(d, lnb->lnb_file_offset, rw); - if (lnb->page == NULL) - GOTO(cleanup, rc = -ENOMEM); + lu_object_get(&dt->do_lu); + } - /* DLM locking protects us from write and truncate competing - * for same region, but truncate can leave dirty page in the - * cache. it's possible the writeout on a such a page is in - * progress when we access it. it's also possible that during - * this writeout we put new (partial) data, but then won't - * be able to proceed in filter_commitrw_write(). thus let's - * just wait for writeout completion, should be rare enough. - * -bzzz */ - wait_on_page_writeback(lnb->page); - BUG_ON(PageWriteback(lnb->page)); + RETURN(i); - lu_object_get(&d->do_lu); - } - rc = i; +cleanup: + if (i > 0) + osd_bufs_put(env, dt, lnb - i, i); + return rc; +} + +#ifndef HAVE_LDISKFS_MAP_BLOCKS + +#ifdef HAVE_EXT_PBLOCK /* Name changed to ext4_ext_pblock for kernel 2.6.35 */ +#define ldiskfs_ext_pblock(ex) ext_pblock((ex)) +#endif + +struct bpointers { + unsigned long *blocks; + unsigned long start; + int num; + int init_num; + int create; +}; + +static long ldiskfs_ext_find_goal(struct inode *inode, + struct ldiskfs_ext_path *path, + unsigned long block, int *aflags) +{ + struct ldiskfs_inode_info *ei = LDISKFS_I(inode); + unsigned long bg_start; + unsigned long colour; + int depth; + + if (path) { + struct ldiskfs_extent *ex; + depth = path->p_depth; + + /* try to predict block placement */ + if ((ex = path[depth].p_ext)) + return ldiskfs_ext_pblock(ex) + + (block - le32_to_cpu(ex->ee_block)); + + /* it looks index is empty + * try to find starting from index itself */ + if (path[depth].p_bh) + return path[depth].p_bh->b_blocknr; + } + + /* OK. use inode's group */ + bg_start = (ei->i_block_group * LDISKFS_BLOCKS_PER_GROUP(inode->i_sb)) + + le32_to_cpu(LDISKFS_SB(inode->i_sb)->s_es->s_first_data_block); + colour = (current->pid % 16) * + (LDISKFS_BLOCKS_PER_GROUP(inode->i_sb) / 16); + return bg_start + colour + block; +} + +static unsigned long new_blocks(handle_t *handle, struct inode *inode, + struct ldiskfs_ext_path *path, + unsigned long block, unsigned long *count, + int *err) +{ + struct ldiskfs_allocation_request ar; + unsigned long pblock; + int aflags; + + /* find neighbour allocated blocks */ + ar.lleft = block; + *err = ldiskfs_ext_search_left(inode, path, &ar.lleft, &ar.pleft); + if (*err) + return 0; + ar.lright = block; + *err = ldiskfs_ext_search_right(inode, path, &ar.lright, &ar.pright); + if (*err) + return 0; + + /* allocate new block */ + ar.goal = ldiskfs_ext_find_goal(inode, path, block, &aflags); + ar.inode = inode; + ar.logical = block; + ar.len = *count; + ar.flags = LDISKFS_MB_HINT_DATA; + pblock = ldiskfs_mb_new_blocks(handle, &ar, err); + *count = ar.len; + return pblock; +} + +static int ldiskfs_ext_new_extent_cb(struct inode *inode, + struct ldiskfs_ext_path *path, + struct ldiskfs_ext_cache *cex, +#ifdef HAVE_EXT_PREPARE_CB_EXTENT + struct ldiskfs_extent *ex, +#endif + void *cbdata) +{ + struct bpointers *bp = cbdata; + struct ldiskfs_extent nex; + unsigned long pblock; + unsigned long tgen; + int err, i; + unsigned long count; + handle_t *handle; + +#ifdef LDISKFS_EXT_CACHE_EXTENT /* until kernel 2.6.37 */ + if (cex->ec_type == LDISKFS_EXT_CACHE_EXTENT) { +#else + if ((cex->ec_len != 0) && (cex->ec_start != 0)) { +#endif + err = EXT_CONTINUE; + goto map; + } + + if (bp->create == 0) { + i = 0; + if (cex->ec_block < bp->start) + i = bp->start - cex->ec_block; + if (i >= cex->ec_len) + CERROR("nothing to do?! i = %d, e_num = %u\n", + i, cex->ec_len); + for (; i < cex->ec_len && bp->num; i++) { + *(bp->blocks) = 0; + bp->blocks++; + bp->num--; + bp->start++; + } + + return EXT_CONTINUE; + } + + tgen = LDISKFS_I(inode)->i_ext_generation; + count = ldiskfs_ext_calc_credits_for_insert(inode, path); + + handle = osd_journal_start(inode, LDISKFS_HT_MISC, + count + LDISKFS_ALLOC_NEEDED + 1); + if (IS_ERR(handle)) { + return PTR_ERR(handle); + } + + if (tgen != LDISKFS_I(inode)->i_ext_generation) { + /* the tree has changed. so path can be invalid at moment */ + ldiskfs_journal_stop(handle); + return EXT_REPEAT; + } + + /* In 2.6.32 kernel, ldiskfs_ext_walk_space()'s callback func is not + * protected by i_data_sem as whole. so we patch it to store + * generation to path and now verify the tree hasn't changed */ + down_write((&LDISKFS_I(inode)->i_data_sem)); + + /* validate extent, make sure the extent tree does not changed */ + if (LDISKFS_I(inode)->i_ext_generation != path[0].p_generation) { + /* cex is invalid, try again */ + up_write(&LDISKFS_I(inode)->i_data_sem); + ldiskfs_journal_stop(handle); + return EXT_REPEAT; + } + count = cex->ec_len; + pblock = new_blocks(handle, inode, path, cex->ec_block, &count, &err); + if (!pblock) + goto out; + BUG_ON(count > cex->ec_len); + + /* insert new extent */ + nex.ee_block = cpu_to_le32(cex->ec_block); + ldiskfs_ext_store_pblock(&nex, pblock); + nex.ee_len = cpu_to_le16(count); + err = ldiskfs_ext_insert_extent(handle, inode, path, &nex, 0); + if (err) { + /* free data blocks we just allocated */ + /* not a good idea to call discard here directly, + * but otherwise we'd need to call it every free() */ + ldiskfs_discard_preallocations(inode); +#ifdef HAVE_EXT_FREE_BLOCK_WITH_BUFFER_HEAD /* Introduced in 2.6.32-rc7 */ + ldiskfs_free_blocks(handle, inode, NULL, + ldiskfs_ext_pblock(&nex), + le16_to_cpu(nex.ee_len), 0); +#else + ldiskfs_free_blocks(handle, inode, ldiskfs_ext_pblock(&nex), + le16_to_cpu(nex.ee_len), 0); +#endif + goto out; + } + + /* + * Putting len of the actual extent we just inserted, + * we are asking ldiskfs_ext_walk_space() to continue + * scaning after that block + */ + cex->ec_len = le16_to_cpu(nex.ee_len); + cex->ec_start = ldiskfs_ext_pblock(&nex); + BUG_ON(le16_to_cpu(nex.ee_len) == 0); + BUG_ON(le32_to_cpu(nex.ee_block) != cex->ec_block); + +out: + up_write((&LDISKFS_I(inode)->i_data_sem)); + ldiskfs_journal_stop(handle); +map: + if (err >= 0) { + /* map blocks */ + if (bp->num == 0) { + CERROR("hmm. why do we find this extent?\n"); + CERROR("initial space: %lu:%u\n", + bp->start, bp->init_num); +#ifdef LDISKFS_EXT_CACHE_EXTENT /* until kernel 2.6.37 */ + CERROR("current extent: %u/%u/%llu %d\n", + cex->ec_block, cex->ec_len, + (unsigned long long)cex->ec_start, + cex->ec_type); +#else + CERROR("current extent: %u/%u/%llu\n", + cex->ec_block, cex->ec_len, + (unsigned long long)cex->ec_start); +#endif + } + i = 0; + if (cex->ec_block < bp->start) + i = bp->start - cex->ec_block; + if (i >= cex->ec_len) + CERROR("nothing to do?! i = %d, e_num = %u\n", + i, cex->ec_len); + for (; i < cex->ec_len && bp->num; i++) { + *(bp->blocks) = cex->ec_start + i; +#ifdef LDISKFS_EXT_CACHE_EXTENT /* until kernel 2.6.37 */ + if (cex->ec_type != LDISKFS_EXT_CACHE_EXTENT) { +#else + if ((cex->ec_len == 0) || (cex->ec_start == 0)) { +#endif + /* unmap any possible underlying metadata from + * the block device mapping. bug 6998. */ + unmap_underlying_metadata(inode->i_sb->s_bdev, + *(bp->blocks)); + } + bp->blocks++; + bp->num--; + bp->start++; + } + } + return err; +} + +static int osd_ldiskfs_map_nblocks(struct inode *inode, unsigned long block, + unsigned long num, unsigned long *blocks, + int create) +{ + struct bpointers bp; + int err; + + CDEBUG(D_OTHER, "blocks %lu-%lu requested for inode %u\n", + block, block + num - 1, (unsigned) inode->i_ino); + + bp.blocks = blocks; + bp.start = block; + bp.init_num = bp.num = num; + bp.create = create; + + err = ldiskfs_ext_walk_space(inode, block, num, + ldiskfs_ext_new_extent_cb, &bp); + ldiskfs_ext_invalidate_cache(inode); + + return err; +} + +static int osd_ldiskfs_map_bm_inode_pages(struct inode *inode, + struct page **page, int pages, + unsigned long *blocks, int create) +{ + int blocks_per_page = PAGE_CACHE_SIZE >> inode->i_blkbits; + pgoff_t bitmap_max_page_index; + unsigned long *b; + int rc = 0, i; + + bitmap_max_page_index = LDISKFS_SB(inode->i_sb)->s_bitmap_maxbytes >> + PAGE_SHIFT; + for (i = 0, b = blocks; i < pages; i++, page++) { + if ((*page)->index + 1 >= bitmap_max_page_index) { + rc = -EFBIG; + break; + } + rc = ldiskfs_map_inode_page(inode, *page, b, create); + if (rc) { + CERROR("ino %lu, blk %lu create %d: rc %d\n", + inode->i_ino, *b, create, rc); + break; + } + b += blocks_per_page; + } + return rc; +} + +static int osd_ldiskfs_map_ext_inode_pages(struct inode *inode, + struct page **page, + int pages, unsigned long *blocks, + int create) +{ + int blocks_per_page = PAGE_CACHE_SIZE >> inode->i_blkbits; + int rc = 0, i = 0; + struct page *fp = NULL; + int clen = 0; + pgoff_t extent_max_page_index; + + extent_max_page_index = inode->i_sb->s_maxbytes >> PAGE_SHIFT; + + CDEBUG(D_OTHER, "inode %lu: map %d pages from %lu\n", + inode->i_ino, pages, (*page)->index); + + /* pages are sorted already. so, we just have to find + * contig. space and process them properly */ + while (i < pages) { + if (fp == NULL) { + /* start new extent */ + fp = *page++; + clen = 1; + i++; + continue; + } else if (fp->index + clen == (*page)->index) { + /* continue the extent */ + page++; + clen++; + i++; + continue; + } + + if (fp->index + i >= extent_max_page_index) + GOTO(cleanup, rc = -EFBIG); + + /* process found extent */ + rc = osd_ldiskfs_map_nblocks(inode, fp->index * blocks_per_page, + clen * blocks_per_page, blocks, + create); + if (rc) + GOTO(cleanup, rc); + + /* look for next extent */ + fp = NULL; + blocks += blocks_per_page * clen; + } + + if (fp) + rc = osd_ldiskfs_map_nblocks(inode, fp->index * blocks_per_page, + clen * blocks_per_page, blocks, + create); cleanup: - RETURN(rc); + return rc; } -static int osd_bufs_put(const struct lu_env *env, struct dt_object *dt, - struct niobuf_local *lnb, int npages) +static int osd_ldiskfs_map_inode_pages(struct inode *inode, struct page **page, + int pages, unsigned long *blocks, + int create) { - struct osd_thread_info *oti = osd_oti_get(env); - struct osd_iobuf *iobuf = &oti->oti_iobuf; - struct osd_device *d = osd_obj2dev(osd_dt_obj(dt)); - int i; + int rc; - /* to do IO stats, notice we do this here because - * osd_do_bio() doesn't wait for write to complete */ - osd_fini_iobuf(d, iobuf); + if (LDISKFS_I(inode)->i_flags & LDISKFS_EXTENTS_FL) { + rc = osd_ldiskfs_map_ext_inode_pages(inode, page, pages, + blocks, create); + return rc; + } + rc = osd_ldiskfs_map_bm_inode_pages(inode, page, pages, blocks, create); - for (i = 0; i < npages; i++) { - if (lnb[i].page == NULL) - continue; - LASSERT(PageLocked(lnb[i].page)); - unlock_page(lnb[i].page); - page_cache_release(lnb[i].page); - lu_object_put(env, &dt->do_lu); - lnb[i].page = NULL; - } - RETURN(0); + return rc; +} +#else +static int osd_ldiskfs_map_inode_pages(struct inode *inode, struct page **page, + int pages, unsigned long *blocks, + int create) +{ + int blocks_per_page = PAGE_CACHE_SIZE >> inode->i_blkbits; + int rc = 0, i = 0; + struct page *fp = NULL; + int clen = 0; + pgoff_t max_page_index; + + max_page_index = inode->i_sb->s_maxbytes >> PAGE_SHIFT; + + CDEBUG(D_OTHER, "inode %lu: map %d pages from %lu\n", + inode->i_ino, pages, (*page)->index); + + /* pages are sorted already. so, we just have to find + * contig. space and process them properly */ + while (i < pages) { + long blen, total = 0; + handle_t *handle = NULL; + struct ldiskfs_map_blocks map = { 0 }; + + if (fp == NULL) { /* start new extent */ + fp = *page++; + clen = 1; + if (++i != pages) + continue; + } else if (fp->index + clen == (*page)->index) { + /* continue the extent */ + page++; + clen++; + if (++i != pages) + continue; + } + if (fp->index + i >= max_page_index) + GOTO(cleanup, rc = -EFBIG); + /* process found extent */ + map.m_lblk = fp->index * blocks_per_page; + map.m_len = blen = clen * blocks_per_page; + if (create) { + create = LDISKFS_GET_BLOCKS_CREATE; + handle = ldiskfs_journal_current_handle(); + LASSERT(handle != NULL); + } +cont_map: + rc = ldiskfs_map_blocks(handle, inode, &map, create); + if (rc >= 0) { + int c = 0; + for (; total < blen && c < map.m_len; c++, total++) { + if (rc == 0) { + *(blocks + total) = 0; + total++; + break; + } else { + *(blocks + total) = map.m_pblk + c; + /* unmap any possible underlying + * metadata from the block device + * mapping. bug 6998. */ + if ((map.m_flags & LDISKFS_MAP_NEW) && + create) + unmap_underlying_metadata( + inode->i_sb->s_bdev, + map.m_pblk + c); + } + } + rc = 0; + } + if (rc == 0 && total < blen) { + map.m_lblk = fp->index * blocks_per_page + total; + map.m_len = blen - total; + goto cont_map; + } + if (rc != 0) + GOTO(cleanup, rc); + + /* look for next extent */ + fp = NULL; + blocks += blocks_per_page * clen; + } +cleanup: + return rc; } +#endif /* HAVE_LDISKFS_MAP_BLOCKS */ static int osd_write_prep(const struct lu_env *env, struct dt_object *dt, struct niobuf_local *lnb, int npages) @@ -590,55 +991,54 @@ static int osd_write_prep(const struct lu_env *env, struct dt_object *dt, RETURN(rc); isize = i_size_read(inode); - maxidx = ((isize + CFS_PAGE_SIZE - 1) >> CFS_PAGE_SHIFT) - 1; + maxidx = ((isize + PAGE_CACHE_SIZE - 1) >> PAGE_CACHE_SHIFT) - 1; if (osd->od_writethrough_cache) cache = 1; if (isize > osd->od_readcache_max_filesize) cache = 0; - cfs_gettimeofday(&start); - for (i = 0; i < npages; i++) { + do_gettimeofday(&start); + for (i = 0; i < npages; i++) { - if (cache == 0) - generic_error_remove_page(inode->i_mapping, - lnb[i].page); + if (cache == 0) + generic_error_remove_page(inode->i_mapping, + lnb[i].lnb_page); - /* - * till commit the content of the page is undefined - * we'll set it uptodate once bulk is done. otherwise - * subsequent reads can access non-stable data - */ - ClearPageUptodate(lnb[i].page); + /* + * till commit the content of the page is undefined + * we'll set it uptodate once bulk is done. otherwise + * subsequent reads can access non-stable data + */ + ClearPageUptodate(lnb[i].lnb_page); - if (lnb[i].len == CFS_PAGE_SIZE) - continue; + if (lnb[i].lnb_len == PAGE_CACHE_SIZE) + continue; - if (maxidx >= lnb[i].page->index) { - osd_iobuf_add_page(iobuf, lnb[i].page); - } else { - long off; - char *p = kmap(lnb[i].page); + if (maxidx >= lnb[i].lnb_page->index) { + osd_iobuf_add_page(iobuf, lnb[i].lnb_page); + } else { + long off; + char *p = kmap(lnb[i].lnb_page); off = lnb[i].lnb_page_offset; if (off) memset(p, 0, off); - off = (lnb[i].lnb_page_offset + lnb[i].len) & - ~CFS_PAGE_MASK; - if (off) - memset(p + off, 0, CFS_PAGE_SIZE - off); - kunmap(lnb[i].page); - } - } - cfs_gettimeofday(&end); - timediff = cfs_timeval_sub(&end, &start, NULL); - lprocfs_counter_add(osd->od_stats, LPROC_OSD_GET_PAGE, timediff); + off = (lnb[i].lnb_page_offset + lnb[i].lnb_len) & + ~PAGE_MASK; + if (off) + memset(p + off, 0, PAGE_CACHE_SIZE - off); + kunmap(lnb[i].lnb_page); + } + } + do_gettimeofday(&end); + timediff = cfs_timeval_sub(&end, &start, NULL); + lprocfs_counter_add(osd->od_stats, LPROC_OSD_GET_PAGE, timediff); if (iobuf->dr_npages) { - rc = osd->od_fsops->fs_map_inode_pages(inode, iobuf->dr_pages, - iobuf->dr_npages, - iobuf->dr_blocks, - 0, NULL); + rc = osd_ldiskfs_map_inode_pages(inode, iobuf->dr_pages, + iobuf->dr_npages, + iobuf->dr_blocks, 0); if (likely(rc == 0)) { rc = osd_do_bio(osd, inode, iobuf); /* do IO stats for preparation reads */ @@ -649,7 +1049,7 @@ static int osd_write_prep(const struct lu_env *env, struct dt_object *dt, } /* Check if a block is allocated or not */ -static int osd_is_mapped(struct inode *inode, obd_size offset) +static int osd_is_mapped(struct inode *inode, u64 offset) { sector_t (*fs_bmap)(struct address_space *, sector_t); @@ -700,11 +1100,11 @@ static int osd_declare_write_commit(const struct lu_env *env, /* calculate number of extents (probably better to pass nb) */ for (i = 0; i < npages; i++) { if (i && lnb[i].lnb_file_offset != - lnb[i - 1].lnb_file_offset + lnb[i - 1].len) + lnb[i - 1].lnb_file_offset + lnb[i - 1].lnb_len) extents++; if (!osd_is_mapped(inode, lnb[i].lnb_file_offset)) - quota_space += CFS_PAGE_SIZE; + quota_space += PAGE_CACHE_SIZE; /* ignore quota for the whole request if any page is from * client cache or written by root. @@ -715,8 +1115,8 @@ static int osd_declare_write_commit(const struct lu_env *env, * * XXX we could handle this on per-lnb basis as done by * grant. */ - if ((lnb[i].flags & OBD_BRW_NOQUOTA) || - (lnb[i].flags & (OBD_BRW_FROM_GRANT | OBD_BRW_SYNC)) == + if ((lnb[i].lnb_flags & OBD_BRW_NOQUOTA) || + (lnb[i].lnb_flags & (OBD_BRW_FROM_GRANT | OBD_BRW_SYNC)) == OBD_BRW_FROM_GRANT) ignore_quota = true; } @@ -764,19 +1164,19 @@ static int osd_declare_write_commit(const struct lu_env *env, oh->ot_credits += newblocks; /* make sure the over quota flags were not set */ - lnb[0].flags &= ~(OBD_BRW_OVER_USRQUOTA | OBD_BRW_OVER_GRPQUOTA); + lnb[0].lnb_flags &= ~(OBD_BRW_OVER_USRQUOTA | OBD_BRW_OVER_GRPQUOTA); - rc = osd_declare_inode_qid(env, inode->i_uid, inode->i_gid, - quota_space, oh, true, true, &flags, - ignore_quota); + rc = osd_declare_inode_qid(env, i_uid_read(inode), i_gid_read(inode), + quota_space, oh, osd_dt_obj(dt), true, + &flags, ignore_quota); /* we need only to store the overquota flags in the first lnb for * now, once we support multiple objects BRW, this code needs be * revised. */ if (flags & QUOTA_FL_OVER_USRQUOTA) - lnb[0].flags |= OBD_BRW_OVER_USRQUOTA; + lnb[0].lnb_flags |= OBD_BRW_OVER_USRQUOTA; if (flags & QUOTA_FL_OVER_GRPQUOTA) - lnb[0].flags |= OBD_BRW_OVER_GRPQUOTA; + lnb[0].lnb_flags |= OBD_BRW_OVER_GRPQUOTA; RETURN(rc); } @@ -803,46 +1203,46 @@ static int osd_write_commit(const struct lu_env *env, struct dt_object *dt, ll_vfs_dq_init(inode); for (i = 0; i < npages; i++) { - if (lnb[i].rc == -ENOSPC && + if (lnb[i].lnb_rc == -ENOSPC && osd_is_mapped(inode, lnb[i].lnb_file_offset)) { - /* Allow the write to proceed if overwriting an - * existing block */ - lnb[i].rc = 0; - } + /* Allow the write to proceed if overwriting an + * existing block */ + lnb[i].lnb_rc = 0; + } - if (lnb[i].rc) { /* ENOSPC, network RPC error, etc. */ - CDEBUG(D_INODE, "Skipping [%d] == %d\n", i, - lnb[i].rc); - LASSERT(lnb[i].page); - generic_error_remove_page(inode->i_mapping,lnb[i].page); - continue; - } + if (lnb[i].lnb_rc) { /* ENOSPC, network RPC error, etc. */ + CDEBUG(D_INODE, "Skipping [%d] == %d\n", i, + lnb[i].lnb_rc); + LASSERT(lnb[i].lnb_page); + generic_error_remove_page(inode->i_mapping, + lnb[i].lnb_page); + continue; + } - LASSERT(PageLocked(lnb[i].page)); - LASSERT(!PageWriteback(lnb[i].page)); + LASSERT(PageLocked(lnb[i].lnb_page)); + LASSERT(!PageWriteback(lnb[i].lnb_page)); - if (lnb[i].lnb_file_offset + lnb[i].len > isize) - isize = lnb[i].lnb_file_offset + lnb[i].len; + if (lnb[i].lnb_file_offset + lnb[i].lnb_len > isize) + isize = lnb[i].lnb_file_offset + lnb[i].lnb_len; - /* - * Since write and truncate are serialized by oo_sem, even - * partial-page truncate should not leave dirty pages in the - * page cache. - */ - LASSERT(!PageDirty(lnb[i].page)); + /* + * Since write and truncate are serialized by oo_sem, even + * partial-page truncate should not leave dirty pages in the + * page cache. + */ + LASSERT(!PageDirty(lnb[i].lnb_page)); - SetPageUptodate(lnb[i].page); + SetPageUptodate(lnb[i].lnb_page); - osd_iobuf_add_page(iobuf, lnb[i].page); + osd_iobuf_add_page(iobuf, lnb[i].lnb_page); } if (OBD_FAIL_CHECK(OBD_FAIL_OST_MAPBLK_ENOSPC)) { rc = -ENOSPC; } else if (iobuf->dr_npages > 0) { - rc = osd->od_fsops->fs_map_inode_pages(inode, iobuf->dr_pages, - iobuf->dr_npages, - iobuf->dr_blocks, - 1, NULL); + rc = osd_ldiskfs_map_inode_pages(inode, iobuf->dr_pages, + iobuf->dr_npages, + iobuf->dr_blocks, 1); } else { /* no pages to write, no transno is needed */ thandle->th_local = 1; @@ -858,19 +1258,22 @@ static int osd_write_commit(const struct lu_env *env, struct dt_object *dt, rc = osd_do_bio(osd, inode, iobuf); /* we don't do stats here as in read path because * write is async: we'll do this in osd_put_bufs() */ - } + } else { + osd_fini_iobuf(osd, iobuf); + } if (unlikely(rc != 0)) { /* if write fails, we should drop pages from the cache */ for (i = 0; i < npages; i++) { - if (lnb[i].page == NULL) - continue; - LASSERT(PageLocked(lnb[i].page)); - generic_error_remove_page(inode->i_mapping,lnb[i].page); - } - } + if (lnb[i].lnb_page == NULL) + continue; + LASSERT(PageLocked(lnb[i].lnb_page)); + generic_error_remove_page(inode->i_mapping, + lnb[i].lnb_page); + } + } - RETURN(rc); + RETURN(rc); } static int osd_read_prep(const struct lu_env *env, struct dt_object *dt, @@ -882,7 +1285,8 @@ static int osd_read_prep(const struct lu_env *env, struct dt_object *dt, struct osd_device *osd = osd_obj2dev(osd_dt_obj(dt)); struct timeval start, end; unsigned long timediff; - int rc = 0, i, m = 0, cache = 0; + int rc = 0, i, cache = 0, cache_hits = 0, cache_misses = 0; + loff_t isize; LASSERT(inode); @@ -890,47 +1294,55 @@ static int osd_read_prep(const struct lu_env *env, struct dt_object *dt, if (unlikely(rc != 0)) RETURN(rc); + isize = i_size_read(inode); + if (osd->od_read_cache) cache = 1; - if (i_size_read(inode) > osd->od_readcache_max_filesize) + if (isize > osd->od_readcache_max_filesize) cache = 0; - cfs_gettimeofday(&start); - for (i = 0; i < npages; i++) { + do_gettimeofday(&start); + for (i = 0; i < npages; i++) { - if (i_size_read(inode) <= lnb[i].lnb_file_offset) - /* If there's no more data, abort early. - * lnb->rc == 0, so it's easy to detect later. */ - break; + if (isize <= lnb[i].lnb_file_offset) + /* If there's no more data, abort early. + * lnb->lnb_rc == 0, so it's easy to detect later. */ + break; - if (i_size_read(inode) < - lnb[i].lnb_file_offset + lnb[i].len - 1) - lnb[i].rc = i_size_read(inode) - lnb[i].lnb_file_offset; - else - lnb[i].rc = lnb[i].len; - m += lnb[i].len; - - lprocfs_counter_add(osd->od_stats, LPROC_OSD_CACHE_ACCESS, 1); - if (PageUptodate(lnb[i].page)) { - lprocfs_counter_add(osd->od_stats, - LPROC_OSD_CACHE_HIT, 1); - } else { - lprocfs_counter_add(osd->od_stats, - LPROC_OSD_CACHE_MISS, 1); - osd_iobuf_add_page(iobuf, lnb[i].page); - } - if (cache == 0) - generic_error_remove_page(inode->i_mapping,lnb[i].page); - } - cfs_gettimeofday(&end); - timediff = cfs_timeval_sub(&end, &start, NULL); - lprocfs_counter_add(osd->od_stats, LPROC_OSD_GET_PAGE, timediff); + if (isize < lnb[i].lnb_file_offset + lnb[i].lnb_len - 1) + lnb[i].lnb_rc = isize - lnb[i].lnb_file_offset; + else + lnb[i].lnb_rc = lnb[i].lnb_len; + + if (PageUptodate(lnb[i].lnb_page)) { + cache_hits++; + } else { + cache_misses++; + osd_iobuf_add_page(iobuf, lnb[i].lnb_page); + } + + if (cache == 0) + generic_error_remove_page(inode->i_mapping, + lnb[i].lnb_page); + } + do_gettimeofday(&end); + timediff = cfs_timeval_sub(&end, &start, NULL); + lprocfs_counter_add(osd->od_stats, LPROC_OSD_GET_PAGE, timediff); + + if (cache_hits != 0) + lprocfs_counter_add(osd->od_stats, LPROC_OSD_CACHE_HIT, + cache_hits); + if (cache_misses != 0) + lprocfs_counter_add(osd->od_stats, LPROC_OSD_CACHE_MISS, + cache_misses); + if (cache_hits + cache_misses != 0) + lprocfs_counter_add(osd->od_stats, LPROC_OSD_CACHE_ACCESS, + cache_hits + cache_misses); if (iobuf->dr_npages) { - rc = osd->od_fsops->fs_map_inode_pages(inode, iobuf->dr_pages, - iobuf->dr_npages, - iobuf->dr_blocks, - 0, NULL); + rc = osd_ldiskfs_map_inode_pages(inode, iobuf->dr_pages, + iobuf->dr_npages, + iobuf->dr_blocks, 0); rc = osd_do_bio(osd, inode, iobuf); /* IO stats will be done in osd_bufs_put() */ @@ -994,15 +1406,21 @@ int osd_ldiskfs_read(struct inode *inode, void *buf, int size, loff_t *offs) boffs = *offs & (blocksize - 1); csize = min(blocksize - boffs, size); bh = ldiskfs_bread(NULL, inode, block, 0, &err); - if (!bh) { - CERROR("%s: can't read %u@%llu on ino %lu: rc = %d\n", - LDISKFS_SB(inode->i_sb)->s_es->s_volume_name, - csize, *offs, inode->i_ino, err); - return err; - } + if (err != 0) { + CERROR("%s: can't read %u@%llu on ino %lu: rc = %d\n", + LDISKFS_SB(inode->i_sb)->s_es->s_volume_name, + csize, *offs, inode->i_ino, err); + if (bh != NULL) + brelse(bh); + return err; + } - memcpy(buf, bh->b_data + boffs, csize); - brelse(bh); + if (bh != NULL) { + memcpy(buf, bh->b_data + boffs, csize); + brelse(bh); + } else { + memset(buf, 0, csize); + } *offs += csize; buf += csize; @@ -1012,15 +1430,11 @@ int osd_ldiskfs_read(struct inode *inode, void *buf, int size, loff_t *offs) } static ssize_t osd_read(const struct lu_env *env, struct dt_object *dt, - struct lu_buf *buf, loff_t *pos, - struct lustre_capa *capa) + struct lu_buf *buf, loff_t *pos) { struct inode *inode = osd_dt_obj(dt)->oo_inode; int rc; - if (osd_object_auth(env, dt, capa, CAPA_OPC_BODY_READ)) - RETURN(-EACCES); - /* Read small symlink from inode body as we need to maintain correct * on-disk symlinks for ldiskfs. */ @@ -1033,36 +1447,151 @@ static ssize_t osd_read(const struct lu_env *env, struct dt_object *dt, return rc; } +static inline int osd_extents_enabled(struct super_block *sb, + struct inode *inode) +{ + if (inode != NULL) { + if (LDISKFS_I(inode)->i_flags & LDISKFS_EXTENTS_FL) + return 1; + } else if (LDISKFS_HAS_INCOMPAT_FEATURE(sb, + LDISKFS_FEATURE_INCOMPAT_EXTENTS)) { + return 1; + } + return 0; +} + +static inline int osd_calc_bkmap_credits(struct super_block *sb, + struct inode *inode, + const loff_t size, + const loff_t pos, + const int blocks) +{ + int credits, bits, bs, i; + + bits = sb->s_blocksize_bits; + bs = 1 << bits; + + /* legacy blockmap: 3 levels * 3 (bitmap,gd,itself) + * we do not expect blockmaps on the large files, + * so let's shrink it to 2 levels (4GB files) */ + + /* this is default reservation: 2 levels */ + credits = (blocks + 2) * 3; + + /* actual offset is unknown, hard to optimize */ + if (pos == -1) + return credits; + + /* now check for few specific cases to optimize */ + if (pos + size <= LDISKFS_NDIR_BLOCKS * bs) { + /* no indirects */ + credits = blocks; + /* allocate if not allocated */ + if (inode == NULL) { + credits += blocks * 2; + return credits; + } + for (i = (pos >> bits); i < (pos >> bits) + blocks; i++) { + LASSERT(i < LDISKFS_NDIR_BLOCKS); + if (LDISKFS_I(inode)->i_data[i] == 0) + credits += 2; + } + } else if (pos + size <= (LDISKFS_NDIR_BLOCKS + 1024) * bs) { + /* single indirect */ + credits = blocks * 3; + /* probably indirect block has been allocated already */ + if (!inode || LDISKFS_I(inode)->i_data[LDISKFS_IND_BLOCK]) + credits += 3; + } + + return credits; +} + static ssize_t osd_declare_write(const struct lu_env *env, struct dt_object *dt, - const loff_t size, loff_t pos, - struct thandle *handle) + const struct lu_buf *buf, loff_t _pos, + struct thandle *handle) { - struct osd_thandle *oh; - int credits; - struct inode *inode; - int rc; + struct osd_object *obj = osd_dt_obj(dt); + struct inode *inode = obj->oo_inode; + struct super_block *sb = osd_sb(osd_obj2dev(obj)); + struct osd_thandle *oh; + int rc = 0, est = 0, credits, blocks, allocated = 0; + int bits, bs; + int depth, size; + loff_t pos; ENTRY; + LASSERT(buf != NULL); LASSERT(handle != NULL); oh = container_of0(handle, struct osd_thandle, ot_super); LASSERT(oh->ot_handle == NULL); - credits = osd_dto_credits_noquota[DTO_WRITE_BLOCK]; + size = buf->lb_len; + bits = sb->s_blocksize_bits; + bs = 1 << bits; - osd_trans_declare_op(env, oh, OSD_OT_WRITE, credits); + if (_pos == -1) { + /* if this is an append, then we + * should expect cross-block record */ + pos = 0; + } else { + pos = _pos; + } - inode = osd_dt_obj(dt)->oo_inode; + /* blocks to modify */ + blocks = ((pos + size + bs - 1) >> bits) - (pos >> bits); + LASSERT(blocks > 0); + + if (inode != NULL && _pos != -1) { + /* object size in blocks */ + est = (i_size_read(inode) + bs - 1) >> bits; + allocated = inode->i_blocks >> (bits - 9); + if (pos + size <= i_size_read(inode) && est <= allocated) { + /* looks like an overwrite, no need to modify tree */ + credits = blocks; + /* no need to modify i_size */ + goto out; + } + } - /* we may declare write to non-exist llog */ - if (inode == NULL) - RETURN(0); + if (osd_extents_enabled(sb, inode)) { + /* + * many concurrent threads may grow tree by the time + * our transaction starts. so, consider 2 is a min depth + * for every level we may need to allocate a new block + * and take some entries from the old one. so, 3 blocks + * to allocate (bitmap, gd, itself) + old block - 4 per + * level. + */ + depth = inode != NULL ? ext_depth(inode) : 0; + depth = max(depth, 1) + 1; + credits = depth; + /* if not append, then split may need to modify + * existing blocks moving entries into the new ones */ + if (_pos == -1) + credits += depth; + /* blocks to store data: bitmap,gd,itself */ + credits += blocks * 3; + } else { + credits = osd_calc_bkmap_credits(sb, inode, size, _pos, blocks); + } + /* if inode is created as part of the transaction, + * then it's counted already by the creation method */ + if (inode != NULL) + credits++; + +out: + + osd_trans_declare_op(env, oh, OSD_OT_WRITE, credits); /* dt_declare_write() is usually called for system objects, such * as llog or last_rcvd files. We needn't enforce quota on those * objects, so always set the lqi_space as 0. */ - rc = osd_declare_inode_qid(env, inode->i_uid, inode->i_gid, 0, oh, - true, true, NULL, false); + if (inode != NULL) + rc = osd_declare_inode_qid(env, i_uid_read(inode), + i_gid_read(inode), 0, oh, obj, true, + NULL, false); RETURN(rc); } @@ -1122,11 +1651,11 @@ int osd_ldiskfs_write_record(struct inode *inode, void *buf, int bufsize, err); break; } - LASSERTF(boffs + size <= bh->b_size, - "boffs %d size %d bh->b_size %lu", - boffs, size, (unsigned long)bh->b_size); + LASSERTF(boffs + size <= bh->b_size, + "boffs %d size %d bh->b_size %lu\n", + boffs, size, (unsigned long)bh->b_size); memcpy(bh->b_data + boffs, buf, size); - err = ldiskfs_journal_dirty_metadata(handle, bh); + err = ldiskfs_handle_dirty_metadata(handle, NULL, bh); if (err) break; @@ -1161,9 +1690,8 @@ int osd_ldiskfs_write_record(struct inode *inode, void *buf, int bufsize, } static ssize_t osd_write(const struct lu_env *env, struct dt_object *dt, - const struct lu_buf *buf, loff_t *pos, - struct thandle *handle, struct lustre_capa *capa, - int ignore_quota) + const struct lu_buf *buf, loff_t *pos, + struct thandle *handle, int ignore_quota) { struct inode *inode = osd_dt_obj(dt)->oo_inode; struct osd_thandle *oh; @@ -1172,9 +1700,6 @@ static ssize_t osd_write(const struct lu_env *env, struct dt_object *dt, LASSERT(dt_object_exists(dt)); - if (osd_object_auth(env, dt, capa, CAPA_OPC_BODY_WRITE)) - return -EACCES; - LASSERT(handle != NULL); LASSERT(inode != NULL); ll_vfs_dq_init(inode); @@ -1226,44 +1751,50 @@ static int osd_declare_punch(const struct lu_env *env, struct dt_object *dt, inode = osd_dt_obj(dt)->oo_inode; LASSERT(inode); - rc = osd_declare_inode_qid(env, inode->i_uid, inode->i_gid, 0, oh, - true, true, NULL, false); + rc = osd_declare_inode_qid(env, i_uid_read(inode), i_gid_read(inode), + 0, oh, osd_dt_obj(dt), true, NULL, false); RETURN(rc); } static int osd_punch(const struct lu_env *env, struct dt_object *dt, - __u64 start, __u64 end, struct thandle *th, - struct lustre_capa *capa) + __u64 start, __u64 end, struct thandle *th) { - struct osd_thandle *oh; - struct osd_object *obj = osd_dt_obj(dt); - struct inode *inode = obj->oo_inode; - handle_t *h; - tid_t tid; - int rc, rc2 = 0; - ENTRY; + struct osd_thandle *oh; + struct osd_object *obj = osd_dt_obj(dt); + struct inode *inode = obj->oo_inode; + handle_t *h; + tid_t tid; + int rc = 0, rc2 = 0; + ENTRY; - LASSERT(end == OBD_OBJECT_EOF); - LASSERT(dt_object_exists(dt)); - LASSERT(osd_invariant(obj)); + LASSERT(end == OBD_OBJECT_EOF); + LASSERT(dt_object_exists(dt)); + LASSERT(osd_invariant(obj)); LASSERT(inode != NULL); ll_vfs_dq_init(inode); - LASSERT(th); - oh = container_of(th, struct osd_thandle, ot_super); - LASSERT(oh->ot_handle->h_transaction != NULL); + LASSERT(th); + oh = container_of(th, struct osd_thandle, ot_super); + LASSERT(oh->ot_handle->h_transaction != NULL); osd_trans_exec_op(env, th, OSD_OT_PUNCH); - tid = oh->ot_handle->h_transaction->t_tid; + tid = oh->ot_handle->h_transaction->t_tid; - rc = vmtruncate(inode, start); + i_size_write(inode, start); + ll_truncate_pagecache(inode, start); +#ifdef HAVE_INODEOPS_TRUNCATE + if (inode->i_op->truncate) { + inode->i_op->truncate(inode); + } else +#endif + ldiskfs_truncate(inode); - /* - * For a partial-page truncate, flush the page to disk immediately to - * avoid data corruption during direct disk write. b=17397 - */ - if (rc == 0 && (start & ~CFS_PAGE_MASK) != 0) + /* + * For a partial-page truncate, flush the page to disk immediately to + * avoid data corruption during direct disk write. b=17397 + */ + if ((start & ~PAGE_MASK) != 0) rc = filemap_fdatawrite_range(inode->i_mapping, start, start+1); h = journal_current_handle(); @@ -1285,33 +1816,68 @@ static int osd_punch(const struct lu_env *env, struct dt_object *dt, RETURN(rc == 0 ? rc2 : rc); } +static int fiemap_check_ranges(struct inode *inode, + u64 start, u64 len, u64 *new_len) +{ + loff_t maxbytes; + + *new_len = len; + + if (len == 0) + return -EINVAL; + + if (ldiskfs_test_inode_flag(inode, LDISKFS_INODE_EXTENTS)) + maxbytes = inode->i_sb->s_maxbytes; + else + maxbytes = LDISKFS_SB(inode->i_sb)->s_bitmap_maxbytes; + + if (start > maxbytes) + return -EFBIG; + + /* + * Shrink request scope to what the fs can actually handle. + */ + if (len > maxbytes || (maxbytes - len) < start) + *new_len = maxbytes - start; + + return 0; +} + +/* So that the fiemap access checks can't overflow on 32 bit machines. */ +#define FIEMAP_MAX_EXTENTS (UINT_MAX / sizeof(struct fiemap_extent)) + static int osd_fiemap_get(const struct lu_env *env, struct dt_object *dt, - struct ll_user_fiemap *fm) + struct fiemap *fm) { - struct inode *inode = osd_dt_obj(dt)->oo_inode; - struct osd_thread_info *info = osd_oti_get(env); - struct dentry *dentry = &info->oti_obj_dentry; - struct file *file = &info->oti_file; - mm_segment_t saved_fs; - int rc; + struct fiemap_extent_info fieinfo = {0, }; + struct inode *inode = osd_dt_obj(dt)->oo_inode; + u64 len; + int rc; - LASSERT(inode); - dentry->d_inode = inode; - dentry->d_sb = inode->i_sb; - file->f_dentry = dentry; - file->f_mapping = inode->i_mapping; - file->f_op = inode->i_fop; - - saved_fs = get_fs(); - set_fs(get_ds()); - /* ldiskfs_ioctl does not have a inode argument */ - if (inode->i_fop->unlocked_ioctl) - rc = inode->i_fop->unlocked_ioctl(file, FSFILT_IOC_FIEMAP, - (long)fm); - else - rc = -ENOTTY; - set_fs(saved_fs); - return rc; + + LASSERT(inode); + if (inode->i_op->fiemap == NULL) + return -EOPNOTSUPP; + + if (fm->fm_extent_count > FIEMAP_MAX_EXTENTS) + return -EINVAL; + + rc = fiemap_check_ranges(inode, fm->fm_start, fm->fm_length, &len); + if (rc) + return rc; + + fieinfo.fi_flags = fm->fm_flags; + fieinfo.fi_extents_max = fm->fm_extent_count; + fieinfo.fi_extents_start = fm->fm_extents; + + if (fieinfo.fi_flags & FIEMAP_FLAG_SYNC) + filemap_write_and_wait(inode->i_mapping); + + rc = inode->i_op->fiemap(inode, &fieinfo, fm->fm_start, len); + fm->fm_flags = fieinfo.fi_flags; + fm->fm_mapped_extents = fieinfo.fi_extents_mapped; + + return rc; } /*