X-Git-Url: https://git.whamcloud.com/?p=fs%2Flustre-release.git;a=blobdiff_plain;f=lustre%2Fosd-ldiskfs%2Fosd_io.c;h=34346348ee4ed095020795d4a704a5525dc8e957;hp=ef045a5fe0f6b5de59c99a97a27f01261f5c1c85;hb=5a6aa0e6d1583cc0d4c82ae8c95fb7b9856d6284;hpb=9ddf386035767a96b54e21559f3ea0be126dc8cd diff --git a/lustre/osd-ldiskfs/osd_io.c b/lustre/osd-ldiskfs/osd_io.c index ef045a5..3434634 100644 --- a/lustre/osd-ldiskfs/osd_io.c +++ b/lustre/osd-ldiskfs/osd_io.c @@ -15,11 +15,7 @@ * * You should have received a copy of the GNU General Public License * version 2 along with this program; If not, see - * http://www.sun.com/software/products/lustre/docs/GPLv2.pdf - * - * Please contact Sun Microsystems, Inc., 4150 Network Circle, Santa Clara, - * CA 95054 USA or visit www.sun.com if you need additional information or - * have any questions. + * http://www.gnu.org/licenses/gpl-2.0.html * * GPL HEADER END */ @@ -27,7 +23,7 @@ * Copyright (c) 2010, Oracle and/or its affiliates. All rights reserved. * Use is subject to license terms. * - * Copyright (c) 2011, 2012, Whamcloud, Inc. + * Copyright (c) 2012, 2016, Intel Corporation. */ /* * This file is part of Lustre, http://www.lustre.org/ @@ -49,11 +45,6 @@ /* prerequisite for linux/xattr.h */ #include -/* ext_depth() */ -#include -#include -#include - /* * struct OBD_{ALLOC,FREE}*() * OBD_FAIL_CHECK @@ -62,51 +53,67 @@ #include "osd_internal.h" -#ifndef HAVE_PAGE_CONSTANT -#define mapping_cap_page_constant_write(mapping) 0 -#define SetPageConstant(page) do {} while (0) -#define ClearPageConstant(page) do {} while (0) -#endif +/* ext_depth() */ +#include -#ifndef HAS_GENERIC_ERROR_REMOVE_PAGE -int generic_error_remove_page(struct address_space *mapping, struct page *page) +static int __osd_init_iobuf(struct osd_device *d, struct osd_iobuf *iobuf, + int rw, int line, int pages) { - if (mapping == NULL) - return -EINVAL; + int blocks, i; + + LASSERTF(iobuf->dr_elapsed_valid == 0, + "iobuf %p, reqs %d, rw %d, line %d\n", iobuf, + atomic_read(&iobuf->dr_numreqs), iobuf->dr_rw, + iobuf->dr_init_at); + LASSERT(pages <= PTLRPC_MAX_BRW_PAGES); + + init_waitqueue_head(&iobuf->dr_wait); + atomic_set(&iobuf->dr_numreqs, 0); + iobuf->dr_npages = 0; + iobuf->dr_error = 0; + iobuf->dr_dev = d; + iobuf->dr_frags = 0; + iobuf->dr_elapsed = 0; + /* must be counted before, so assert */ + iobuf->dr_rw = rw; + iobuf->dr_init_at = line; + + blocks = pages * (PAGE_SIZE >> osd_sb(d)->s_blocksize_bits); + if (iobuf->dr_bl_buf.lb_len >= blocks * sizeof(iobuf->dr_blocks[0])) { + LASSERT(iobuf->dr_pg_buf.lb_len >= + pages * sizeof(iobuf->dr_pages[0])); + return 0; + } - if (mapping != page->mapping) - return -EIO; - /* - * Only punch for normal data pages for now. - * Handling other types like directories would need more auditing. - */ - if (!S_ISREG(mapping->host->i_mode)) - return -EIO; + /* start with 1MB for 4K blocks */ + i = 256; + while (i <= PTLRPC_MAX_BRW_PAGES && i < pages) + i <<= 1; - if (page_mapped(page)) { - unmap_mapping_range(mapping, - (loff_t)page->index << PAGE_CACHE_SHIFT, - PAGE_CACHE_SIZE, 0); - } - truncate_complete_page(mapping, page); - return 0; -} -#endif + CDEBUG(D_OTHER, "realloc %u for %u (%u) pages\n", + (unsigned)(pages * sizeof(iobuf->dr_pages[0])), i, pages); + pages = i; + blocks = pages * (PAGE_SIZE >> osd_sb(d)->s_blocksize_bits); + iobuf->dr_max_pages = 0; + CDEBUG(D_OTHER, "realloc %u for %u blocks\n", + (unsigned)(blocks * sizeof(iobuf->dr_blocks[0])), blocks); -static void osd_init_iobuf(struct osd_device *d, struct osd_iobuf *iobuf,int rw) -{ - cfs_waitq_init(&iobuf->dr_wait); - cfs_atomic_set(&iobuf->dr_numreqs, 0); - iobuf->dr_max_pages = PTLRPC_MAX_BRW_PAGES; - iobuf->dr_npages = 0; - iobuf->dr_error = 0; - iobuf->dr_dev = d; - iobuf->dr_frags = 0; - iobuf->dr_elapsed = 0; - /* must be counted before, so assert */ - LASSERT(iobuf->dr_elapsed_valid == 0); - iobuf->dr_rw = rw; + lu_buf_realloc(&iobuf->dr_bl_buf, blocks * sizeof(iobuf->dr_blocks[0])); + iobuf->dr_blocks = iobuf->dr_bl_buf.lb_buf; + if (unlikely(iobuf->dr_blocks == NULL)) + return -ENOMEM; + + lu_buf_realloc(&iobuf->dr_pg_buf, pages * sizeof(iobuf->dr_pages[0])); + iobuf->dr_pages = iobuf->dr_pg_buf.lb_buf; + if (unlikely(iobuf->dr_pages == NULL)) + return -ENOMEM; + + iobuf->dr_max_pages = pages; + + return 0; } +#define osd_init_iobuf(dev, iobuf, rw, pages) \ + __osd_init_iobuf(dev, iobuf, rw, __LINE__, pages) static void osd_iobuf_add_page(struct osd_iobuf *iobuf, struct page *page) { @@ -130,100 +137,108 @@ void osd_fini_iobuf(struct osd_device *d, struct osd_iobuf *iobuf) } } -#ifdef HAVE_BIO_ENDIO_2ARG -#define DIO_RETURN(a) -static void dio_complete_routine(struct bio *bio, int error) -#else -#define DIO_RETURN(a) return(a) -static int dio_complete_routine(struct bio *bio, unsigned int done, int error) +#ifndef REQ_WRITE /* pre-2.6.35 */ +#define __REQ_WRITE BIO_RW #endif + +#ifdef HAVE_BIO_ENDIO_USES_ONE_ARG +static void dio_complete_routine(struct bio *bio) { - struct osd_iobuf *iobuf = bio->bi_private; - struct bio_vec *bvl; - int i; + int error = bio->bi_error; +#else +static void dio_complete_routine(struct bio *bio, int error) +{ +#endif + struct osd_iobuf *iobuf = bio->bi_private; + int iter; + struct bio_vec *bvl; /* CAVEAT EMPTOR: possibly in IRQ context * DO NOT record procfs stats here!!! */ - if (unlikely(iobuf == NULL)) { - CERROR("***** bio->bi_private is NULL! This should never " - "happen. Normally, I would crash here, but instead I " - "will dump the bio contents to the console. Please " - "report this to , along " - "with any interesting messages leading up to this point " - "(like SCSI errors, perhaps). Because bi_private is " - "NULL, I can't wake up the thread that initiated this " - "IO - you will probably have to reboot this node.\n"); - CERROR("bi_next: %p, bi_flags: %lx, bi_rw: %lu, bi_vcnt: %d, " - "bi_idx: %d, bi->size: %d, bi_end_io: %p, bi_cnt: %d, " - "bi_private: %p\n", bio->bi_next, bio->bi_flags, - bio->bi_rw, bio->bi_vcnt, bio->bi_idx, bio->bi_size, - bio->bi_end_io, cfs_atomic_read(&bio->bi_cnt), - bio->bi_private); - DIO_RETURN(0); - } - - /* the check is outside of the cycle for performance reason -bzzz */ - if (!cfs_test_bit(BIO_RW, &bio->bi_rw)) { - bio_for_each_segment(bvl, bio, i) { - if (likely(error == 0)) - SetPageUptodate(bvl->bv_page); - LASSERT(PageLocked(bvl->bv_page)); - ClearPageConstant(bvl->bv_page); - } - cfs_atomic_dec(&iobuf->dr_dev->od_r_in_flight); - } else { - struct page *p = iobuf->dr_pages[0]; - if (p->mapping) { - if (mapping_cap_page_constant_write(p->mapping)) { - bio_for_each_segment(bvl, bio, i) { - ClearPageConstant(bvl->bv_page); - } - } - } - cfs_atomic_dec(&iobuf->dr_dev->od_w_in_flight); - } + if (unlikely(iobuf == NULL)) { + CERROR("***** bio->bi_private is NULL! This should never " + "happen. Normally, I would crash here, but instead I " + "will dump the bio contents to the console. Please " + "report this to , along " + "with any interesting messages leading up to this point " + "(like SCSI errors, perhaps). Because bi_private is " + "NULL, I can't wake up the thread that initiated this " + "IO - you will probably have to reboot this node.\n"); + CERROR("bi_next: %p, bi_flags: %lx, bi_rw: %lu, bi_vcnt: %d, " + "bi_idx: %d, bi->size: %d, bi_end_io: %p, bi_cnt: %d, " + "bi_private: %p\n", bio->bi_next, + (unsigned long)bio->bi_flags, + bio->bi_rw, bio->bi_vcnt, bio_idx(bio), + bio_sectors(bio) << 9, bio->bi_end_io, +#ifdef HAVE_BI_CNT + atomic_read(&bio->bi_cnt), +#else + atomic_read(&bio->__bi_cnt), +#endif + bio->bi_private); + return; + } - /* any real error is good enough -bzzz */ - if (error != 0 && iobuf->dr_error == 0) - iobuf->dr_error = error; + /* the check is outside of the cycle for performance reason -bzzz */ + if (!test_bit(__REQ_WRITE, &bio->bi_rw)) { + bio_for_each_segment_all(bvl, bio, iter) { + if (likely(error == 0)) + SetPageUptodate(bvl_to_page(bvl)); + LASSERT(PageLocked(bvl_to_page(bvl))); + } + atomic_dec(&iobuf->dr_dev->od_r_in_flight); + } else { + atomic_dec(&iobuf->dr_dev->od_w_in_flight); + } - if (cfs_atomic_dec_and_test(&iobuf->dr_numreqs)) { - iobuf->dr_elapsed = jiffies - iobuf->dr_start_time; - iobuf->dr_elapsed_valid = 1; - cfs_waitq_signal(&iobuf->dr_wait); - } + /* any real error is good enough -bzzz */ + if (error != 0 && iobuf->dr_error == 0) + iobuf->dr_error = error; - /* Completed bios used to be chained off iobuf->dr_bios and freed in - * filter_clear_dreq(). It was then possible to exhaust the biovec-256 - * mempool when serious on-disk fragmentation was encountered, - * deadlocking the OST. The bios are now released as soon as complete - * so the pool cannot be exhausted while IOs are competing. bug 10076 */ - bio_put(bio); - DIO_RETURN(0); + /* + * set dr_elapsed before dr_numreqs turns to 0, otherwise + * it's possible that service thread will see dr_numreqs + * is zero, but dr_elapsed is not set yet, leading to lost + * data in this processing and an assertion in a subsequent + * call to OSD. + */ + if (atomic_read(&iobuf->dr_numreqs) == 1) { + iobuf->dr_elapsed = jiffies - iobuf->dr_start_time; + iobuf->dr_elapsed_valid = 1; + } + if (atomic_dec_and_test(&iobuf->dr_numreqs)) + wake_up(&iobuf->dr_wait); + + /* Completed bios used to be chained off iobuf->dr_bios and freed in + * filter_clear_dreq(). It was then possible to exhaust the biovec-256 + * mempool when serious on-disk fragmentation was encountered, + * deadlocking the OST. The bios are now released as soon as complete + * so the pool cannot be exhausted while IOs are competing. bug 10076 */ + bio_put(bio); } static void record_start_io(struct osd_iobuf *iobuf, int size) { - struct osd_device *osd = iobuf->dr_dev; - struct obd_histogram *h = osd->od_brw_stats.hist; - - iobuf->dr_frags++; - cfs_atomic_inc(&iobuf->dr_numreqs); - - if (iobuf->dr_rw == 0) { - cfs_atomic_inc(&osd->od_r_in_flight); - lprocfs_oh_tally(&h[BRW_R_RPC_HIST], - cfs_atomic_read(&osd->od_r_in_flight)); - lprocfs_oh_tally_log2(&h[BRW_R_DISK_IOSIZE], size); - } else if (iobuf->dr_rw == 1) { - cfs_atomic_inc(&osd->od_w_in_flight); - lprocfs_oh_tally(&h[BRW_W_RPC_HIST], - cfs_atomic_read(&osd->od_w_in_flight)); - lprocfs_oh_tally_log2(&h[BRW_W_DISK_IOSIZE], size); - } else { - LBUG(); - } + struct osd_device *osd = iobuf->dr_dev; + struct obd_histogram *h = osd->od_brw_stats.hist; + + iobuf->dr_frags++; + atomic_inc(&iobuf->dr_numreqs); + + if (iobuf->dr_rw == 0) { + atomic_inc(&osd->od_r_in_flight); + lprocfs_oh_tally(&h[BRW_R_RPC_HIST], + atomic_read(&osd->od_r_in_flight)); + lprocfs_oh_tally_log2(&h[BRW_R_DISK_IOSIZE], size); + } else if (iobuf->dr_rw == 1) { + atomic_inc(&osd->od_w_in_flight); + lprocfs_oh_tally(&h[BRW_W_RPC_HIST], + atomic_read(&osd->od_w_in_flight)); + lprocfs_oh_tally_log2(&h[BRW_W_DISK_IOSIZE], size); + } else { + LBUG(); + } } static void osd_submit_bio(int rw, struct bio *bio) @@ -237,35 +252,32 @@ static void osd_submit_bio(int rw, struct bio *bio) static int can_be_merged(struct bio *bio, sector_t sector) { - unsigned int size; - - if (!bio) - return 0; + if (bio == NULL) + return 0; - size = bio->bi_size >> 9; - return bio->bi_sector + size == sector ? 1 : 0; + return bio_end_sector(bio) == sector ? 1 : 0; } static int osd_do_bio(struct osd_device *osd, struct inode *inode, struct osd_iobuf *iobuf) { - int blocks_per_page = CFS_PAGE_SIZE >> inode->i_blkbits; - struct page **pages = iobuf->dr_pages; - int npages = iobuf->dr_npages; - unsigned long *blocks = iobuf->dr_blocks; - int total_blocks = npages * blocks_per_page; - int sector_bits = inode->i_sb->s_blocksize_bits - 9; - unsigned int blocksize = inode->i_sb->s_blocksize; - struct bio *bio = NULL; - struct page *page; - unsigned int page_offset; - sector_t sector; - int nblocks; - int block_idx; - int page_idx; - int i; - int rc = 0; - ENTRY; + int blocks_per_page = PAGE_SIZE >> inode->i_blkbits; + struct page **pages = iobuf->dr_pages; + int npages = iobuf->dr_npages; + sector_t *blocks = iobuf->dr_blocks; + int total_blocks = npages * blocks_per_page; + int sector_bits = inode->i_sb->s_blocksize_bits - 9; + unsigned int blocksize = inode->i_sb->s_blocksize; + struct bio *bio = NULL; + struct page *page; + unsigned int page_offset; + sector_t sector; + int nblocks; + int block_idx; + int page_idx; + int i; + int rc = 0; + ENTRY; LASSERT(iobuf->dr_npages == npages); @@ -303,44 +315,34 @@ static int osd_do_bio(struct osd_device *osd, struct inode *inode, sector_bits)) nblocks++; - /* I only set the page to be constant only if it - * is mapped to a contiguous underlying disk block(s). - * It will then make sure the corresponding device - * cache of raid5 will be overwritten by this page. - * - jay */ - if (iobuf->dr_rw && (nblocks == blocks_per_page) && - mapping_cap_page_constant_write(inode->i_mapping)) - SetPageConstant(page); - if (bio != NULL && can_be_merged(bio, sector) && bio_add_page(bio, page, blocksize * nblocks, page_offset) != 0) continue; /* added this frag OK */ - if (bio != NULL) { - struct request_queue *q = - bdev_get_queue(bio->bi_bdev); - - /* Dang! I have to fragment this I/O */ - CDEBUG(D_INODE, "bio++ sz %d vcnt %d(%d) " - "sectors %d(%d) psg %d(%d) hsg %d(%d)\n", - bio->bi_size, - bio->bi_vcnt, bio->bi_max_vecs, - bio->bi_size >> 9, queue_max_sectors(q), + if (bio != NULL) { + struct request_queue *q = + bdev_get_queue(bio->bi_bdev); + unsigned int bi_size = bio_sectors(bio) << 9; + + /* Dang! I have to fragment this I/O */ + CDEBUG(D_INODE, "bio++ sz %d vcnt %d(%d) " + "sectors %d(%d) psg %d(%d) hsg %d(%d)\n", + bi_size, bio->bi_vcnt, bio->bi_max_vecs, + bio_sectors(bio), + queue_max_sectors(q), bio_phys_segments(q, bio), queue_max_phys_segments(q), - bio_hw_segments(q, bio), - queue_max_hw_segments(q)); - - record_start_io(iobuf, bio->bi_size); - osd_submit_bio(iobuf->dr_rw, bio); - } - - /* allocate new bio, limited by max BIO size, b=9945 */ - bio = bio_alloc(GFP_NOIO, max(BIO_MAX_PAGES, - (npages - page_idx) * - blocks_per_page)); + 0, queue_max_hw_segments(q)); + record_start_io(iobuf, bi_size); + osd_submit_bio(iobuf->dr_rw, bio); + } + + /* allocate new bio */ + bio = bio_alloc(GFP_NOIO, min(BIO_MAX_PAGES, + (npages - page_idx) * + blocks_per_page)); if (bio == NULL) { CERROR("Can't allocate bio %u*%u = %u pages\n", (npages - page_idx), blocks_per_page, @@ -349,37 +351,38 @@ static int osd_do_bio(struct osd_device *osd, struct inode *inode, goto out; } - bio->bi_bdev = inode->i_sb->s_bdev; - bio->bi_sector = sector; + bio->bi_bdev = inode->i_sb->s_bdev; + bio_set_sector(bio, sector); bio->bi_rw = (iobuf->dr_rw == 0) ? READ : WRITE; - bio->bi_end_io = dio_complete_routine; - bio->bi_private = iobuf; + bio->bi_end_io = dio_complete_routine; + bio->bi_private = iobuf; - rc = bio_add_page(bio, page, - blocksize * nblocks, page_offset); - LASSERT(rc != 0); - } - } + rc = bio_add_page(bio, page, + blocksize * nblocks, page_offset); + LASSERT(rc != 0); + } + } - if (bio != NULL) { - record_start_io(iobuf, bio->bi_size); - osd_submit_bio(iobuf->dr_rw, bio); - rc = 0; - } + if (bio != NULL) { + record_start_io(iobuf, bio_sectors(bio) << 9); + osd_submit_bio(iobuf->dr_rw, bio); + rc = 0; + } - out: - /* in order to achieve better IO throughput, we don't wait for writes - * completion here. instead we proceed with transaction commit in - * parallel and wait for IO completion once transaction is stopped - * see osd_trans_stop() for more details -bzzz */ - if (iobuf->dr_rw == 0) { - cfs_wait_event(iobuf->dr_wait, - cfs_atomic_read(&iobuf->dr_numreqs) == 0); - } +out: + /* in order to achieve better IO throughput, we don't wait for writes + * completion here. instead we proceed with transaction commit in + * parallel and wait for IO completion once transaction is stopped + * see osd_trans_stop() for more details -bzzz */ + if (iobuf->dr_rw == 0) { + wait_event(iobuf->dr_wait, + atomic_read(&iobuf->dr_numreqs) == 0); + osd_fini_iobuf(osd, iobuf); + } - if (rc == 0) - rc = iobuf->dr_error; - RETURN(rc); + if (rc == 0) + rc = iobuf->dr_error; + RETURN(rc); } static int osd_map_remote_to_local(loff_t offset, ssize_t len, int *nrpages, @@ -390,18 +393,18 @@ static int osd_map_remote_to_local(loff_t offset, ssize_t len, int *nrpages, *nrpages = 0; while (len > 0) { - int poff = offset & (CFS_PAGE_SIZE - 1); - int plen = CFS_PAGE_SIZE - poff; + int poff = offset & (PAGE_SIZE - 1); + int plen = PAGE_SIZE - poff; if (plen > len) plen = len; lnb->lnb_file_offset = offset; lnb->lnb_page_offset = poff; - lnb->len = plen; - /* lb->flags = rnb->flags; */ - lnb->flags = 0; - lnb->page = NULL; - lnb->rc = 0; + lnb->lnb_len = plen; + /* lnb->lnb_flags = rnb->rnb_flags; */ + lnb->lnb_flags = 0; + lnb->lnb_page = NULL; + lnb->lnb_rc = 0; LASSERTF(plen <= len, "plen %u, len %lld\n", plen, (long long) len); @@ -414,7 +417,7 @@ static int osd_map_remote_to_local(loff_t offset, ssize_t len, int *nrpages, RETURN(0); } -struct page *osd_get_page(struct dt_object *dt, loff_t offset, int rw) +static struct page *osd_get_page(struct dt_object *dt, loff_t offset, int rw) { struct inode *inode = osd_dt_obj(dt)->oo_inode; struct osd_device *d = osd_obj2dev(osd_dt_obj(dt)); @@ -422,7 +425,7 @@ struct page *osd_get_page(struct dt_object *dt, loff_t offset, int rw) LASSERT(inode); - page = find_or_create_page(inode->i_mapping, offset >> CFS_PAGE_SHIFT, + page = find_or_create_page(inode->i_mapping, offset >> PAGE_SHIFT, GFP_NOFS | __GFP_HIGHMEM); if (unlikely(page == NULL)) lprocfs_counter_add(d->od_stats, LPROC_OSD_NO_PAGE, 1); @@ -433,86 +436,534 @@ struct page *osd_get_page(struct dt_object *dt, loff_t offset, int rw) /* * there are following "locks": * journal_start - * i_alloc_sem * i_mutex * page lock + * + * osd write path: + * - lock page(s) + * - journal_start + * - truncate_sem + * + * ext4 vmtruncate: + * - lock pages, unlock + * - journal_start + * - lock partial page + * - i_data_sem + * + */ + +/** + * Unlock and release pages loaded by osd_bufs_get() + * + * Unlock \a npages pages from \a lnb and drop the refcount on them. + * + * \param env thread execution environment + * \param dt dt object undergoing IO (OSD object + methods) + * \param lnb array of pages undergoing IO + * \param npages number of pages in \a lnb + * + * \retval 0 always + */ +static int osd_bufs_put(const struct lu_env *env, struct dt_object *dt, + struct niobuf_local *lnb, int npages) +{ + int i; - * osd write path - * lock page(s) - * journal_start - * truncate_sem + for (i = 0; i < npages; i++) { + if (lnb[i].lnb_page == NULL) + continue; + LASSERT(PageLocked(lnb[i].lnb_page)); + unlock_page(lnb[i].lnb_page); + put_page(lnb[i].lnb_page); + lu_object_put(env, &dt->do_lu); + lnb[i].lnb_page = NULL; + } - * ext4 vmtruncate: - * lock pages, unlock - * journal_start - * lock partial page - * i_data_sem - -*/ -int osd_bufs_get(const struct lu_env *env, struct dt_object *d, loff_t pos, - ssize_t len, struct niobuf_local *lnb, int rw, - struct lustre_capa *capa) + RETURN(0); +} + +/** + * Load and lock pages undergoing IO + * + * Pages as described in the \a lnb array are fetched (from disk or cache) + * and locked for IO by the caller. + * + * DLM locking protects us from write and truncate competing for same region, + * but partial-page truncate can leave dirty pages in the cache for ldiskfs. + * It's possible the writeout on a such a page is in progress when we access + * it. It's also possible that during this writeout we put new (partial) data + * into the page, but won't be able to proceed in filter_commitrw_write(). + * Therefore, just wait for writeout completion as it should be rare enough. + * + * \param env thread execution environment + * \param dt dt object undergoing IO (OSD object + methods) + * \param pos byte offset of IO start + * \param len number of bytes of IO + * \param lnb array of extents undergoing IO + * \param rw read or write operation? + * \param capa capabilities + * + * \retval pages (zero or more) loaded successfully + * \retval -ENOMEM on memory/page allocation error + */ +static int osd_bufs_get(const struct lu_env *env, struct dt_object *dt, + loff_t pos, ssize_t len, struct niobuf_local *lnb, + int rw) { - struct osd_object *obj = osd_dt_obj(d); - int npages, i, rc = 0; + struct osd_object *obj = osd_dt_obj(dt); + int npages, i, rc = 0; - LASSERT(obj->oo_inode); + LASSERT(obj->oo_inode); - osd_map_remote_to_local(pos, len, &npages, lnb); + osd_map_remote_to_local(pos, len, &npages, lnb); - for (i = 0; i < npages; i++, lnb++) { + for (i = 0; i < npages; i++, lnb++) { + lnb->lnb_page = osd_get_page(dt, lnb->lnb_file_offset, rw); + if (lnb->lnb_page == NULL) + GOTO(cleanup, rc = -ENOMEM); - /* We still set up for ungranted pages so that granted pages - * can be written to disk as they were promised, and portals - * needs to keep the pages all aligned properly. */ - lnb->dentry = (void *) obj; + wait_on_page_writeback(lnb->lnb_page); + BUG_ON(PageWriteback(lnb->lnb_page)); - lnb->page = osd_get_page(d, lnb->lnb_file_offset, rw); - if (lnb->page == NULL) - GOTO(cleanup, rc = -ENOMEM); + lu_object_get(&dt->do_lu); + } - /* DLM locking protects us from write and truncate competing - * for same region, but truncate can leave dirty page in the - * cache. it's possible the writeout on a such a page is in - * progress when we access it. it's also possible that during - * this writeout we put new (partial) data, but then won't - * be able to proceed in filter_commitrw_write(). thus let's - * just wait for writeout completion, should be rare enough. - * -bzzz */ - wait_on_page_writeback(lnb->page); - BUG_ON(PageWriteback(lnb->page)); + RETURN(i); - lu_object_get(&d->do_lu); - } - rc = i; +cleanup: + if (i > 0) + osd_bufs_put(env, dt, lnb - i, i); + return rc; +} + +#ifndef HAVE_LDISKFS_MAP_BLOCKS + +#ifdef HAVE_EXT_PBLOCK /* Name changed to ext4_ext_pblock for kernel 2.6.35 */ +#define ldiskfs_ext_pblock(ex) ext_pblock((ex)) +#endif + +struct bpointers { + sector_t *blocks; + unsigned long start; + int num; + int init_num; + int create; +}; + +static long ldiskfs_ext_find_goal(struct inode *inode, + struct ldiskfs_ext_path *path, + unsigned long block, int *aflags) +{ + struct ldiskfs_inode_info *ei = LDISKFS_I(inode); + unsigned long bg_start; + unsigned long colour; + int depth; + + if (path) { + struct ldiskfs_extent *ex; + depth = path->p_depth; + + /* try to predict block placement */ + if ((ex = path[depth].p_ext)) + return ldiskfs_ext_pblock(ex) + + (block - le32_to_cpu(ex->ee_block)); + + /* it looks index is empty + * try to find starting from index itself */ + if (path[depth].p_bh) + return path[depth].p_bh->b_blocknr; + } + + /* OK. use inode's group */ + bg_start = (ei->i_block_group * LDISKFS_BLOCKS_PER_GROUP(inode->i_sb)) + + le32_to_cpu(LDISKFS_SB(inode->i_sb)->s_es->s_first_data_block); + colour = (current->pid % 16) * + (LDISKFS_BLOCKS_PER_GROUP(inode->i_sb) / 16); + return bg_start + colour + block; +} + +static unsigned long new_blocks(handle_t *handle, struct inode *inode, + struct ldiskfs_ext_path *path, + unsigned long block, unsigned long *count, + int *err) +{ + struct ldiskfs_allocation_request ar; + unsigned long pblock; + int aflags; + + /* find neighbour allocated blocks */ + ar.lleft = block; + *err = ldiskfs_ext_search_left(inode, path, &ar.lleft, &ar.pleft); + if (*err) + return 0; + ar.lright = block; + *err = ldiskfs_ext_search_right(inode, path, &ar.lright, &ar.pright); + if (*err) + return 0; + + /* allocate new block */ + ar.goal = ldiskfs_ext_find_goal(inode, path, block, &aflags); + ar.inode = inode; + ar.logical = block; + ar.len = *count; + ar.flags = LDISKFS_MB_HINT_DATA; + pblock = ldiskfs_mb_new_blocks(handle, &ar, err); + *count = ar.len; + return pblock; +} + +static int ldiskfs_ext_new_extent_cb(struct inode *inode, + struct ldiskfs_ext_path *path, + struct ldiskfs_ext_cache *cex, +#ifdef HAVE_EXT_PREPARE_CB_EXTENT + struct ldiskfs_extent *ex, +#endif + void *cbdata) +{ + struct bpointers *bp = cbdata; + struct ldiskfs_extent nex; + unsigned long pblock = 0; + unsigned long tgen; + int err, i; + unsigned long count; + handle_t *handle; + +#ifdef LDISKFS_EXT_CACHE_EXTENT /* until kernel 2.6.37 */ + if (cex->ec_type == LDISKFS_EXT_CACHE_EXTENT) { +#else + if ((cex->ec_len != 0) && (cex->ec_start != 0)) { +#endif + err = EXT_CONTINUE; + goto map; + } + + if (bp->create == 0) { + i = 0; + if (cex->ec_block < bp->start) + i = bp->start - cex->ec_block; + if (i >= cex->ec_len) + CERROR("nothing to do?! i = %d, e_num = %u\n", + i, cex->ec_len); + for (; i < cex->ec_len && bp->num; i++) { + *(bp->blocks) = 0; + bp->blocks++; + bp->num--; + bp->start++; + } + + return EXT_CONTINUE; + } + + tgen = LDISKFS_I(inode)->i_ext_generation; + count = ldiskfs_ext_calc_credits_for_insert(inode, path); + + handle = osd_journal_start(inode, LDISKFS_HT_MISC, + count + LDISKFS_ALLOC_NEEDED + 1); + if (IS_ERR(handle)) { + return PTR_ERR(handle); + } + + if (tgen != LDISKFS_I(inode)->i_ext_generation) { + /* the tree has changed. so path can be invalid at moment */ + ldiskfs_journal_stop(handle); + return EXT_REPEAT; + } + + /* In 2.6.32 kernel, ldiskfs_ext_walk_space()'s callback func is not + * protected by i_data_sem as whole. so we patch it to store + * generation to path and now verify the tree hasn't changed */ + down_write((&LDISKFS_I(inode)->i_data_sem)); + + /* validate extent, make sure the extent tree does not changed */ + if (LDISKFS_I(inode)->i_ext_generation != path[0].p_generation) { + /* cex is invalid, try again */ + up_write(&LDISKFS_I(inode)->i_data_sem); + ldiskfs_journal_stop(handle); + return EXT_REPEAT; + } + + count = cex->ec_len; + pblock = new_blocks(handle, inode, path, cex->ec_block, &count, &err); + if (!pblock) + goto out; + BUG_ON(count > cex->ec_len); + + /* insert new extent */ + nex.ee_block = cpu_to_le32(cex->ec_block); + ldiskfs_ext_store_pblock(&nex, pblock); + nex.ee_len = cpu_to_le16(count); + err = ldiskfs_ext_insert_extent(handle, inode, path, &nex, 0); + if (err) { + /* free data blocks we just allocated */ + /* not a good idea to call discard here directly, + * but otherwise we'd need to call it every free() */ + ldiskfs_discard_preallocations(inode); +#ifdef HAVE_EXT_FREE_BLOCK_WITH_BUFFER_HEAD /* Introduced in 2.6.32-rc7 */ + ldiskfs_free_blocks(handle, inode, NULL, + ldiskfs_ext_pblock(&nex), + le16_to_cpu(nex.ee_len), 0); +#else + ldiskfs_free_blocks(handle, inode, ldiskfs_ext_pblock(&nex), + le16_to_cpu(nex.ee_len), 0); +#endif + goto out; + } + + /* + * Putting len of the actual extent we just inserted, + * we are asking ldiskfs_ext_walk_space() to continue + * scaning after that block + */ + cex->ec_len = le16_to_cpu(nex.ee_len); + cex->ec_start = ldiskfs_ext_pblock(&nex); + BUG_ON(le16_to_cpu(nex.ee_len) == 0); + BUG_ON(le32_to_cpu(nex.ee_block) != cex->ec_block); + +out: + up_write((&LDISKFS_I(inode)->i_data_sem)); + ldiskfs_journal_stop(handle); +map: + if (err >= 0) { + /* map blocks */ + if (bp->num == 0) { + CERROR("hmm. why do we find this extent?\n"); + CERROR("initial space: %lu:%u\n", + bp->start, bp->init_num); +#ifdef LDISKFS_EXT_CACHE_EXTENT /* until kernel 2.6.37 */ + CERROR("current extent: %u/%u/%llu %d\n", + cex->ec_block, cex->ec_len, + (unsigned long long)cex->ec_start, + cex->ec_type); +#else + CERROR("current extent: %u/%u/%llu\n", + cex->ec_block, cex->ec_len, + (unsigned long long)cex->ec_start); +#endif + } + i = 0; + if (cex->ec_block < bp->start) + i = bp->start - cex->ec_block; + if (i >= cex->ec_len) + CERROR("nothing to do?! i = %d, e_num = %u\n", + i, cex->ec_len); + for (; i < cex->ec_len && bp->num; i++) { + *(bp->blocks) = cex->ec_start + i; + if (pblock != 0) { + /* unmap any possible underlying metadata from + * the block device mapping. bug 6998. */ + unmap_underlying_metadata(inode->i_sb->s_bdev, + *(bp->blocks)); + } + bp->blocks++; + bp->num--; + bp->start++; + } + } + return err; +} + +static int osd_ldiskfs_map_nblocks(struct inode *inode, unsigned long index, + int clen, sector_t *blocks, int create) +{ + int blocks_per_page = PAGE_SIZE >> inode->i_blkbits; + struct bpointers bp; + int err; + + if (index + clen >= inode->i_sb->s_maxbytes >> PAGE_SHIFT) + return -EFBIG; + + bp.blocks = blocks; + bp.start = index * blocks_per_page; + bp.init_num = bp.num = clen * blocks_per_page; + bp.create = create; + + CDEBUG(D_OTHER, "blocks %lu-%lu requested for inode %u\n", + bp.start, bp.start + bp.num - 1, (unsigned)inode->i_ino); + + err = ldiskfs_ext_walk_space(inode, bp.start, bp.num, + ldiskfs_ext_new_extent_cb, &bp); + ldiskfs_ext_invalidate_cache(inode); + + return err; +} + +static int osd_ldiskfs_map_bm_inode_pages(struct inode *inode, + struct page **page, int pages, + sector_t *blocks, int create) +{ + int blocks_per_page = PAGE_SIZE >> inode->i_blkbits; + pgoff_t bitmap_max_page_index; + sector_t *b; + int rc = 0, i; + + bitmap_max_page_index = LDISKFS_SB(inode->i_sb)->s_bitmap_maxbytes >> + PAGE_SHIFT; + for (i = 0, b = blocks; i < pages; i++, page++) { + if ((*page)->index + 1 >= bitmap_max_page_index) { + rc = -EFBIG; + break; + } + rc = ldiskfs_map_inode_page(inode, *page, b, create); + if (rc) { + CERROR("ino %lu, blk %llu create %d: rc %d\n", + inode->i_ino, + (unsigned long long)*b, create, rc); + break; + } + b += blocks_per_page; + } + return rc; +} + +static int osd_ldiskfs_map_ext_inode_pages(struct inode *inode, + struct page **page, + int pages, sector_t *blocks, + int create) +{ + int rc = 0, i = 0, clen = 0; + struct page *fp = NULL; + + CDEBUG(D_OTHER, "inode %lu: map %d pages from %lu\n", + inode->i_ino, pages, (*page)->index); + + /* pages are sorted already. so, we just have to find + * contig. space and process them properly */ + while (i < pages) { + if (fp == NULL) { + /* start new extent */ + fp = *page++; + clen = 1; + i++; + continue; + } else if (fp->index + clen == (*page)->index) { + /* continue the extent */ + page++; + clen++; + i++; + continue; + } + + /* process found extent */ + rc = osd_ldiskfs_map_nblocks(inode, fp->index, clen, + blocks, create); + if (rc) + GOTO(cleanup, rc); + + /* look for next extent */ + fp = NULL; + blocks += clen * (PAGE_SIZE >> inode->i_blkbits); + } + + if (fp) + rc = osd_ldiskfs_map_nblocks(inode, fp->index, clen, + blocks, create); cleanup: - RETURN(rc); + return rc; } -static int osd_bufs_put(const struct lu_env *env, struct dt_object *dt, - struct niobuf_local *lnb, int npages) +static int osd_ldiskfs_map_inode_pages(struct inode *inode, struct page **page, + int pages, sector_t *blocks, + int create) { - struct osd_thread_info *oti = osd_oti_get(env); - struct osd_iobuf *iobuf = &oti->oti_iobuf; - struct osd_device *d = osd_obj2dev(osd_dt_obj(dt)); - int i; + int rc; - /* to do IO stats, notice we do this here because - * osd_do_bio() doesn't wait for write to complete */ - osd_fini_iobuf(d, iobuf); + if (LDISKFS_I(inode)->i_flags & LDISKFS_EXTENTS_FL) { + rc = osd_ldiskfs_map_ext_inode_pages(inode, page, pages, + blocks, create); + return rc; + } + rc = osd_ldiskfs_map_bm_inode_pages(inode, page, pages, blocks, create); - for (i = 0; i < npages; i++) { - if (lnb[i].page == NULL) - continue; - LASSERT(PageLocked(lnb[i].page)); - unlock_page(lnb[i].page); - page_cache_release(lnb[i].page); - lu_object_put(env, &dt->do_lu); - lnb[i].page = NULL; - } - RETURN(0); + return rc; } +#else +static int osd_ldiskfs_map_inode_pages(struct inode *inode, struct page **page, + int pages, sector_t *blocks, + int create) +{ + int blocks_per_page = PAGE_SIZE >> inode->i_blkbits; + int rc = 0, i = 0; + struct page *fp = NULL; + int clen = 0; + pgoff_t max_page_index; + handle_t *handle = NULL; + + max_page_index = inode->i_sb->s_maxbytes >> PAGE_SHIFT; + + CDEBUG(D_OTHER, "inode %lu: map %d pages from %lu\n", + inode->i_ino, pages, (*page)->index); + + if (create) { + create = LDISKFS_GET_BLOCKS_CREATE; + handle = ldiskfs_journal_current_handle(); + LASSERT(handle != NULL); + rc = osd_attach_jinode(inode); + if (rc) + return rc; + } + /* pages are sorted already. so, we just have to find + * contig. space and process them properly */ + while (i < pages) { + long blen, total = 0; + struct ldiskfs_map_blocks map = { 0 }; + + if (fp == NULL) { /* start new extent */ + fp = *page++; + clen = 1; + if (++i != pages) + continue; + } else if (fp->index + clen == (*page)->index) { + /* continue the extent */ + page++; + clen++; + if (++i != pages) + continue; + } + if (fp->index + clen >= max_page_index) + GOTO(cleanup, rc = -EFBIG); + /* process found extent */ + map.m_lblk = fp->index * blocks_per_page; + map.m_len = blen = clen * blocks_per_page; +cont_map: + rc = ldiskfs_map_blocks(handle, inode, &map, create); + if (rc >= 0) { + int c = 0; + for (; total < blen && c < map.m_len; c++, total++) { + if (rc == 0) { + *(blocks + total) = 0; + total++; + break; + } else { + *(blocks + total) = map.m_pblk + c; + /* unmap any possible underlying + * metadata from the block device + * mapping. bug 6998. */ + if ((map.m_flags & LDISKFS_MAP_NEW) && + create) + unmap_underlying_metadata( + inode->i_sb->s_bdev, + map.m_pblk + c); + } + } + rc = 0; + } + if (rc == 0 && total < blen) { + map.m_lblk = fp->index * blocks_per_page + total; + map.m_len = blen - total; + goto cont_map; + } + if (rc != 0) + GOTO(cleanup, rc); + + /* look for next extent */ + fp = NULL; + blocks += blocks_per_page * clen; + } +cleanup: + return rc; +} +#endif /* HAVE_LDISKFS_MAP_BLOCKS */ static int osd_write_prep(const struct lu_env *env, struct dt_object *dt, struct niobuf_local *lnb, int npages) @@ -521,9 +972,9 @@ static int osd_write_prep(const struct lu_env *env, struct dt_object *dt, struct osd_iobuf *iobuf = &oti->oti_iobuf; struct inode *inode = osd_dt_obj(dt)->oo_inode; struct osd_device *osd = osd_obj2dev(osd_dt_obj(dt)); - struct timeval start; - struct timeval end; - unsigned long timediff; + ktime_t start; + ktime_t end; + s64 timediff; ssize_t isize; __s64 maxidx; int rc = 0; @@ -532,59 +983,59 @@ static int osd_write_prep(const struct lu_env *env, struct dt_object *dt, LASSERT(inode); - osd_init_iobuf(osd, iobuf, 0); + rc = osd_init_iobuf(osd, iobuf, 0, npages); + if (unlikely(rc != 0)) + RETURN(rc); - isize = i_size_read(inode); - maxidx = ((isize + CFS_PAGE_SIZE - 1) >> CFS_PAGE_SHIFT) - 1; + isize = i_size_read(inode); + maxidx = ((isize + PAGE_SIZE - 1) >> PAGE_SHIFT) - 1; if (osd->od_writethrough_cache) cache = 1; if (isize > osd->od_readcache_max_filesize) cache = 0; - cfs_gettimeofday(&start); - for (i = 0; i < npages; i++) { + start = ktime_get(); + for (i = 0; i < npages; i++) { - if (cache == 0) - generic_error_remove_page(inode->i_mapping, - lnb[i].page); + if (cache == 0) + generic_error_remove_page(inode->i_mapping, + lnb[i].lnb_page); - /* - * till commit the content of the page is undefined - * we'll set it uptodate once bulk is done. otherwise - * subsequent reads can access non-stable data - */ - ClearPageUptodate(lnb[i].page); + /* + * till commit the content of the page is undefined + * we'll set it uptodate once bulk is done. otherwise + * subsequent reads can access non-stable data + */ + ClearPageUptodate(lnb[i].lnb_page); - if (lnb[i].len == CFS_PAGE_SIZE) - continue; + if (lnb[i].lnb_len == PAGE_SIZE) + continue; - if (maxidx >= lnb[i].page->index) { - osd_iobuf_add_page(iobuf, lnb[i].page); - } else { - long off; - char *p = kmap(lnb[i].page); + if (maxidx >= lnb[i].lnb_page->index) { + osd_iobuf_add_page(iobuf, lnb[i].lnb_page); + } else { + long off; + char *p = kmap(lnb[i].lnb_page); off = lnb[i].lnb_page_offset; if (off) memset(p, 0, off); - off = (lnb[i].lnb_page_offset + lnb[i].len) & - ~CFS_PAGE_MASK; - if (off) - memset(p + off, 0, CFS_PAGE_SIZE - off); - kunmap(lnb[i].page); - } - } - cfs_gettimeofday(&end); - timediff = cfs_timeval_sub(&end, &start, NULL); - lprocfs_counter_add(osd->od_stats, LPROC_OSD_GET_PAGE, timediff); + off = (lnb[i].lnb_page_offset + lnb[i].lnb_len) & + ~PAGE_MASK; + if (off) + memset(p + off, 0, PAGE_SIZE - off); + kunmap(lnb[i].lnb_page); + } + } + end = ktime_get(); + timediff = ktime_us_delta(end, start); + lprocfs_counter_add(osd->od_stats, LPROC_OSD_GET_PAGE, timediff); if (iobuf->dr_npages) { - rc = osd->od_fsops->fs_map_inode_pages(inode, iobuf->dr_pages, - iobuf->dr_npages, - iobuf->dr_blocks, - oti->oti_created, - 0, NULL); + rc = osd_ldiskfs_map_inode_pages(inode, iobuf->dr_pages, + iobuf->dr_npages, + iobuf->dr_blocks, 0); if (likely(rc == 0)) { rc = osd_do_bio(osd, inode, iobuf); /* do IO stats for preparation reads */ @@ -594,29 +1045,57 @@ static int osd_write_prep(const struct lu_env *env, struct dt_object *dt, RETURN(rc); } -/* Check if a block is allocated or not */ -static int osd_is_mapped(struct inode *inode, obd_size offset) -{ - sector_t (*fs_bmap)(struct address_space *, sector_t); +struct osd_fextent { + sector_t start; + sector_t end; + unsigned int mapped:1; +}; - fs_bmap = inode->i_mapping->a_ops->bmap; +static int osd_is_mapped(struct dt_object *dt, __u64 offset, + struct osd_fextent *cached_extent) +{ + struct inode *inode = osd_dt_obj(dt)->oo_inode; + sector_t block = offset >> inode->i_blkbits; + sector_t start; + struct fiemap_extent_info fei = { 0 }; + struct fiemap_extent fe = { 0 }; + mm_segment_t saved_fs; + int rc; - /* We can't know if we are overwriting or not */ - if (unlikely(fs_bmap == NULL)) - return 0; + if (block >= cached_extent->start && block < cached_extent->end) + return cached_extent->mapped; if (i_size_read(inode) == 0) return 0; /* Beyond EOF, must not be mapped */ - if (((i_size_read(inode) - 1) >> inode->i_blkbits) < - (offset >> inode->i_blkbits)) + if (((i_size_read(inode) - 1) >> inode->i_blkbits) < block) return 0; - if (fs_bmap(inode->i_mapping, offset >> inode->i_blkbits) == 0) + fei.fi_extents_max = 1; + fei.fi_extents_start = &fe; + + saved_fs = get_fs(); + set_fs(get_ds()); + rc = inode->i_op->fiemap(inode, &fei, offset, FIEMAP_MAX_OFFSET-offset); + set_fs(saved_fs); + if (rc != 0) return 0; - return 1; + start = fe.fe_logical >> inode->i_blkbits; + + if (start > block) { + cached_extent->start = block; + cached_extent->end = start; + cached_extent->mapped = 0; + } else { + cached_extent->start = start; + cached_extent->end = (fe.fe_logical + fe.fe_length) >> + inode->i_blkbits; + cached_extent->mapped = 1; + } + + return cached_extent->mapped; } static int osd_declare_write_commit(const struct lu_env *env, @@ -633,8 +1112,10 @@ static int osd_declare_write_commit(const struct lu_env *env, int newblocks; int rc = 0; int flags = 0; + int credits = 0; bool ignore_quota = false; long long quota_space = 0; + struct osd_fextent extent = { 0 }; ENTRY; LASSERT(handle != NULL); @@ -646,19 +1127,24 @@ static int osd_declare_write_commit(const struct lu_env *env, /* calculate number of extents (probably better to pass nb) */ for (i = 0; i < npages; i++) { if (i && lnb[i].lnb_file_offset != - lnb[i - 1].lnb_file_offset + lnb[i - 1].len) + lnb[i - 1].lnb_file_offset + lnb[i - 1].lnb_len) extents++; - if (!osd_is_mapped(inode, lnb[i].lnb_file_offset)) - quota_space += CFS_PAGE_SIZE; + if (!osd_is_mapped(dt, lnb[i].lnb_file_offset, &extent)) + quota_space += PAGE_SIZE; /* ignore quota for the whole request if any page is from * client cache or written by root. * + * XXX once we drop the 1.8 client support, the checking + * for whether page is from cache can be simplified as: + * !(lnb[i].flags & OBD_BRW_SYNC) + * * XXX we could handle this on per-lnb basis as done by * grant. */ - if ((lnb[i].flags & OBD_BRW_NOQUOTA) || - !(lnb[i].flags & OBD_BRW_SYNC)) + if ((lnb[i].lnb_flags & OBD_BRW_NOQUOTA) || + (lnb[i].lnb_flags & (OBD_BRW_FROM_GRANT | OBD_BRW_SYNC)) == + OBD_BRW_FROM_GRANT) ignore_quota = true; } @@ -675,14 +1161,14 @@ static int osd_declare_write_commit(const struct lu_env *env, depth = ext_depth(inode); depth = max(depth, 1) + 1; newblocks += depth; - oh->ot_credits++; /* inode */ - oh->ot_credits += depth * 2 * extents; - } else { - depth = 3; - newblocks += depth; - oh->ot_credits++; /* inode */ - oh->ot_credits += depth * extents; - } + credits++; /* inode */ + credits += depth * 2 * extents; + } else { + depth = 3; + newblocks += depth; + credits++; /* inode */ + credits += depth * extents; + } /* quota space for metadata blocks */ quota_space += depth * extents * LDISKFS_BLOCK_SIZE(osd_sb(osd)); @@ -694,30 +1180,32 @@ static int osd_declare_write_commit(const struct lu_env *env, /* we can't dirty more bitmap blocks than exist */ if (newblocks > LDISKFS_SB(osd_sb(osd))->s_groups_count) - oh->ot_credits += LDISKFS_SB(osd_sb(osd))->s_groups_count; + credits += LDISKFS_SB(osd_sb(osd))->s_groups_count; else - oh->ot_credits += newblocks; + credits += newblocks; - /* we can't dirty more gd blocks than exist */ - if (newblocks > LDISKFS_SB(osd_sb(osd))->s_gdb_count) - oh->ot_credits += LDISKFS_SB(osd_sb(osd))->s_gdb_count; - else - oh->ot_credits += newblocks; + /* we can't dirty more gd blocks than exist */ + if (newblocks > LDISKFS_SB(osd_sb(osd))->s_gdb_count) + credits += LDISKFS_SB(osd_sb(osd))->s_gdb_count; + else + credits += newblocks; + + osd_trans_declare_op(env, oh, OSD_OT_WRITE, credits); /* make sure the over quota flags were not set */ - lnb[0].flags &= ~(OBD_BRW_OVER_USRQUOTA | OBD_BRW_OVER_GRPQUOTA); + lnb[0].lnb_flags &= ~(OBD_BRW_OVER_USRQUOTA | OBD_BRW_OVER_GRPQUOTA); - rc = osd_declare_inode_qid(env, inode->i_uid, inode->i_gid, - quota_space, oh, true, true, &flags, - ignore_quota); + rc = osd_declare_inode_qid(env, i_uid_read(inode), i_gid_read(inode), + quota_space, oh, osd_dt_obj(dt), true, + &flags, ignore_quota); /* we need only to store the overquota flags in the first lnb for * now, once we support multiple objects BRW, this code needs be * revised. */ if (flags & QUOTA_FL_OVER_USRQUOTA) - lnb[0].flags |= OBD_BRW_OVER_USRQUOTA; + lnb[0].lnb_flags |= OBD_BRW_OVER_USRQUOTA; if (flags & QUOTA_FL_OVER_GRPQUOTA) - lnb[0].flags |= OBD_BRW_OVER_GRPQUOTA; + lnb[0].lnb_flags |= OBD_BRW_OVER_GRPQUOTA; RETURN(rc); } @@ -733,83 +1221,97 @@ static int osd_write_commit(const struct lu_env *env, struct dt_object *dt, struct osd_device *osd = osd_obj2dev(osd_dt_obj(dt)); loff_t isize; int rc = 0, i; + struct osd_fextent extent = { 0 }; LASSERT(inode); - osd_init_iobuf(osd, iobuf, 1); - isize = i_size_read(inode); + rc = osd_init_iobuf(osd, iobuf, 1, npages); + if (unlikely(rc != 0)) + RETURN(rc); + + isize = i_size_read(inode); ll_vfs_dq_init(inode); for (i = 0; i < npages; i++) { - if (lnb[i].rc == -ENOSPC && - osd_is_mapped(inode, lnb[i].lnb_file_offset)) { - /* Allow the write to proceed if overwriting an - * existing block */ - lnb[i].rc = 0; - } + if (lnb[i].lnb_rc == -ENOSPC && + osd_is_mapped(dt, lnb[i].lnb_file_offset, &extent)) { + /* Allow the write to proceed if overwriting an + * existing block */ + lnb[i].lnb_rc = 0; + } - if (lnb[i].rc) { /* ENOSPC, network RPC error, etc. */ - CDEBUG(D_INODE, "Skipping [%d] == %d\n", i, - lnb[i].rc); - LASSERT(lnb[i].page); - generic_error_remove_page(inode->i_mapping,lnb[i].page); - continue; - } + if (lnb[i].lnb_rc) { /* ENOSPC, network RPC error, etc. */ + CDEBUG(D_INODE, "Skipping [%d] == %d\n", i, + lnb[i].lnb_rc); + LASSERT(lnb[i].lnb_page); + generic_error_remove_page(inode->i_mapping, + lnb[i].lnb_page); + continue; + } - LASSERT(PageLocked(lnb[i].page)); - LASSERT(!PageWriteback(lnb[i].page)); + LASSERT(PageLocked(lnb[i].lnb_page)); + LASSERT(!PageWriteback(lnb[i].lnb_page)); - if (lnb[i].lnb_file_offset + lnb[i].len > isize) - isize = lnb[i].lnb_file_offset + lnb[i].len; + if (lnb[i].lnb_file_offset + lnb[i].lnb_len > isize) + isize = lnb[i].lnb_file_offset + lnb[i].lnb_len; - /* - * Since write and truncate are serialized by oo_sem, even - * partial-page truncate should not leave dirty pages in the - * page cache. - */ - LASSERT(!PageDirty(lnb[i].page)); + /* + * Since write and truncate are serialized by oo_sem, even + * partial-page truncate should not leave dirty pages in the + * page cache. + */ + LASSERT(!PageDirty(lnb[i].lnb_page)); - SetPageUptodate(lnb[i].page); + SetPageUptodate(lnb[i].lnb_page); - osd_iobuf_add_page(iobuf, lnb[i].page); + osd_iobuf_add_page(iobuf, lnb[i].lnb_page); } + osd_trans_exec_op(env, thandle, OSD_OT_WRITE); + if (OBD_FAIL_CHECK(OBD_FAIL_OST_MAPBLK_ENOSPC)) { rc = -ENOSPC; } else if (iobuf->dr_npages > 0) { - rc = osd->od_fsops->fs_map_inode_pages(inode, iobuf->dr_pages, - iobuf->dr_npages, - iobuf->dr_blocks, - oti->oti_created, - 1, NULL); + rc = osd_ldiskfs_map_inode_pages(inode, iobuf->dr_pages, + iobuf->dr_npages, + iobuf->dr_blocks, 1); } else { /* no pages to write, no transno is needed */ thandle->th_local = 1; } - if (likely(rc == 0)) { - if (isize > i_size_read(inode)) { - i_size_write(inode, isize); - LDISKFS_I(inode)->i_disksize = isize; - inode->i_sb->s_op->dirty_inode(inode); - } + if (likely(rc == 0)) { + spin_lock(&inode->i_lock); + if (isize > i_size_read(inode)) { + i_size_write(inode, isize); + LDISKFS_I(inode)->i_disksize = isize; + spin_unlock(&inode->i_lock); + ll_dirty_inode(inode, I_DIRTY_DATASYNC); + } else { + spin_unlock(&inode->i_lock); + } - rc = osd_do_bio(osd, inode, iobuf); - /* we don't do stats here as in read path because - * write is async: we'll do this in osd_put_bufs() */ - } + rc = osd_do_bio(osd, inode, iobuf); + /* we don't do stats here as in read path because + * write is async: we'll do this in osd_put_bufs() */ + } else { + osd_fini_iobuf(osd, iobuf); + } - if (unlikely(rc != 0)) { - /* if write fails, we should drop pages from the cache */ - for (i = 0; i < npages; i++) { - if (lnb[i].page == NULL) - continue; - LASSERT(PageLocked(lnb[i].page)); - generic_error_remove_page(inode->i_mapping,lnb[i].page); - } - } + osd_trans_exec_check(env, thandle, OSD_OT_WRITE); - RETURN(rc); + if (unlikely(rc != 0)) { + /* if write fails, we should drop pages from the cache */ + for (i = 0; i < npages; i++) { + if (lnb[i].lnb_page == NULL) + continue; + LASSERT(PageLocked(lnb[i].lnb_page)); + generic_error_remove_page(inode->i_mapping, + lnb[i].lnb_page); + } + } + + RETURN(rc); } static int osd_read_prep(const struct lu_env *env, struct dt_object *dt, @@ -819,56 +1321,70 @@ static int osd_read_prep(const struct lu_env *env, struct dt_object *dt, struct osd_iobuf *iobuf = &oti->oti_iobuf; struct inode *inode = osd_dt_obj(dt)->oo_inode; struct osd_device *osd = osd_obj2dev(osd_dt_obj(dt)); - struct timeval start, end; - unsigned long timediff; - int rc = 0, i, m = 0, cache = 0; + int rc = 0, i, cache = 0, cache_hits = 0, cache_misses = 0; + ktime_t start, end; + s64 timediff; + loff_t isize; LASSERT(inode); - osd_init_iobuf(osd, iobuf, 0); + rc = osd_init_iobuf(osd, iobuf, 0, npages); + if (unlikely(rc != 0)) + RETURN(rc); - if (osd->od_read_cache) - cache = 1; - if (i_size_read(inode) > osd->od_readcache_max_filesize) - cache = 0; + isize = i_size_read(inode); - cfs_gettimeofday(&start); - for (i = 0; i < npages; i++) { + if (osd->od_read_cache) + cache = 1; + if (isize > osd->od_readcache_max_filesize) + cache = 0; - if (i_size_read(inode) <= lnb[i].lnb_file_offset) - /* If there's no more data, abort early. - * lnb->rc == 0, so it's easy to detect later. */ - break; + start = ktime_get(); + for (i = 0; i < npages; i++) { - if (i_size_read(inode) < - lnb[i].lnb_file_offset + lnb[i].len - 1) - lnb[i].rc = i_size_read(inode) - lnb[i].lnb_file_offset; - else - lnb[i].rc = lnb[i].len; - m += lnb[i].len; - - lprocfs_counter_add(osd->od_stats, LPROC_OSD_CACHE_ACCESS, 1); - if (PageUptodate(lnb[i].page)) { - lprocfs_counter_add(osd->od_stats, - LPROC_OSD_CACHE_HIT, 1); - } else { - lprocfs_counter_add(osd->od_stats, - LPROC_OSD_CACHE_MISS, 1); - osd_iobuf_add_page(iobuf, lnb[i].page); - } - if (cache == 0) - generic_error_remove_page(inode->i_mapping,lnb[i].page); - } - cfs_gettimeofday(&end); - timediff = cfs_timeval_sub(&end, &start, NULL); - lprocfs_counter_add(osd->od_stats, LPROC_OSD_GET_PAGE, timediff); + if (isize <= lnb[i].lnb_file_offset) + /* If there's no more data, abort early. + * lnb->lnb_rc == 0, so it's easy to detect later. */ + break; + + if (isize < lnb[i].lnb_file_offset + lnb[i].lnb_len) + lnb[i].lnb_rc = isize - lnb[i].lnb_file_offset; + else + lnb[i].lnb_rc = lnb[i].lnb_len; + + /* Bypass disk read if fail_loc is set properly */ + if (OBD_FAIL_CHECK(OBD_FAIL_OST_FAKE_RW)) + SetPageUptodate(lnb[i].lnb_page); + + if (PageUptodate(lnb[i].lnb_page)) { + cache_hits++; + } else { + cache_misses++; + osd_iobuf_add_page(iobuf, lnb[i].lnb_page); + } + + if (cache == 0) + generic_error_remove_page(inode->i_mapping, + lnb[i].lnb_page); + } + end = ktime_get(); + timediff = ktime_us_delta(end, start); + lprocfs_counter_add(osd->od_stats, LPROC_OSD_GET_PAGE, timediff); + + if (cache_hits != 0) + lprocfs_counter_add(osd->od_stats, LPROC_OSD_CACHE_HIT, + cache_hits); + if (cache_misses != 0) + lprocfs_counter_add(osd->od_stats, LPROC_OSD_CACHE_MISS, + cache_misses); + if (cache_hits + cache_misses != 0) + lprocfs_counter_add(osd->od_stats, LPROC_OSD_CACHE_ACCESS, + cache_hits + cache_misses); if (iobuf->dr_npages) { - rc = osd->od_fsops->fs_map_inode_pages(inode, iobuf->dr_pages, - iobuf->dr_npages, - iobuf->dr_blocks, - oti->oti_created, - 0, NULL); + rc = osd_ldiskfs_map_inode_pages(inode, iobuf->dr_pages, + iobuf->dr_npages, + iobuf->dr_blocks, 0); rc = osd_do_bio(osd, inode, iobuf); /* IO stats will be done in osd_bufs_put() */ @@ -905,13 +1421,12 @@ int osd_ldiskfs_read(struct inode *inode, void *buf, int size, loff_t *offs) int blocksize; int csize; int boffs; - int err; /* prevent reading after eof */ - cfs_spin_lock(&inode->i_lock); - if (i_size_read(inode) < *offs + size) { + spin_lock(&inode->i_lock); + if (i_size_read(inode) < *offs + size) { loff_t diff = i_size_read(inode) - *offs; - cfs_spin_unlock(&inode->i_lock); + spin_unlock(&inode->i_lock); if (diff < 0) { CDEBUG(D_EXT2, "size %llu is too short to read @%llu\n", i_size_read(inode), *offs); @@ -921,9 +1436,9 @@ int osd_ldiskfs_read(struct inode *inode, void *buf, int size, loff_t *offs) } else { size = diff; } - } else { - cfs_spin_unlock(&inode->i_lock); - } + } else { + spin_unlock(&inode->i_lock); + } blocksize = 1 << inode->i_blkbits; osize = size; @@ -931,16 +1446,21 @@ int osd_ldiskfs_read(struct inode *inode, void *buf, int size, loff_t *offs) block = *offs >> inode->i_blkbits; boffs = *offs & (blocksize - 1); csize = min(blocksize - boffs, size); - bh = ldiskfs_bread(NULL, inode, block, 0, &err); - if (!bh) { - CERROR("%s: can't read %u@%llu on ino %lu: rc = %d\n", - LDISKFS_SB(inode->i_sb)->s_es->s_volume_name, - csize, *offs, inode->i_ino, err); - return err; - } + bh = __ldiskfs_bread(NULL, inode, block, 0); + if (IS_ERR(bh)) { + CERROR("%s: can't read %u@%llu on ino %lu: rc = %ld\n", + LDISKFS_SB(inode->i_sb)->s_es->s_volume_name, + csize, *offs, inode->i_ino, + PTR_ERR(bh)); + return PTR_ERR(bh); + } - memcpy(buf, bh->b_data + boffs, csize); - brelse(bh); + if (bh != NULL) { + memcpy(buf, bh->b_data + boffs, csize); + brelse(bh); + } else { + memset(buf, 0, csize); + } *offs += csize; buf += csize; @@ -950,20 +1470,16 @@ int osd_ldiskfs_read(struct inode *inode, void *buf, int size, loff_t *offs) } static ssize_t osd_read(const struct lu_env *env, struct dt_object *dt, - struct lu_buf *buf, loff_t *pos, - struct lustre_capa *capa) + struct lu_buf *buf, loff_t *pos) { struct inode *inode = osd_dt_obj(dt)->oo_inode; int rc; - if (osd_object_auth(env, dt, capa, CAPA_OPC_BODY_READ)) - RETURN(-EACCES); - /* Read small symlink from inode body as we need to maintain correct * on-disk symlinks for ldiskfs. */ if (S_ISLNK(dt->do_lu.lo_header->loh_attr) && - (buf->lb_len <= sizeof(LDISKFS_I(inode)->i_data))) + (buf->lb_len < sizeof(LDISKFS_I(inode)->i_data))) rc = osd_ldiskfs_readlink(inode, buf->lb_buf, buf->lb_len); else rc = osd_ldiskfs_read(inode, buf->lb_buf, buf->lb_len, pos); @@ -971,59 +1487,168 @@ static ssize_t osd_read(const struct lu_env *env, struct dt_object *dt, return rc; } +static inline int osd_extents_enabled(struct super_block *sb, + struct inode *inode) +{ + if (inode != NULL) { + if (LDISKFS_I(inode)->i_flags & LDISKFS_EXTENTS_FL) + return 1; + } else if (LDISKFS_HAS_INCOMPAT_FEATURE(sb, + LDISKFS_FEATURE_INCOMPAT_EXTENTS)) { + return 1; + } + return 0; +} + +int osd_calc_bkmap_credits(struct super_block *sb, struct inode *inode, + const loff_t size, const loff_t pos, + const int blocks) +{ + int credits, bits, bs, i; + + bits = sb->s_blocksize_bits; + bs = 1 << bits; + + /* legacy blockmap: 3 levels * 3 (bitmap,gd,itself) + * we do not expect blockmaps on the large files, + * so let's shrink it to 2 levels (4GB files) */ + + /* this is default reservation: 2 levels */ + credits = (blocks + 2) * 3; + + /* actual offset is unknown, hard to optimize */ + if (pos == -1) + return credits; + + /* now check for few specific cases to optimize */ + if (pos + size <= LDISKFS_NDIR_BLOCKS * bs) { + /* no indirects */ + credits = blocks; + /* allocate if not allocated */ + if (inode == NULL) { + credits += blocks * 2; + return credits; + } + for (i = (pos >> bits); i < (pos >> bits) + blocks; i++) { + LASSERT(i < LDISKFS_NDIR_BLOCKS); + if (LDISKFS_I(inode)->i_data[i] == 0) + credits += 2; + } + } else if (pos + size <= (LDISKFS_NDIR_BLOCKS + 1024) * bs) { + /* single indirect */ + credits = blocks * 3; + if (inode == NULL || + LDISKFS_I(inode)->i_data[LDISKFS_IND_BLOCK] == 0) + credits += 3; + else + /* The indirect block may be modified. */ + credits += 1; + } + + return credits; +} + static ssize_t osd_declare_write(const struct lu_env *env, struct dt_object *dt, - const loff_t size, loff_t pos, - struct thandle *handle) + const struct lu_buf *buf, loff_t _pos, + struct thandle *handle) { - struct osd_thandle *oh; - int credits; - struct inode *inode; - int rc; + struct osd_object *obj = osd_dt_obj(dt); + struct inode *inode = obj->oo_inode; + struct super_block *sb = osd_sb(osd_obj2dev(obj)); + struct osd_thandle *oh; + int rc = 0, est = 0, credits, blocks, allocated = 0; + int bits, bs; + int depth, size; + loff_t pos; ENTRY; + LASSERT(buf != NULL); LASSERT(handle != NULL); oh = container_of0(handle, struct osd_thandle, ot_super); LASSERT(oh->ot_handle == NULL); - /* XXX: size == 0 or INT_MAX indicating a catalog header update or - * llog write, see comment in mdd_declare_llog_record(). - * - * This hack will be removed with llog over OSD landing - */ - if (size == DECLARE_LLOG_REWRITE) - credits = 2; - else if (size == DECLARE_LLOG_WRITE) - credits = 6; - else - credits = osd_dto_credits_noquota[DTO_WRITE_BLOCK]; + size = buf->lb_len; + bits = sb->s_blocksize_bits; + bs = 1 << bits; - OSD_DECLARE_OP(oh, write); - oh->ot_credits += credits; + if (_pos == -1) { + /* if this is an append, then we + * should expect cross-block record */ + pos = 0; + } else { + pos = _pos; + } - inode = osd_dt_obj(dt)->oo_inode; + /* blocks to modify */ + blocks = ((pos + size + bs - 1) >> bits) - (pos >> bits); + LASSERT(blocks > 0); + + if (inode != NULL && _pos != -1) { + /* object size in blocks */ + est = (i_size_read(inode) + bs - 1) >> bits; + allocated = inode->i_blocks >> (bits - 9); + if (pos + size <= i_size_read(inode) && est <= allocated) { + /* looks like an overwrite, no need to modify tree */ + credits = blocks; + /* no need to modify i_size */ + goto out; + } + } + + if (osd_extents_enabled(sb, inode)) { + /* + * many concurrent threads may grow tree by the time + * our transaction starts. so, consider 2 is a min depth + * for every level we may need to allocate a new block + * and take some entries from the old one. so, 3 blocks + * to allocate (bitmap, gd, itself) + old block - 4 per + * level. + */ + depth = inode != NULL ? ext_depth(inode) : 0; + depth = max(depth, 1) + 1; + credits = depth; + /* if not append, then split may need to modify + * existing blocks moving entries into the new ones */ + if (_pos == -1) + credits += depth; + /* blocks to store data: bitmap,gd,itself */ + credits += blocks * 3; + } else { + credits = osd_calc_bkmap_credits(sb, inode, size, _pos, blocks); + } + /* if inode is created as part of the transaction, + * then it's counted already by the creation method */ + if (inode != NULL) + credits++; - /* we may declare write to non-exist llog */ - if (inode == NULL) - RETURN(0); +out: + + osd_trans_declare_op(env, oh, OSD_OT_WRITE, credits); /* dt_declare_write() is usually called for system objects, such * as llog or last_rcvd files. We needn't enforce quota on those * objects, so always set the lqi_space as 0. */ - rc = osd_declare_inode_qid(env, inode->i_uid, inode->i_gid, 0, oh, - true, true, NULL, false); + if (inode != NULL) + rc = osd_declare_inode_qid(env, i_uid_read(inode), + i_gid_read(inode), 0, oh, obj, true, + NULL, false); RETURN(rc); } static int osd_ldiskfs_writelink(struct inode *inode, char *buffer, int buflen) { + /* LU-2634: clear the extent format for fast symlink */ + ldiskfs_clear_inode_flag(inode, LDISKFS_INODE_EXTENTS); - memcpy((char *)&LDISKFS_I(inode)->i_data, (char *)buffer, buflen); - LDISKFS_I(inode)->i_disksize = buflen; - i_size_write(inode, buflen); - inode->i_sb->s_op->dirty_inode(inode); + memcpy((char *)&LDISKFS_I(inode)->i_data, (char *)buffer, buflen); + spin_lock(&inode->i_lock); + LDISKFS_I(inode)->i_disksize = buflen; + i_size_write(inode, buflen); + spin_unlock(&inode->i_lock); + ll_dirty_inode(inode, I_DIRTY_DATASYNC); - return 0; + return 0; } int osd_ldiskfs_write_record(struct inode *inode, void *buf, int bufsize, @@ -1052,11 +1677,17 @@ int osd_ldiskfs_write_record(struct inode *inode, void *buf, int bufsize, if (bh != NULL) brelse(bh); - block = offset >> inode->i_blkbits; - boffs = offset & (blocksize - 1); - size = min(blocksize - boffs, bufsize); - bh = ldiskfs_bread(handle, inode, block, 1, &err); - if (!bh) { + block = offset >> inode->i_blkbits; + boffs = offset & (blocksize - 1); + size = min(blocksize - boffs, bufsize); + bh = __ldiskfs_bread(handle, inode, block, 1); + if (IS_ERR_OR_NULL(bh)) { + if (bh == NULL) { + err = -EIO; + } else { + err = PTR_ERR(bh); + bh = NULL; + } CERROR("%s: error reading offset %llu (block %lu): " "rc = %d\n", inode->i_sb->s_id, offset, block, err); @@ -1069,11 +1700,11 @@ int osd_ldiskfs_write_record(struct inode *inode, void *buf, int bufsize, err); break; } - LASSERTF(boffs + size <= bh->b_size, - "boffs %d size %d bh->b_size %lu", - boffs, size, (unsigned long)bh->b_size); + LASSERTF(boffs + size <= bh->b_size, + "boffs %d size %d bh->b_size %lu\n", + boffs, size, (unsigned long)bh->b_size); memcpy(bh->b_data + boffs, buf, size); - err = ldiskfs_journal_dirty_metadata(handle, bh); + err = ldiskfs_handle_dirty_metadata(handle, NULL, bh); if (err) break; @@ -1088,18 +1719,18 @@ int osd_ldiskfs_write_record(struct inode *inode, void *buf, int bufsize, if (write_NUL) --new_size; - /* correct in-core and on-disk sizes */ - if (new_size > i_size_read(inode)) { - cfs_spin_lock(&inode->i_lock); - if (new_size > i_size_read(inode)) - i_size_write(inode, new_size); - if (i_size_read(inode) > LDISKFS_I(inode)->i_disksize) { - LDISKFS_I(inode)->i_disksize = i_size_read(inode); - dirty_inode = 1; - } - cfs_spin_unlock(&inode->i_lock); - if (dirty_inode) - inode->i_sb->s_op->dirty_inode(inode); + /* correct in-core and on-disk sizes */ + if (new_size > i_size_read(inode)) { + spin_lock(&inode->i_lock); + if (new_size > i_size_read(inode)) + i_size_write(inode, new_size); + if (i_size_read(inode) > LDISKFS_I(inode)->i_disksize) { + LDISKFS_I(inode)->i_disksize = i_size_read(inode); + dirty_inode = 1; + } + spin_unlock(&inode->i_lock); + if (dirty_inode) + ll_dirty_inode(inode, I_DIRTY_DATASYNC); } if (err == 0) @@ -1108,9 +1739,8 @@ int osd_ldiskfs_write_record(struct inode *inode, void *buf, int bufsize, } static ssize_t osd_write(const struct lu_env *env, struct dt_object *dt, - const struct lu_buf *buf, loff_t *pos, - struct thandle *handle, struct lustre_capa *capa, - int ignore_quota) + const struct lu_buf *buf, loff_t *pos, + struct thandle *handle, int ignore_quota) { struct inode *inode = osd_dt_obj(dt)->oo_inode; struct osd_thandle *oh; @@ -1119,16 +1749,17 @@ static ssize_t osd_write(const struct lu_env *env, struct dt_object *dt, LASSERT(dt_object_exists(dt)); - if (osd_object_auth(env, dt, capa, CAPA_OPC_BODY_WRITE)) - return -EACCES; - LASSERT(handle != NULL); + LASSERT(inode != NULL); + ll_vfs_dq_init(inode); /* XXX: don't check: one declared chunk can be used many times */ - /* OSD_EXEC_OP(handle, write); */ + /* osd_trans_exec_op(env, handle, OSD_OT_WRITE); */ oh = container_of(handle, struct osd_thandle, ot_super); LASSERT(oh->ot_handle->h_transaction != NULL); + osd_trans_exec_op(env, handle, OSD_OT_WRITE); + /* Write small symlink to inode body as we need to maintain correct * on-disk symlinks for ldiskfs. * Note: the buf->lb_buf contains a NUL terminator while buf->lb_len @@ -1141,9 +1772,12 @@ static ssize_t osd_write(const struct lu_env *env, struct dt_object *dt, result = osd_ldiskfs_write_record(inode, buf->lb_buf, buf->lb_len, is_link, pos, oh->ot_handle); - if (result == 0) - result = buf->lb_len; - return result; + if (result == 0) + result = buf->lb_len; + + osd_trans_exec_check(env, handle, OSD_OT_WRITE); + + return result; } static int osd_declare_punch(const struct lu_env *env, struct dt_object *dt, @@ -1157,8 +1791,6 @@ static int osd_declare_punch(const struct lu_env *env, struct dt_object *dt, LASSERT(th); oh = container_of(th, struct osd_thandle, ot_super); - OSD_DECLARE_OP(oh, punch); - /* * we don't need to reserve credits for whole truncate * it's not possible as truncate may need to free too many @@ -1167,54 +1799,68 @@ static int osd_declare_punch(const struct lu_env *env, struct dt_object *dt, * orphan list. if needed truncate will extend or restart * transaction */ - oh->ot_credits += osd_dto_credits_noquota[DTO_ATTR_SET_BASE]; - oh->ot_credits += 3; + osd_trans_declare_op(env, oh, OSD_OT_PUNCH, + osd_dto_credits_noquota[DTO_ATTR_SET_BASE] + 3); inode = osd_dt_obj(dt)->oo_inode; LASSERT(inode); - rc = osd_declare_inode_qid(env, inode->i_uid, inode->i_gid, 0, oh, - true, true, NULL, false); + rc = osd_declare_inode_qid(env, i_uid_read(inode), i_gid_read(inode), + 0, oh, osd_dt_obj(dt), true, NULL, false); RETURN(rc); } static int osd_punch(const struct lu_env *env, struct dt_object *dt, - __u64 start, __u64 end, struct thandle *th, - struct lustre_capa *capa) + __u64 start, __u64 end, struct thandle *th) { - struct osd_thandle *oh; - struct osd_object *obj = osd_dt_obj(dt); - struct inode *inode = obj->oo_inode; - handle_t *h; - tid_t tid; - int rc, rc2 = 0; - ENTRY; + struct osd_thandle *oh; + struct osd_object *obj = osd_dt_obj(dt); + struct inode *inode = obj->oo_inode; + handle_t *h; + tid_t tid; + int rc = 0, rc2 = 0; + ENTRY; - LASSERT(end == OBD_OBJECT_EOF); - LASSERT(dt_object_exists(dt)); - LASSERT(osd_invariant(obj)); + LASSERT(end == OBD_OBJECT_EOF); + LASSERT(dt_object_exists(dt)); + LASSERT(osd_invariant(obj)); + LASSERT(inode != NULL); + ll_vfs_dq_init(inode); - LASSERT(th); - oh = container_of(th, struct osd_thandle, ot_super); - LASSERT(oh->ot_handle->h_transaction != NULL); + LASSERT(th); + oh = container_of(th, struct osd_thandle, ot_super); + LASSERT(oh->ot_handle->h_transaction != NULL); - OSD_EXEC_OP(th, punch); + osd_trans_exec_op(env, th, OSD_OT_PUNCH); - tid = oh->ot_handle->h_transaction->t_tid; + tid = oh->ot_handle->h_transaction->t_tid; - rc = vmtruncate(inode, start); + spin_lock(&inode->i_lock); + i_size_write(inode, start); + spin_unlock(&inode->i_lock); + ll_truncate_pagecache(inode, start); +#ifdef HAVE_INODEOPS_TRUNCATE + if (inode->i_op->truncate) { + inode->i_op->truncate(inode); + } else +#endif + ldiskfs_truncate(inode); - /* - * For a partial-page truncate, flush the page to disk immediately to - * avoid data corruption during direct disk write. b=17397 - */ - if (rc == 0 && (start & ~CFS_PAGE_MASK) != 0) + /* + * For a partial-page truncate, flush the page to disk immediately to + * avoid data corruption during direct disk write. b=17397 + */ + if ((start & ~PAGE_MASK) != 0) rc = filemap_fdatawrite_range(inode->i_mapping, start, start+1); h = journal_current_handle(); LASSERT(h != NULL); LASSERT(h == oh->ot_handle); + /* do not check credits with osd_trans_exec_check() as the truncate + * can restart the transaction internally and we restart the + * transaction in this case */ + if (tid != h->h_transaction->t_tid) { int credits = oh->ot_credits; /* @@ -1230,32 +1876,91 @@ static int osd_punch(const struct lu_env *env, struct dt_object *dt, RETURN(rc == 0 ? rc2 : rc); } +static int fiemap_check_ranges(struct inode *inode, + u64 start, u64 len, u64 *new_len) +{ + loff_t maxbytes; + + *new_len = len; + + if (len == 0) + return -EINVAL; + + if (ldiskfs_test_inode_flag(inode, LDISKFS_INODE_EXTENTS)) + maxbytes = inode->i_sb->s_maxbytes; + else + maxbytes = LDISKFS_SB(inode->i_sb)->s_bitmap_maxbytes; + + if (start > maxbytes) + return -EFBIG; + + /* + * Shrink request scope to what the fs can actually handle. + */ + if (len > maxbytes || (maxbytes - len) < start) + *new_len = maxbytes - start; + + return 0; +} + +/* So that the fiemap access checks can't overflow on 32 bit machines. */ +#define FIEMAP_MAX_EXTENTS (UINT_MAX / sizeof(struct fiemap_extent)) + static int osd_fiemap_get(const struct lu_env *env, struct dt_object *dt, - struct ll_user_fiemap *fm) + struct fiemap *fm) { - struct inode *inode = osd_dt_obj(dt)->oo_inode; - struct osd_thread_info *info = osd_oti_get(env); - struct dentry *dentry = &info->oti_obj_dentry; - struct file *file = &info->oti_file; - mm_segment_t saved_fs; - int rc; + struct fiemap_extent_info fieinfo = {0, }; + struct inode *inode = osd_dt_obj(dt)->oo_inode; + u64 len; + int rc; - LASSERT(inode); - dentry->d_inode = inode; - file->f_dentry = dentry; - file->f_mapping = inode->i_mapping; - file->f_op = inode->i_fop; - - saved_fs = get_fs(); - set_fs(get_ds()); - /* ldiskfs_ioctl does not have a inode argument */ - if (inode->i_fop->unlocked_ioctl) - rc = inode->i_fop->unlocked_ioctl(file, FSFILT_IOC_FIEMAP, - (long)fm); - else - rc = -ENOTTY; - set_fs(saved_fs); - return rc; + + LASSERT(inode); + if (inode->i_op->fiemap == NULL) + return -EOPNOTSUPP; + + if (fm->fm_extent_count > FIEMAP_MAX_EXTENTS) + return -EINVAL; + + rc = fiemap_check_ranges(inode, fm->fm_start, fm->fm_length, &len); + if (rc) + return rc; + + fieinfo.fi_flags = fm->fm_flags; + fieinfo.fi_extents_max = fm->fm_extent_count; + fieinfo.fi_extents_start = fm->fm_extents; + + if (fieinfo.fi_flags & FIEMAP_FLAG_SYNC) + filemap_write_and_wait(inode->i_mapping); + + rc = inode->i_op->fiemap(inode, &fieinfo, fm->fm_start, len); + fm->fm_flags = fieinfo.fi_flags; + fm->fm_mapped_extents = fieinfo.fi_extents_mapped; + + return rc; +} + +static int osd_ladvise(const struct lu_env *env, struct dt_object *dt, + __u64 start, __u64 end, enum lu_ladvise_type advice) +{ + int rc = 0; + struct inode *inode = osd_dt_obj(dt)->oo_inode; + ENTRY; + + switch (advice) { + case LU_LADVISE_DONTNEED: + if (end == 0) + break; + invalidate_mapping_pages(inode->i_mapping, + start >> PAGE_CACHE_SHIFT, + (end - 1) >> PAGE_CACHE_SHIFT); + break; + default: + rc = -ENOTSUPP; + break; + } + + RETURN(rc); } /* @@ -1267,17 +1972,17 @@ const struct dt_body_operations osd_body_ops_new = { }; const struct dt_body_operations osd_body_ops = { - .dbo_read = osd_read, - .dbo_declare_write = osd_declare_write, - .dbo_write = osd_write, - .dbo_bufs_get = osd_bufs_get, - .dbo_bufs_put = osd_bufs_put, - .dbo_write_prep = osd_write_prep, - .dbo_declare_write_commit = osd_declare_write_commit, - .dbo_write_commit = osd_write_commit, - .dbo_read_prep = osd_read_prep, - .do_declare_punch = osd_declare_punch, - .do_punch = osd_punch, - .dbo_fiemap_get = osd_fiemap_get, + .dbo_read = osd_read, + .dbo_declare_write = osd_declare_write, + .dbo_write = osd_write, + .dbo_bufs_get = osd_bufs_get, + .dbo_bufs_put = osd_bufs_put, + .dbo_write_prep = osd_write_prep, + .dbo_declare_write_commit = osd_declare_write_commit, + .dbo_write_commit = osd_write_commit, + .dbo_read_prep = osd_read_prep, + .dbo_declare_punch = osd_declare_punch, + .dbo_punch = osd_punch, + .dbo_fiemap_get = osd_fiemap_get, + .dbo_ladvise = osd_ladvise, }; -