X-Git-Url: https://git.whamcloud.com/?p=fs%2Flustre-release.git;a=blobdiff_plain;f=lustre%2Fosd-ldiskfs%2Fosd_io.c;h=b8182f02eaca31d597731f555a4eeeb599df0a48;hp=555b9fc8debb42e1f488301a64fa3d43b936a420;hb=66bb2d13f8c4eb9670c8c98655221593f8aaf77d;hpb=5ecb7fd0b0af45a94af94ddaaa4f4fe1e59162bf diff --git a/lustre/osd-ldiskfs/osd_io.c b/lustre/osd-ldiskfs/osd_io.c index 555b9fc..b8182f0 100644 --- a/lustre/osd-ldiskfs/osd_io.c +++ b/lustre/osd-ldiskfs/osd_io.c @@ -15,11 +15,7 @@ * * You should have received a copy of the GNU General Public License * version 2 along with this program; If not, see - * http://www.sun.com/software/products/lustre/docs/GPLv2.pdf - * - * Please contact Sun Microsystems, Inc., 4150 Network Circle, Santa Clara, - * CA 95054 USA or visit www.sun.com if you need additional information or - * have any questions. + * http://www.gnu.org/licenses/gpl-2.0.html * * GPL HEADER END */ @@ -27,7 +23,7 @@ * Copyright (c) 2010, Oracle and/or its affiliates. All rights reserved. * Use is subject to license terms. * - * Copyright (c) 2012, 2013, Intel Corporation. + * Copyright (c) 2012, 2016, Intel Corporation. */ /* * This file is part of Lustre, http://www.lustre.org/ @@ -42,8 +38,6 @@ * */ -/* LUSTRE_VERSION_CODE */ -#include /* prerequisite for linux/xattr.h */ #include /* prerequisite for linux/xattr.h */ @@ -82,7 +76,7 @@ static int __osd_init_iobuf(struct osd_device *d, struct osd_iobuf *iobuf, iobuf->dr_rw = rw; iobuf->dr_init_at = line; - blocks = pages * (PAGE_CACHE_SIZE >> osd_sb(d)->s_blocksize_bits); + blocks = pages * (PAGE_SIZE >> osd_sb(d)->s_blocksize_bits); if (iobuf->dr_bl_buf.lb_len >= blocks * sizeof(iobuf->dr_blocks[0])) { LASSERT(iobuf->dr_pg_buf.lb_len >= pages * sizeof(iobuf->dr_pages[0])); @@ -97,7 +91,7 @@ static int __osd_init_iobuf(struct osd_device *d, struct osd_iobuf *iobuf, CDEBUG(D_OTHER, "realloc %u for %u (%u) pages\n", (unsigned)(pages * sizeof(iobuf->dr_pages[0])), i, pages); pages = i; - blocks = pages * (PAGE_CACHE_SIZE >> osd_sb(d)->s_blocksize_bits); + blocks = pages * (PAGE_SIZE >> osd_sb(d)->s_blocksize_bits); iobuf->dr_max_pages = 0; CDEBUG(D_OTHER, "realloc %u for %u blocks\n", (unsigned)(blocks * sizeof(iobuf->dr_blocks[0])), blocks); @@ -141,15 +135,17 @@ void osd_fini_iobuf(struct osd_device *d, struct osd_iobuf *iobuf) } } -#ifndef REQ_WRITE /* pre-2.6.35 */ -#define __REQ_WRITE BIO_RW -#endif - +#ifdef HAVE_BIO_ENDIO_USES_ONE_ARG +static void dio_complete_routine(struct bio *bio) +{ + int error = bio->bi_error; +#else static void dio_complete_routine(struct bio *bio, int error) { - struct osd_iobuf *iobuf = bio->bi_private; - struct bio_vec *bvl; - int i; +#endif + struct osd_iobuf *iobuf = bio->bi_private; + int iter; + struct bio_vec *bvl; /* CAVEAT EMPTOR: possibly in IRQ context * DO NOT record procfs stats here!!! */ @@ -158,31 +154,47 @@ static void dio_complete_routine(struct bio *bio, int error) CERROR("***** bio->bi_private is NULL! This should never " "happen. Normally, I would crash here, but instead I " "will dump the bio contents to the console. Please " - "report this to , along " + "report this to , along " "with any interesting messages leading up to this point " "(like SCSI errors, perhaps). Because bi_private is " "NULL, I can't wake up the thread that initiated this " "IO - you will probably have to reboot this node.\n"); - CERROR("bi_next: %p, bi_flags: %lx, bi_rw: %lu, bi_vcnt: %d, " - "bi_idx: %d, bi->size: %d, bi_end_io: %p, bi_cnt: %d, " - "bi_private: %p\n", bio->bi_next, bio->bi_flags, - bio->bi_rw, bio->bi_vcnt, bio->bi_idx, bio->bi_size, - bio->bi_end_io, atomic_read(&bio->bi_cnt), - bio->bi_private); + CERROR("bi_next: %p, bi_flags: %lx, " +#ifdef HAVE_BI_RW + "bi_rw: %lu," +#else + "bi_opf: %u," +#endif + "bi_vcnt: %d, bi_idx: %d, bi->size: %d, bi_end_io: %p," + "bi_cnt: %d, bi_private: %p\n", bio->bi_next, + (unsigned long)bio->bi_flags, +#ifdef HAVE_BI_RW + bio->bi_rw, +#else + bio->bi_opf, +#endif + bio->bi_vcnt, bio_idx(bio), + bio_sectors(bio) << 9, bio->bi_end_io, +#ifdef HAVE_BI_CNT + atomic_read(&bio->bi_cnt), +#else + atomic_read(&bio->__bi_cnt), +#endif + bio->bi_private); return; } - /* the check is outside of the cycle for performance reason -bzzz */ - if (!test_bit(__REQ_WRITE, &bio->bi_rw)) { - bio_for_each_segment(bvl, bio, i) { - if (likely(error == 0)) - SetPageUptodate(bvl->bv_page); - LASSERT(PageLocked(bvl->bv_page)); - } + /* the check is outside of the cycle for performance reason -bzzz */ + if (!bio_data_dir(bio)) { + bio_for_each_segment_all(bvl, bio, iter) { + if (likely(error == 0)) + SetPageUptodate(bvl_to_page(bvl)); + LASSERT(PageLocked(bvl_to_page(bvl))); + } atomic_dec(&iobuf->dr_dev->od_r_in_flight); - } else { + } else { atomic_dec(&iobuf->dr_dev->od_w_in_flight); - } + } /* any real error is good enough -bzzz */ if (error != 0 && iobuf->dr_error == 0) @@ -236,49 +248,53 @@ static void record_start_io(struct osd_iobuf *iobuf, int size) static void osd_submit_bio(int rw, struct bio *bio) { LASSERTF(rw == 0 || rw == 1, "%x\n", rw); +#ifdef HAVE_SUBMIT_BIO_2ARGS if (rw == 0) submit_bio(READ, bio); else submit_bio(WRITE, bio); +#else + bio->bi_opf |= rw; + submit_bio(bio); +#endif } static int can_be_merged(struct bio *bio, sector_t sector) { - unsigned int size; - - if (!bio) - return 0; + if (bio == NULL) + return 0; - size = bio->bi_size >> 9; - return bio->bi_sector + size == sector ? 1 : 0; + return bio_end_sector(bio) == sector ? 1 : 0; } static int osd_do_bio(struct osd_device *osd, struct inode *inode, struct osd_iobuf *iobuf) { - int blocks_per_page = PAGE_CACHE_SIZE >> inode->i_blkbits; - struct page **pages = iobuf->dr_pages; - int npages = iobuf->dr_npages; - unsigned long *blocks = iobuf->dr_blocks; - int total_blocks = npages * blocks_per_page; - int sector_bits = inode->i_sb->s_blocksize_bits - 9; - unsigned int blocksize = inode->i_sb->s_blocksize; - struct bio *bio = NULL; - struct page *page; - unsigned int page_offset; - sector_t sector; - int nblocks; - int block_idx; - int page_idx; - int i; - int rc = 0; - ENTRY; + int blocks_per_page = PAGE_SIZE >> inode->i_blkbits; + struct page **pages = iobuf->dr_pages; + int npages = iobuf->dr_npages; + sector_t *blocks = iobuf->dr_blocks; + int total_blocks = npages * blocks_per_page; + int sector_bits = inode->i_sb->s_blocksize_bits - 9; + unsigned int blocksize = inode->i_sb->s_blocksize; + struct bio *bio = NULL; + struct page *page; + unsigned int page_offset; + sector_t sector; + int nblocks; + int block_idx; + int page_idx; + int i; + int rc = 0; + DECLARE_PLUG(plug); + ENTRY; LASSERT(iobuf->dr_npages == npages); osd_brw_stats_update(osd, iobuf); iobuf->dr_start_time = cfs_time_current(); + blk_start_plug(&plug); for (page_idx = 0, block_idx = 0; page_idx < npages; page_idx++, block_idx += blocks_per_page) { @@ -316,23 +332,23 @@ static int osd_do_bio(struct osd_device *osd, struct inode *inode, blocksize * nblocks, page_offset) != 0) continue; /* added this frag OK */ - if (bio != NULL) { - struct request_queue *q = - bdev_get_queue(bio->bi_bdev); - - /* Dang! I have to fragment this I/O */ - CDEBUG(D_INODE, "bio++ sz %d vcnt %d(%d) " - "sectors %d(%d) psg %d(%d) hsg %d(%d)\n", - bio->bi_size, - bio->bi_vcnt, bio->bi_max_vecs, - bio->bi_size >> 9, queue_max_sectors(q), + if (bio != NULL) { + struct request_queue *q = + bdev_get_queue(bio->bi_bdev); + unsigned int bi_size = bio_sectors(bio) << 9; + + /* Dang! I have to fragment this I/O */ + CDEBUG(D_INODE, "bio++ sz %d vcnt %d(%d) " + "sectors %d(%d) psg %d(%d) hsg %d(%d)\n", + bi_size, bio->bi_vcnt, bio->bi_max_vecs, + bio_sectors(bio), + queue_max_sectors(q), bio_phys_segments(q, bio), queue_max_phys_segments(q), 0, queue_max_hw_segments(q)); - - record_start_io(iobuf, bio->bi_size); - osd_submit_bio(iobuf->dr_rw, bio); - } + record_start_io(iobuf, bi_size); + osd_submit_bio(iobuf->dr_rw, bio); + } /* allocate new bio */ bio = bio_alloc(GFP_NOIO, min(BIO_MAX_PAGES, @@ -346,25 +362,31 @@ static int osd_do_bio(struct osd_device *osd, struct inode *inode, goto out; } - bio->bi_bdev = inode->i_sb->s_bdev; - bio->bi_sector = sector; + bio->bi_bdev = inode->i_sb->s_bdev; + bio_set_sector(bio, sector); +#ifdef HAVE_BI_RW bio->bi_rw = (iobuf->dr_rw == 0) ? READ : WRITE; - bio->bi_end_io = dio_complete_routine; - bio->bi_private = iobuf; +#else + bio->bi_opf = (iobuf->dr_rw == 0) ? READ : WRITE; +#endif + bio->bi_end_io = dio_complete_routine; + bio->bi_private = iobuf; - rc = bio_add_page(bio, page, - blocksize * nblocks, page_offset); - LASSERT(rc != 0); - } - } + rc = bio_add_page(bio, page, + blocksize * nblocks, page_offset); + LASSERT(rc != 0); + } + } - if (bio != NULL) { - record_start_io(iobuf, bio->bi_size); - osd_submit_bio(iobuf->dr_rw, bio); - rc = 0; - } + if (bio != NULL) { + record_start_io(iobuf, bio_sectors(bio) << 9); + osd_submit_bio(iobuf->dr_rw, bio); + rc = 0; + } out: + blk_finish_plug(&plug); + /* in order to achieve better IO throughput, we don't wait for writes * completion here. instead we proceed with transaction commit in * parallel and wait for IO completion once transaction is stopped @@ -388,18 +410,18 @@ static int osd_map_remote_to_local(loff_t offset, ssize_t len, int *nrpages, *nrpages = 0; while (len > 0) { - int poff = offset & (PAGE_CACHE_SIZE - 1); - int plen = PAGE_CACHE_SIZE - poff; + int poff = offset & (PAGE_SIZE - 1); + int plen = PAGE_SIZE - poff; if (plen > len) plen = len; lnb->lnb_file_offset = offset; lnb->lnb_page_offset = poff; - lnb->len = plen; - /* lb->flags = rnb->flags; */ - lnb->flags = 0; - lnb->page = NULL; - lnb->rc = 0; + lnb->lnb_len = plen; + /* lnb->lnb_flags = rnb->rnb_flags; */ + lnb->lnb_flags = 0; + lnb->lnb_page = NULL; + lnb->lnb_rc = 0; LASSERTF(plen <= len, "plen %u, len %lld\n", plen, (long long) len); @@ -412,16 +434,18 @@ static int osd_map_remote_to_local(loff_t offset, ssize_t len, int *nrpages, RETURN(0); } -struct page *osd_get_page(struct dt_object *dt, loff_t offset, int rw) +static struct page *osd_get_page(struct dt_object *dt, loff_t offset, + gfp_t gfp_mask) { - struct inode *inode = osd_dt_obj(dt)->oo_inode; - struct osd_device *d = osd_obj2dev(osd_dt_obj(dt)); - struct page *page; + struct inode *inode = osd_dt_obj(dt)->oo_inode; + struct osd_device *d = osd_obj2dev(osd_dt_obj(dt)); + struct page *page; LASSERT(inode); - page = find_or_create_page(inode->i_mapping, offset >> PAGE_CACHE_SHIFT, - GFP_NOFS | __GFP_HIGHMEM); + page = find_or_create_page(inode->i_mapping, offset >> PAGE_SHIFT, + gfp_mask); + if (unlikely(page == NULL)) lprocfs_counter_add(d->od_stats, LPROC_OSD_NO_PAGE, 1); @@ -433,77 +457,117 @@ struct page *osd_get_page(struct dt_object *dt, loff_t offset, int rw) * journal_start * i_mutex * page lock - - * osd write path - * lock page(s) - * journal_start - * truncate_sem - + * + * osd write path: + * - lock page(s) + * - journal_start + * - truncate_sem + * * ext4 vmtruncate: - * lock pages, unlock - * journal_start - * lock partial page - * i_data_sem - -*/ -int osd_bufs_get(const struct lu_env *env, struct dt_object *d, loff_t pos, - ssize_t len, struct niobuf_local *lnb, int rw, - struct lustre_capa *capa) + * - lock pages, unlock + * - journal_start + * - lock partial page + * - i_data_sem + * + */ + +/** + * Unlock and release pages loaded by osd_bufs_get() + * + * Unlock \a npages pages from \a lnb and drop the refcount on them. + * + * \param env thread execution environment + * \param dt dt object undergoing IO (OSD object + methods) + * \param lnb array of pages undergoing IO + * \param npages number of pages in \a lnb + * + * \retval 0 always + */ +static int osd_bufs_put(const struct lu_env *env, struct dt_object *dt, + struct niobuf_local *lnb, int npages) { - struct osd_object *obj = osd_dt_obj(d); - int npages, i, rc = 0; - - LASSERT(obj->oo_inode); - - osd_map_remote_to_local(pos, len, &npages, lnb); - - for (i = 0; i < npages; i++, lnb++) { - lnb->page = osd_get_page(d, lnb->lnb_file_offset, rw); - if (lnb->page == NULL) - GOTO(cleanup, rc = -ENOMEM); - - /* DLM locking protects us from write and truncate competing - * for same region, but truncate can leave dirty page in the - * cache. it's possible the writeout on a such a page is in - * progress when we access it. it's also possible that during - * this writeout we put new (partial) data, but then won't - * be able to proceed in filter_commitrw_write(). thus let's - * just wait for writeout completion, should be rare enough. - * -bzzz */ - wait_on_page_writeback(lnb->page); - BUG_ON(PageWriteback(lnb->page)); - - lu_object_get(&d->do_lu); - } - rc = i; + int i; -cleanup: - RETURN(rc); + for (i = 0; i < npages; i++) { + if (lnb[i].lnb_page == NULL) + continue; + LASSERT(PageLocked(lnb[i].lnb_page)); + unlock_page(lnb[i].lnb_page); + put_page(lnb[i].lnb_page); + dt_object_put(env, dt); + lnb[i].lnb_page = NULL; + } + + RETURN(0); } -static int osd_bufs_put(const struct lu_env *env, struct dt_object *dt, - struct niobuf_local *lnb, int npages) +/** + * Load and lock pages undergoing IO + * + * Pages as described in the \a lnb array are fetched (from disk or cache) + * and locked for IO by the caller. + * + * DLM locking protects us from write and truncate competing for same region, + * but partial-page truncate can leave dirty pages in the cache for ldiskfs. + * It's possible the writeout on a such a page is in progress when we access + * it. It's also possible that during this writeout we put new (partial) data + * into the page, but won't be able to proceed in filter_commitrw_write(). + * Therefore, just wait for writeout completion as it should be rare enough. + * + * \param env thread execution environment + * \param dt dt object undergoing IO (OSD object + methods) + * \param pos byte offset of IO start + * \param len number of bytes of IO + * \param lnb array of extents undergoing IO + * \param rw read or write operation, and other flags + * \param capa capabilities + * + * \retval pages (zero or more) loaded successfully + * \retval -ENOMEM on memory/page allocation error + */ +static int osd_bufs_get(const struct lu_env *env, struct dt_object *dt, + loff_t pos, ssize_t len, struct niobuf_local *lnb, + enum dt_bufs_type rw) { - int i; + struct osd_object *obj = osd_dt_obj(dt); + int npages, i, rc = 0; + gfp_t gfp_mask; - for (i = 0; i < npages; i++) { - if (lnb[i].page == NULL) - continue; - LASSERT(PageLocked(lnb[i].page)); - unlock_page(lnb[i].page); - page_cache_release(lnb[i].page); - lu_object_put(env, &dt->do_lu); - lnb[i].page = NULL; - } - RETURN(0); + LASSERT(obj->oo_inode); + + osd_map_remote_to_local(pos, len, &npages, lnb); + + /* this could also try less hard for DT_BUFS_TYPE_READAHEAD pages */ + gfp_mask = rw & DT_BUFS_TYPE_LOCAL ? (GFP_NOFS | __GFP_HIGHMEM) : + GFP_HIGHUSER; + for (i = 0; i < npages; i++, lnb++) { + lnb->lnb_page = osd_get_page(dt, lnb->lnb_file_offset, + gfp_mask); + if (lnb->lnb_page == NULL) + GOTO(cleanup, rc = -ENOMEM); + + wait_on_page_writeback(lnb->lnb_page); + BUG_ON(PageWriteback(lnb->lnb_page)); + + lu_object_get(&dt->do_lu); + } + + RETURN(i); + +cleanup: + if (i > 0) + osd_bufs_put(env, dt, lnb - i, i); + return rc; } +#ifndef HAVE_LDISKFS_MAP_BLOCKS + #ifdef HAVE_EXT_PBLOCK /* Name changed to ext4_ext_pblock for kernel 2.6.35 */ #define ldiskfs_ext_pblock(ex) ext_pblock((ex)) #endif struct bpointers { - unsigned long *blocks; + sector_t *blocks; unsigned long start; int num; int init_num; @@ -582,7 +646,7 @@ static int ldiskfs_ext_new_extent_cb(struct inode *inode, { struct bpointers *bp = cbdata; struct ldiskfs_extent nex; - unsigned long pblock; + unsigned long pblock = 0; unsigned long tgen; int err, i; unsigned long count; @@ -617,7 +681,8 @@ static int ldiskfs_ext_new_extent_cb(struct inode *inode, tgen = LDISKFS_I(inode)->i_ext_generation; count = ldiskfs_ext_calc_credits_for_insert(inode, path); - handle = ldiskfs_journal_start(inode, count + LDISKFS_ALLOC_NEEDED + 1); + handle = osd_journal_start(inode, LDISKFS_HT_MISC, + count + LDISKFS_ALLOC_NEEDED + 1); if (IS_ERR(handle)) { return PTR_ERR(handle); } @@ -707,15 +772,16 @@ map: i, cex->ec_len); for (; i < cex->ec_len && bp->num; i++) { *(bp->blocks) = cex->ec_start + i; -#ifdef LDISKFS_EXT_CACHE_EXTENT /* until kernel 2.6.37 */ - if (cex->ec_type != LDISKFS_EXT_CACHE_EXTENT) { -#else - if ((cex->ec_len == 0) || (cex->ec_start == 0)) { -#endif + if (pblock != 0) { /* unmap any possible underlying metadata from * the block device mapping. bug 6998. */ +#ifndef HAVE_CLEAN_BDEV_ALIASES unmap_underlying_metadata(inode->i_sb->s_bdev, *(bp->blocks)); +#else + clean_bdev_aliases(inode->i_sb->s_bdev, + *(bp->blocks), 1); +#endif } bp->blocks++; bp->num--; @@ -725,36 +791,66 @@ map: return err; } -int osd_ldiskfs_map_nblocks(struct inode *inode, unsigned long block, - unsigned long num, unsigned long *blocks, - int create) +static int osd_ldiskfs_map_nblocks(struct inode *inode, unsigned long index, + int clen, sector_t *blocks, int create) { + int blocks_per_page = PAGE_SIZE >> inode->i_blkbits; struct bpointers bp; int err; - CDEBUG(D_OTHER, "blocks %lu-%lu requested for inode %u\n", - block, block + num - 1, (unsigned) inode->i_ino); + if (index + clen >= inode->i_sb->s_maxbytes >> PAGE_SHIFT) + return -EFBIG; bp.blocks = blocks; - bp.start = block; - bp.init_num = bp.num = num; + bp.start = index * blocks_per_page; + bp.init_num = bp.num = clen * blocks_per_page; bp.create = create; - err = ldiskfs_ext_walk_space(inode, block, num, - ldiskfs_ext_new_extent_cb, &bp); + CDEBUG(D_OTHER, "blocks %lu-%lu requested for inode %u\n", + bp.start, bp.start + bp.num - 1, (unsigned)inode->i_ino); + + err = ldiskfs_ext_walk_space(inode, bp.start, bp.num, + ldiskfs_ext_new_extent_cb, &bp); ldiskfs_ext_invalidate_cache(inode); return err; } -int osd_ldiskfs_map_ext_inode_pages(struct inode *inode, struct page **page, - int pages, unsigned long *blocks, - int create) +static int osd_ldiskfs_map_bm_inode_pages(struct inode *inode, + struct page **page, int pages, + sector_t *blocks, int create) { - int blocks_per_page = PAGE_CACHE_SIZE >> inode->i_blkbits; - int rc = 0, i = 0; + int blocks_per_page = PAGE_SIZE >> inode->i_blkbits; + pgoff_t bitmap_max_page_index; + sector_t *b; + int rc = 0, i; + + bitmap_max_page_index = LDISKFS_SB(inode->i_sb)->s_bitmap_maxbytes >> + PAGE_SHIFT; + for (i = 0, b = blocks; i < pages; i++, page++) { + if ((*page)->index + 1 >= bitmap_max_page_index) { + rc = -EFBIG; + break; + } + rc = ldiskfs_map_inode_page(inode, *page, b, create); + if (rc) { + CERROR("ino %lu, blk %llu create %d: rc %d\n", + inode->i_ino, + (unsigned long long)*b, create, rc); + break; + } + b += blocks_per_page; + } + return rc; +} + +static int osd_ldiskfs_map_ext_inode_pages(struct inode *inode, + struct page **page, + int pages, sector_t *blocks, + int create) +{ + int rc = 0, i = 0, clen = 0; struct page *fp = NULL; - int clen = 0; CDEBUG(D_OTHER, "inode %lu: map %d pages from %lu\n", inode->i_ino, pages, (*page)->index); @@ -777,49 +873,27 @@ int osd_ldiskfs_map_ext_inode_pages(struct inode *inode, struct page **page, } /* process found extent */ - rc = osd_ldiskfs_map_nblocks(inode, fp->index * blocks_per_page, - clen * blocks_per_page, blocks, - create); + rc = osd_ldiskfs_map_nblocks(inode, fp->index, clen, + blocks, create); if (rc) GOTO(cleanup, rc); /* look for next extent */ fp = NULL; - blocks += blocks_per_page * clen; + blocks += clen * (PAGE_SIZE >> inode->i_blkbits); } if (fp) - rc = osd_ldiskfs_map_nblocks(inode, fp->index * blocks_per_page, - clen * blocks_per_page, blocks, - create); -cleanup: - return rc; -} - -int osd_ldiskfs_map_bm_inode_pages(struct inode *inode, struct page **page, - int pages, unsigned long *blocks, - int create) -{ - int blocks_per_page = PAGE_CACHE_SIZE >> inode->i_blkbits; - unsigned long *b; - int rc = 0, i; + rc = osd_ldiskfs_map_nblocks(inode, fp->index, clen, + blocks, create); - for (i = 0, b = blocks; i < pages; i++, page++) { - rc = ldiskfs_map_inode_page(inode, *page, b, create); - if (rc) { - CERROR("ino %lu, blk %lu create %d: rc %d\n", - inode->i_ino, *b, create, rc); - break; - } - - b += blocks_per_page; - } +cleanup: return rc; } static int osd_ldiskfs_map_inode_pages(struct inode *inode, struct page **page, - int pages, unsigned long *blocks, - int create, struct mutex *optional_mutex) + int pages, sector_t *blocks, + int create) { int rc; @@ -828,14 +902,103 @@ static int osd_ldiskfs_map_inode_pages(struct inode *inode, struct page **page, blocks, create); return rc; } - if (optional_mutex != NULL) - mutex_lock(optional_mutex); rc = osd_ldiskfs_map_bm_inode_pages(inode, page, pages, blocks, create); - if (optional_mutex != NULL) - mutex_unlock(optional_mutex); return rc; } +#else +static int osd_ldiskfs_map_inode_pages(struct inode *inode, struct page **page, + int pages, sector_t *blocks, + int create) +{ + int blocks_per_page = PAGE_SIZE >> inode->i_blkbits; + int rc = 0, i = 0; + struct page *fp = NULL; + int clen = 0; + pgoff_t max_page_index; + handle_t *handle = NULL; + + max_page_index = inode->i_sb->s_maxbytes >> PAGE_SHIFT; + + CDEBUG(D_OTHER, "inode %lu: map %d pages from %lu\n", + inode->i_ino, pages, (*page)->index); + + if (create) { + create = LDISKFS_GET_BLOCKS_CREATE; + handle = ldiskfs_journal_current_handle(); + LASSERT(handle != NULL); + rc = osd_attach_jinode(inode); + if (rc) + return rc; + } + /* pages are sorted already. so, we just have to find + * contig. space and process them properly */ + while (i < pages) { + long blen, total = 0; + struct ldiskfs_map_blocks map = { 0 }; + + if (fp == NULL) { /* start new extent */ + fp = *page++; + clen = 1; + if (++i != pages) + continue; + } else if (fp->index + clen == (*page)->index) { + /* continue the extent */ + page++; + clen++; + if (++i != pages) + continue; + } + if (fp->index + clen >= max_page_index) + GOTO(cleanup, rc = -EFBIG); + /* process found extent */ + map.m_lblk = fp->index * blocks_per_page; + map.m_len = blen = clen * blocks_per_page; +cont_map: + rc = ldiskfs_map_blocks(handle, inode, &map, create); + if (rc >= 0) { + int c = 0; + for (; total < blen && c < map.m_len; c++, total++) { + if (rc == 0) { + *(blocks + total) = 0; + total++; + break; + } else { + *(blocks + total) = map.m_pblk + c; + /* unmap any possible underlying + * metadata from the block device + * mapping. bug 6998. */ + if ((map.m_flags & LDISKFS_MAP_NEW) && + create) +#ifndef HAVE_CLEAN_BDEV_ALIASES + unmap_underlying_metadata( + inode->i_sb->s_bdev, + map.m_pblk + c); +#else + clean_bdev_aliases( + inode->i_sb->s_bdev, + map.m_pblk + c, 1); +#endif + } + } + rc = 0; + } + if (rc == 0 && total < blen) { + map.m_lblk = fp->index * blocks_per_page + total; + map.m_len = blen - total; + goto cont_map; + } + if (rc != 0) + GOTO(cleanup, rc); + + /* look for next extent */ + fp = NULL; + blocks += blocks_per_page * clen; + } +cleanup: + return rc; +} +#endif /* HAVE_LDISKFS_MAP_BLOCKS */ static int osd_write_prep(const struct lu_env *env, struct dt_object *dt, struct niobuf_local *lnb, int npages) @@ -844,9 +1007,9 @@ static int osd_write_prep(const struct lu_env *env, struct dt_object *dt, struct osd_iobuf *iobuf = &oti->oti_iobuf; struct inode *inode = osd_dt_obj(dt)->oo_inode; struct osd_device *osd = osd_obj2dev(osd_dt_obj(dt)); - struct timeval start; - struct timeval end; - unsigned long timediff; + ktime_t start; + ktime_t end; + s64 timediff; ssize_t isize; __s64 maxidx; int rc = 0; @@ -860,55 +1023,54 @@ static int osd_write_prep(const struct lu_env *env, struct dt_object *dt, RETURN(rc); isize = i_size_read(inode); - maxidx = ((isize + PAGE_CACHE_SIZE - 1) >> PAGE_CACHE_SHIFT) - 1; + maxidx = ((isize + PAGE_SIZE - 1) >> PAGE_SHIFT) - 1; if (osd->od_writethrough_cache) cache = 1; if (isize > osd->od_readcache_max_filesize) cache = 0; - do_gettimeofday(&start); + start = ktime_get(); for (i = 0; i < npages; i++) { - if (cache == 0) - generic_error_remove_page(inode->i_mapping, - lnb[i].page); + if (cache == 0) + generic_error_remove_page(inode->i_mapping, + lnb[i].lnb_page); - /* - * till commit the content of the page is undefined - * we'll set it uptodate once bulk is done. otherwise - * subsequent reads can access non-stable data - */ - ClearPageUptodate(lnb[i].page); + /* + * till commit the content of the page is undefined + * we'll set it uptodate once bulk is done. otherwise + * subsequent reads can access non-stable data + */ + ClearPageUptodate(lnb[i].lnb_page); - if (lnb[i].len == PAGE_CACHE_SIZE) - continue; + if (lnb[i].lnb_len == PAGE_SIZE) + continue; - if (maxidx >= lnb[i].page->index) { - osd_iobuf_add_page(iobuf, lnb[i].page); - } else { - long off; - char *p = kmap(lnb[i].page); + if (maxidx >= lnb[i].lnb_page->index) { + osd_iobuf_add_page(iobuf, lnb[i].lnb_page); + } else { + long off; + char *p = kmap(lnb[i].lnb_page); off = lnb[i].lnb_page_offset; if (off) memset(p, 0, off); - off = (lnb[i].lnb_page_offset + lnb[i].len) & - ~CFS_PAGE_MASK; - if (off) - memset(p + off, 0, PAGE_CACHE_SIZE - off); - kunmap(lnb[i].page); + off = (lnb[i].lnb_page_offset + lnb[i].lnb_len) & + ~PAGE_MASK; + if (off) + memset(p + off, 0, PAGE_SIZE - off); + kunmap(lnb[i].lnb_page); } } - do_gettimeofday(&end); - timediff = cfs_timeval_sub(&end, &start, NULL); + end = ktime_get(); + timediff = ktime_us_delta(end, start); lprocfs_counter_add(osd->od_stats, LPROC_OSD_GET_PAGE, timediff); if (iobuf->dr_npages) { rc = osd_ldiskfs_map_inode_pages(inode, iobuf->dr_pages, iobuf->dr_npages, - iobuf->dr_blocks, - 0, NULL); + iobuf->dr_blocks, 0); if (likely(rc == 0)) { rc = osd_do_bio(osd, inode, iobuf); /* do IO stats for preparation reads */ @@ -918,29 +1080,57 @@ static int osd_write_prep(const struct lu_env *env, struct dt_object *dt, RETURN(rc); } -/* Check if a block is allocated or not */ -static int osd_is_mapped(struct inode *inode, obd_size offset) -{ - sector_t (*fs_bmap)(struct address_space *, sector_t); +struct osd_fextent { + sector_t start; + sector_t end; + unsigned int mapped:1; +}; - fs_bmap = inode->i_mapping->a_ops->bmap; +static int osd_is_mapped(struct dt_object *dt, __u64 offset, + struct osd_fextent *cached_extent) +{ + struct inode *inode = osd_dt_obj(dt)->oo_inode; + sector_t block = offset >> inode->i_blkbits; + sector_t start; + struct fiemap_extent_info fei = { 0 }; + struct fiemap_extent fe = { 0 }; + mm_segment_t saved_fs; + int rc; - /* We can't know if we are overwriting or not */ - if (unlikely(fs_bmap == NULL)) - return 0; + if (block >= cached_extent->start && block < cached_extent->end) + return cached_extent->mapped; if (i_size_read(inode) == 0) return 0; /* Beyond EOF, must not be mapped */ - if (((i_size_read(inode) - 1) >> inode->i_blkbits) < - (offset >> inode->i_blkbits)) + if (((i_size_read(inode) - 1) >> inode->i_blkbits) < block) return 0; - if (fs_bmap(inode->i_mapping, offset >> inode->i_blkbits) == 0) + fei.fi_extents_max = 1; + fei.fi_extents_start = &fe; + + saved_fs = get_fs(); + set_fs(get_ds()); + rc = inode->i_op->fiemap(inode, &fei, offset, FIEMAP_MAX_OFFSET-offset); + set_fs(saved_fs); + if (rc != 0) return 0; - return 1; + start = fe.fe_logical >> inode->i_blkbits; + + if (start > block) { + cached_extent->start = block; + cached_extent->end = start; + cached_extent->mapped = 0; + } else { + cached_extent->start = start; + cached_extent->end = (fe.fe_logical + fe.fe_length) >> + inode->i_blkbits; + cached_extent->mapped = 1; + } + + return cached_extent->mapped; } static int osd_declare_write_commit(const struct lu_env *env, @@ -948,17 +1138,19 @@ static int osd_declare_write_commit(const struct lu_env *env, struct niobuf_local *lnb, int npages, struct thandle *handle) { - const struct osd_device *osd = osd_obj2dev(osd_dt_obj(dt)); - struct inode *inode = osd_dt_obj(dt)->oo_inode; - struct osd_thandle *oh; - int extents = 1; - int depth; - int i; - int newblocks; - int rc = 0; - int flags = 0; - bool ignore_quota = false; - long long quota_space = 0; + const struct osd_device *osd = osd_obj2dev(osd_dt_obj(dt)); + struct inode *inode = osd_dt_obj(dt)->oo_inode; + struct osd_thandle *oh; + int extents = 1; + int depth; + int i; + int newblocks; + int rc = 0; + int flags = 0; + int credits = 0; + long long quota_space = 0; + struct osd_fextent extent = { 0 }; + enum osd_qid_declare_flags declare_flags = OSD_QID_BLK; ENTRY; LASSERT(handle != NULL); @@ -970,11 +1162,11 @@ static int osd_declare_write_commit(const struct lu_env *env, /* calculate number of extents (probably better to pass nb) */ for (i = 0; i < npages; i++) { if (i && lnb[i].lnb_file_offset != - lnb[i - 1].lnb_file_offset + lnb[i - 1].len) + lnb[i - 1].lnb_file_offset + lnb[i - 1].lnb_len) extents++; - if (!osd_is_mapped(inode, lnb[i].lnb_file_offset)) - quota_space += PAGE_CACHE_SIZE; + if (!osd_is_mapped(dt, lnb[i].lnb_file_offset, &extent)) + quota_space += PAGE_SIZE; /* ignore quota for the whole request if any page is from * client cache or written by root. @@ -985,10 +1177,10 @@ static int osd_declare_write_commit(const struct lu_env *env, * * XXX we could handle this on per-lnb basis as done by * grant. */ - if ((lnb[i].flags & OBD_BRW_NOQUOTA) || - (lnb[i].flags & (OBD_BRW_FROM_GRANT | OBD_BRW_SYNC)) == + if ((lnb[i].lnb_flags & OBD_BRW_NOQUOTA) || + (lnb[i].lnb_flags & (OBD_BRW_FROM_GRANT | OBD_BRW_SYNC)) == OBD_BRW_FROM_GRANT) - ignore_quota = true; + declare_flags |= OSD_QID_FORCE; } /* @@ -1004,14 +1196,14 @@ static int osd_declare_write_commit(const struct lu_env *env, depth = ext_depth(inode); depth = max(depth, 1) + 1; newblocks += depth; - oh->ot_credits++; /* inode */ - oh->ot_credits += depth * 2 * extents; - } else { - depth = 3; - newblocks += depth; - oh->ot_credits++; /* inode */ - oh->ot_credits += depth * extents; - } + credits++; /* inode */ + credits += depth * 2 * extents; + } else { + depth = 3; + newblocks += depth; + credits++; /* inode */ + credits += depth * extents; + } /* quota space for metadata blocks */ quota_space += depth * extents * LDISKFS_BLOCK_SIZE(osd_sb(osd)); @@ -1023,30 +1215,34 @@ static int osd_declare_write_commit(const struct lu_env *env, /* we can't dirty more bitmap blocks than exist */ if (newblocks > LDISKFS_SB(osd_sb(osd))->s_groups_count) - oh->ot_credits += LDISKFS_SB(osd_sb(osd))->s_groups_count; + credits += LDISKFS_SB(osd_sb(osd))->s_groups_count; else - oh->ot_credits += newblocks; + credits += newblocks; - /* we can't dirty more gd blocks than exist */ - if (newblocks > LDISKFS_SB(osd_sb(osd))->s_gdb_count) - oh->ot_credits += LDISKFS_SB(osd_sb(osd))->s_gdb_count; - else - oh->ot_credits += newblocks; + /* we can't dirty more gd blocks than exist */ + if (newblocks > LDISKFS_SB(osd_sb(osd))->s_gdb_count) + credits += LDISKFS_SB(osd_sb(osd))->s_gdb_count; + else + credits += newblocks; + + osd_trans_declare_op(env, oh, OSD_OT_WRITE, credits); /* make sure the over quota flags were not set */ - lnb[0].flags &= ~(OBD_BRW_OVER_USRQUOTA | OBD_BRW_OVER_GRPQUOTA); + lnb[0].lnb_flags &= ~OBD_BRW_OVER_ALLQUOTA; rc = osd_declare_inode_qid(env, i_uid_read(inode), i_gid_read(inode), - quota_space, oh, true, true, &flags, - ignore_quota); + i_projid_read(inode), quota_space, oh, + osd_dt_obj(dt), &flags, declare_flags); /* we need only to store the overquota flags in the first lnb for * now, once we support multiple objects BRW, this code needs be * revised. */ if (flags & QUOTA_FL_OVER_USRQUOTA) - lnb[0].flags |= OBD_BRW_OVER_USRQUOTA; + lnb[0].lnb_flags |= OBD_BRW_OVER_USRQUOTA; if (flags & QUOTA_FL_OVER_GRPQUOTA) - lnb[0].flags |= OBD_BRW_OVER_GRPQUOTA; + lnb[0].lnb_flags |= OBD_BRW_OVER_GRPQUOTA; + if (flags & QUOTA_FL_OVER_PRJQUOTA) + lnb[0].lnb_flags |= OBD_BRW_OVER_PRJQUOTA; RETURN(rc); } @@ -1062,6 +1258,7 @@ static int osd_write_commit(const struct lu_env *env, struct dt_object *dt, struct osd_device *osd = osd_obj2dev(osd_dt_obj(dt)); loff_t isize; int rc = 0, i; + struct osd_fextent extent = { 0 }; LASSERT(inode); @@ -1073,76 +1270,85 @@ static int osd_write_commit(const struct lu_env *env, struct dt_object *dt, ll_vfs_dq_init(inode); for (i = 0; i < npages; i++) { - if (lnb[i].rc == -ENOSPC && - osd_is_mapped(inode, lnb[i].lnb_file_offset)) { - /* Allow the write to proceed if overwriting an - * existing block */ - lnb[i].rc = 0; - } + if (lnb[i].lnb_rc == -ENOSPC && + osd_is_mapped(dt, lnb[i].lnb_file_offset, &extent)) { + /* Allow the write to proceed if overwriting an + * existing block */ + lnb[i].lnb_rc = 0; + } - if (lnb[i].rc) { /* ENOSPC, network RPC error, etc. */ - CDEBUG(D_INODE, "Skipping [%d] == %d\n", i, - lnb[i].rc); - LASSERT(lnb[i].page); - generic_error_remove_page(inode->i_mapping,lnb[i].page); - continue; - } + if (lnb[i].lnb_rc) { /* ENOSPC, network RPC error, etc. */ + CDEBUG(D_INODE, "Skipping [%d] == %d\n", i, + lnb[i].lnb_rc); + LASSERT(lnb[i].lnb_page); + generic_error_remove_page(inode->i_mapping, + lnb[i].lnb_page); + continue; + } - LASSERT(PageLocked(lnb[i].page)); - LASSERT(!PageWriteback(lnb[i].page)); + LASSERT(PageLocked(lnb[i].lnb_page)); + LASSERT(!PageWriteback(lnb[i].lnb_page)); - if (lnb[i].lnb_file_offset + lnb[i].len > isize) - isize = lnb[i].lnb_file_offset + lnb[i].len; + if (lnb[i].lnb_file_offset + lnb[i].lnb_len > isize) + isize = lnb[i].lnb_file_offset + lnb[i].lnb_len; - /* - * Since write and truncate are serialized by oo_sem, even - * partial-page truncate should not leave dirty pages in the - * page cache. - */ - LASSERT(!PageDirty(lnb[i].page)); + /* + * Since write and truncate are serialized by oo_sem, even + * partial-page truncate should not leave dirty pages in the + * page cache. + */ + LASSERT(!PageDirty(lnb[i].lnb_page)); - SetPageUptodate(lnb[i].page); + SetPageUptodate(lnb[i].lnb_page); - osd_iobuf_add_page(iobuf, lnb[i].page); + osd_iobuf_add_page(iobuf, lnb[i].lnb_page); } + osd_trans_exec_op(env, thandle, OSD_OT_WRITE); + if (OBD_FAIL_CHECK(OBD_FAIL_OST_MAPBLK_ENOSPC)) { rc = -ENOSPC; } else if (iobuf->dr_npages > 0) { rc = osd_ldiskfs_map_inode_pages(inode, iobuf->dr_pages, iobuf->dr_npages, - iobuf->dr_blocks, - 1, NULL); + iobuf->dr_blocks, 1); } else { /* no pages to write, no transno is needed */ thandle->th_local = 1; } - if (likely(rc == 0)) { - if (isize > i_size_read(inode)) { - i_size_write(inode, isize); - LDISKFS_I(inode)->i_disksize = isize; + if (likely(rc == 0)) { + spin_lock(&inode->i_lock); + if (isize > i_size_read(inode)) { + i_size_write(inode, isize); + LDISKFS_I(inode)->i_disksize = isize; + spin_unlock(&inode->i_lock); ll_dirty_inode(inode, I_DIRTY_DATASYNC); - } + } else { + spin_unlock(&inode->i_lock); + } - rc = osd_do_bio(osd, inode, iobuf); - /* we don't do stats here as in read path because - * write is async: we'll do this in osd_put_bufs() */ + rc = osd_do_bio(osd, inode, iobuf); + /* we don't do stats here as in read path because + * write is async: we'll do this in osd_put_bufs() */ } else { osd_fini_iobuf(osd, iobuf); } - if (unlikely(rc != 0)) { - /* if write fails, we should drop pages from the cache */ - for (i = 0; i < npages; i++) { - if (lnb[i].page == NULL) - continue; - LASSERT(PageLocked(lnb[i].page)); - generic_error_remove_page(inode->i_mapping,lnb[i].page); - } - } + osd_trans_exec_check(env, thandle, OSD_OT_WRITE); - RETURN(rc); + if (unlikely(rc != 0)) { + /* if write fails, we should drop pages from the cache */ + for (i = 0; i < npages; i++) { + if (lnb[i].lnb_page == NULL) + continue; + LASSERT(PageLocked(lnb[i].lnb_page)); + generic_error_remove_page(inode->i_mapping, + lnb[i].lnb_page); + } + } + + RETURN(rc); } static int osd_read_prep(const struct lu_env *env, struct dt_object *dt, @@ -1152,9 +1358,10 @@ static int osd_read_prep(const struct lu_env *env, struct dt_object *dt, struct osd_iobuf *iobuf = &oti->oti_iobuf; struct inode *inode = osd_dt_obj(dt)->oo_inode; struct osd_device *osd = osd_obj2dev(osd_dt_obj(dt)); - struct timeval start, end; - unsigned long timediff; - int rc = 0, i, m = 0, cache = 0; + int rc = 0, i, cache = 0, cache_hits = 0, cache_misses = 0; + ktime_t start, end; + s64 timediff; + loff_t isize; LASSERT(inode); @@ -1162,47 +1369,59 @@ static int osd_read_prep(const struct lu_env *env, struct dt_object *dt, if (unlikely(rc != 0)) RETURN(rc); + isize = i_size_read(inode); + if (osd->od_read_cache) cache = 1; - if (i_size_read(inode) > osd->od_readcache_max_filesize) + if (isize > osd->od_readcache_max_filesize) cache = 0; - do_gettimeofday(&start); + start = ktime_get(); for (i = 0; i < npages; i++) { - if (i_size_read(inode) <= lnb[i].lnb_file_offset) - /* If there's no more data, abort early. - * lnb->rc == 0, so it's easy to detect later. */ - break; + if (isize <= lnb[i].lnb_file_offset) + /* If there's no more data, abort early. + * lnb->lnb_rc == 0, so it's easy to detect later. */ + break; + + if (isize < lnb[i].lnb_file_offset + lnb[i].lnb_len) + lnb[i].lnb_rc = isize - lnb[i].lnb_file_offset; + else + lnb[i].lnb_rc = lnb[i].lnb_len; + + /* Bypass disk read if fail_loc is set properly */ + if (OBD_FAIL_CHECK(OBD_FAIL_OST_FAKE_RW)) + SetPageUptodate(lnb[i].lnb_page); + + if (PageUptodate(lnb[i].lnb_page)) { + cache_hits++; + } else { + cache_misses++; + osd_iobuf_add_page(iobuf, lnb[i].lnb_page); + } - if (i_size_read(inode) < - lnb[i].lnb_file_offset + lnb[i].len - 1) - lnb[i].rc = i_size_read(inode) - lnb[i].lnb_file_offset; - else - lnb[i].rc = lnb[i].len; - m += lnb[i].len; - - lprocfs_counter_add(osd->od_stats, LPROC_OSD_CACHE_ACCESS, 1); - if (PageUptodate(lnb[i].page)) { - lprocfs_counter_add(osd->od_stats, - LPROC_OSD_CACHE_HIT, 1); - } else { - lprocfs_counter_add(osd->od_stats, - LPROC_OSD_CACHE_MISS, 1); - osd_iobuf_add_page(iobuf, lnb[i].page); - } if (cache == 0) - generic_error_remove_page(inode->i_mapping,lnb[i].page); + generic_error_remove_page(inode->i_mapping, + lnb[i].lnb_page); } - do_gettimeofday(&end); - timediff = cfs_timeval_sub(&end, &start, NULL); + end = ktime_get(); + timediff = ktime_us_delta(end, start); lprocfs_counter_add(osd->od_stats, LPROC_OSD_GET_PAGE, timediff); + if (cache_hits != 0) + lprocfs_counter_add(osd->od_stats, LPROC_OSD_CACHE_HIT, + cache_hits); + if (cache_misses != 0) + lprocfs_counter_add(osd->od_stats, LPROC_OSD_CACHE_MISS, + cache_misses); + if (cache_hits + cache_misses != 0) + lprocfs_counter_add(osd->od_stats, LPROC_OSD_CACHE_ACCESS, + cache_hits + cache_misses); + if (iobuf->dr_npages) { rc = osd_ldiskfs_map_inode_pages(inode, iobuf->dr_pages, iobuf->dr_npages, - iobuf->dr_blocks, - 0, NULL); + iobuf->dr_blocks, 0); rc = osd_do_bio(osd, inode, iobuf); /* IO stats will be done in osd_bufs_put() */ @@ -1239,7 +1458,6 @@ int osd_ldiskfs_read(struct inode *inode, void *buf, int size, loff_t *offs) int blocksize; int csize; int boffs; - int err; /* prevent reading after eof */ spin_lock(&inode->i_lock); @@ -1265,16 +1483,21 @@ int osd_ldiskfs_read(struct inode *inode, void *buf, int size, loff_t *offs) block = *offs >> inode->i_blkbits; boffs = *offs & (blocksize - 1); csize = min(blocksize - boffs, size); - bh = ldiskfs_bread(NULL, inode, block, 0, &err); - if (!bh) { - CERROR("%s: can't read %u@%llu on ino %lu: rc = %d\n", - LDISKFS_SB(inode->i_sb)->s_es->s_volume_name, - csize, *offs, inode->i_ino, err); - return err; - } + bh = __ldiskfs_bread(NULL, inode, block, 0); + if (IS_ERR(bh)) { + CERROR("%s: can't read %u@%llu on ino %lu: " + "rc = %ld\n", osd_ino2name(inode), + csize, *offs, inode->i_ino, + PTR_ERR(bh)); + return PTR_ERR(bh); + } - memcpy(buf, bh->b_data + boffs, csize); - brelse(bh); + if (bh != NULL) { + memcpy(buf, bh->b_data + boffs, csize); + brelse(bh); + } else { + memset(buf, 0, csize); + } *offs += csize; buf += csize; @@ -1284,15 +1507,11 @@ int osd_ldiskfs_read(struct inode *inode, void *buf, int size, loff_t *offs) } static ssize_t osd_read(const struct lu_env *env, struct dt_object *dt, - struct lu_buf *buf, loff_t *pos, - struct lustre_capa *capa) + struct lu_buf *buf, loff_t *pos) { struct inode *inode = osd_dt_obj(dt)->oo_inode; int rc; - if (osd_object_auth(env, dt, capa, CAPA_OPC_BODY_READ)) - RETURN(-EACCES); - /* Read small symlink from inode body as we need to maintain correct * on-disk symlinks for ldiskfs. */ @@ -1311,17 +1530,16 @@ static inline int osd_extents_enabled(struct super_block *sb, if (inode != NULL) { if (LDISKFS_I(inode)->i_flags & LDISKFS_EXTENTS_FL) return 1; - } else if (test_opt(sb, EXTENTS)) { + } else if (LDISKFS_HAS_INCOMPAT_FEATURE(sb, + LDISKFS_FEATURE_INCOMPAT_EXTENTS)) { return 1; } return 0; } -static inline int osd_calc_bkmap_credits(struct super_block *sb, - struct inode *inode, - const loff_t size, - const loff_t pos, - const int blocks) +int osd_calc_bkmap_credits(struct super_block *sb, struct inode *inode, + const loff_t size, const loff_t pos, + const int blocks) { int credits, bits, bs, i; @@ -1356,9 +1574,12 @@ static inline int osd_calc_bkmap_credits(struct super_block *sb, } else if (pos + size <= (LDISKFS_NDIR_BLOCKS + 1024) * bs) { /* single indirect */ credits = blocks * 3; - /* probably indirect block has been allocated already */ - if (!inode || LDISKFS_I(inode)->i_data[LDISKFS_IND_BLOCK]) + if (inode == NULL || + LDISKFS_I(inode)->i_data[LDISKFS_IND_BLOCK] == 0) credits += 3; + else + /* The indirect block may be modified. */ + credits += 1; } return credits; @@ -1426,7 +1647,7 @@ static ssize_t osd_declare_write(const struct lu_env *env, struct dt_object *dt, credits = depth; /* if not append, then split may need to modify * existing blocks moving entries into the new ones */ - if (_pos == -1) + if (_pos != -1) credits += depth; /* blocks to store data: bitmap,gd,itself */ credits += blocks * 3; @@ -1447,8 +1668,9 @@ out: * objects, so always set the lqi_space as 0. */ if (inode != NULL) rc = osd_declare_inode_qid(env, i_uid_read(inode), - i_gid_read(inode), 0, oh, true, - true, NULL, false); + i_gid_read(inode), + i_projid_read(inode), 0, + oh, obj, NULL, OSD_QID_BLK); RETURN(rc); } @@ -1457,12 +1679,14 @@ static int osd_ldiskfs_writelink(struct inode *inode, char *buffer, int buflen) /* LU-2634: clear the extent format for fast symlink */ ldiskfs_clear_inode_flag(inode, LDISKFS_INODE_EXTENTS); - memcpy((char *)&LDISKFS_I(inode)->i_data, (char *)buffer, buflen); - LDISKFS_I(inode)->i_disksize = buflen; - i_size_write(inode, buflen); + memcpy((char *)&LDISKFS_I(inode)->i_data, (char *)buffer, buflen); + spin_lock(&inode->i_lock); + LDISKFS_I(inode)->i_disksize = buflen; + i_size_write(inode, buflen); + spin_unlock(&inode->i_lock); ll_dirty_inode(inode, I_DIRTY_DATASYNC); - return 0; + return 0; } int osd_ldiskfs_write_record(struct inode *inode, void *buf, int bufsize, @@ -1487,18 +1711,29 @@ int osd_ldiskfs_write_record(struct inode *inode, void *buf, int bufsize, ((char *)buf)[bufsize] = '\0'; ++bufsize; } - while (bufsize > 0) { - if (bh != NULL) - brelse(bh); - - block = offset >> inode->i_blkbits; - boffs = offset & (blocksize - 1); - size = min(blocksize - boffs, bufsize); - bh = ldiskfs_bread(handle, inode, block, 1, &err); - if (!bh) { - CERROR("%s: error reading offset %llu (block %lu): " - "rc = %d\n", - inode->i_sb->s_id, offset, block, err); + + while (bufsize > 0) { + int credits = handle->h_buffer_credits; + + if (bh) + brelse(bh); + + block = offset >> inode->i_blkbits; + boffs = offset & (blocksize - 1); + size = min(blocksize - boffs, bufsize); + bh = __ldiskfs_bread(handle, inode, block, 1); + if (IS_ERR_OR_NULL(bh)) { + if (bh == NULL) { + err = -EIO; + } else { + err = PTR_ERR(bh); + bh = NULL; + } + + CERROR("%s: error reading offset %llu (block %lu, " + "size %d, offs %llu), credits %d/%d: rc = %d\n", + inode->i_sb->s_id, offset, block, bufsize, *offs, + credits, handle->h_buffer_credits, err); break; } @@ -1508,11 +1743,11 @@ int osd_ldiskfs_write_record(struct inode *inode, void *buf, int bufsize, err); break; } - LASSERTF(boffs + size <= bh->b_size, - "boffs %d size %d bh->b_size %lu", - boffs, size, (unsigned long)bh->b_size); + LASSERTF(boffs + size <= bh->b_size, + "boffs %d size %d bh->b_size %lu\n", + boffs, size, (unsigned long)bh->b_size); memcpy(bh->b_data + boffs, buf, size); - err = ldiskfs_journal_dirty_metadata(handle, bh); + err = ldiskfs_handle_dirty_metadata(handle, NULL, bh); if (err) break; @@ -1527,8 +1762,8 @@ int osd_ldiskfs_write_record(struct inode *inode, void *buf, int bufsize, if (write_NUL) --new_size; - /* correct in-core and on-disk sizes */ - if (new_size > i_size_read(inode)) { + /* correct in-core and on-disk sizes */ + if (new_size > i_size_read(inode)) { spin_lock(&inode->i_lock); if (new_size > i_size_read(inode)) i_size_write(inode, new_size); @@ -1547,9 +1782,8 @@ int osd_ldiskfs_write_record(struct inode *inode, void *buf, int bufsize, } static ssize_t osd_write(const struct lu_env *env, struct dt_object *dt, - const struct lu_buf *buf, loff_t *pos, - struct thandle *handle, struct lustre_capa *capa, - int ignore_quota) + const struct lu_buf *buf, loff_t *pos, + struct thandle *handle, int ignore_quota) { struct inode *inode = osd_dt_obj(dt)->oo_inode; struct osd_thandle *oh; @@ -1558,9 +1792,6 @@ static ssize_t osd_write(const struct lu_env *env, struct dt_object *dt, LASSERT(dt_object_exists(dt)); - if (osd_object_auth(env, dt, capa, CAPA_OPC_BODY_WRITE)) - return -EACCES; - LASSERT(handle != NULL); LASSERT(inode != NULL); ll_vfs_dq_init(inode); @@ -1570,6 +1801,8 @@ static ssize_t osd_write(const struct lu_env *env, struct dt_object *dt, oh = container_of(handle, struct osd_thandle, ot_super); LASSERT(oh->ot_handle->h_transaction != NULL); + osd_trans_exec_op(env, handle, OSD_OT_WRITE); + /* Write small symlink to inode body as we need to maintain correct * on-disk symlinks for ldiskfs. * Note: the buf->lb_buf contains a NUL terminator while buf->lb_len @@ -1582,9 +1815,12 @@ static ssize_t osd_write(const struct lu_env *env, struct dt_object *dt, result = osd_ldiskfs_write_record(inode, buf->lb_buf, buf->lb_len, is_link, pos, oh->ot_handle); - if (result == 0) - result = buf->lb_len; - return result; + if (result == 0) + result = buf->lb_len; + + osd_trans_exec_check(env, handle, OSD_OT_WRITE); + + return result; } static int osd_declare_punch(const struct lu_env *env, struct dt_object *dt, @@ -1613,13 +1849,13 @@ static int osd_declare_punch(const struct lu_env *env, struct dt_object *dt, LASSERT(inode); rc = osd_declare_inode_qid(env, i_uid_read(inode), i_gid_read(inode), - 0, oh, true, true, NULL, false); + i_projid_read(inode), 0, oh, osd_dt_obj(dt), + NULL, OSD_QID_BLK); RETURN(rc); } static int osd_punch(const struct lu_env *env, struct dt_object *dt, - __u64 start, __u64 end, struct thandle *th, - struct lustre_capa *capa) + __u64 start, __u64 end, struct thandle *th) { struct osd_thandle *oh; struct osd_object *obj = osd_dt_obj(dt); @@ -1643,7 +1879,9 @@ static int osd_punch(const struct lu_env *env, struct dt_object *dt, tid = oh->ot_handle->h_transaction->t_tid; + spin_lock(&inode->i_lock); i_size_write(inode, start); + spin_unlock(&inode->i_lock); ll_truncate_pagecache(inode, start); #ifdef HAVE_INODEOPS_TRUNCATE if (inode->i_op->truncate) { @@ -1656,13 +1894,17 @@ static int osd_punch(const struct lu_env *env, struct dt_object *dt, * For a partial-page truncate, flush the page to disk immediately to * avoid data corruption during direct disk write. b=17397 */ - if ((start & ~CFS_PAGE_MASK) != 0) + if ((start & ~PAGE_MASK) != 0) rc = filemap_fdatawrite_range(inode->i_mapping, start, start+1); h = journal_current_handle(); LASSERT(h != NULL); LASSERT(h == oh->ot_handle); + /* do not check credits with osd_trans_exec_check() as the truncate + * can restart the transaction internally and we restart the + * transaction in this case */ + if (tid != h->h_transaction->t_tid) { int credits = oh->ot_credits; /* @@ -1678,34 +1920,91 @@ static int osd_punch(const struct lu_env *env, struct dt_object *dt, RETURN(rc == 0 ? rc2 : rc); } +static int fiemap_check_ranges(struct inode *inode, + u64 start, u64 len, u64 *new_len) +{ + loff_t maxbytes; + + *new_len = len; + + if (len == 0) + return -EINVAL; + + if (ldiskfs_test_inode_flag(inode, LDISKFS_INODE_EXTENTS)) + maxbytes = inode->i_sb->s_maxbytes; + else + maxbytes = LDISKFS_SB(inode->i_sb)->s_bitmap_maxbytes; + + if (start > maxbytes) + return -EFBIG; + + /* + * Shrink request scope to what the fs can actually handle. + */ + if (len > maxbytes || (maxbytes - len) < start) + *new_len = maxbytes - start; + + return 0; +} + +/* So that the fiemap access checks can't overflow on 32 bit machines. */ +#define FIEMAP_MAX_EXTENTS (UINT_MAX / sizeof(struct fiemap_extent)) + static int osd_fiemap_get(const struct lu_env *env, struct dt_object *dt, - struct ll_user_fiemap *fm) + struct fiemap *fm) { - struct inode *inode = osd_dt_obj(dt)->oo_inode; - struct osd_thread_info *info = osd_oti_get(env); - struct dentry *dentry = &info->oti_obj_dentry; - struct file *file = &info->oti_file; - mm_segment_t saved_fs; - int rc; + struct fiemap_extent_info fieinfo = {0, }; + struct inode *inode = osd_dt_obj(dt)->oo_inode; + u64 len; + int rc; - LASSERT(inode); - dentry->d_inode = inode; - dentry->d_sb = inode->i_sb; - file->f_dentry = dentry; - file->f_mapping = inode->i_mapping; - file->f_op = inode->i_fop; - set_file_inode(file, inode); - - saved_fs = get_fs(); - set_fs(get_ds()); - /* ldiskfs_ioctl does not have a inode argument */ - if (inode->i_fop->unlocked_ioctl) - rc = inode->i_fop->unlocked_ioctl(file, FSFILT_IOC_FIEMAP, - (long)fm); - else - rc = -ENOTTY; - set_fs(saved_fs); - return rc; + + LASSERT(inode); + if (inode->i_op->fiemap == NULL) + return -EOPNOTSUPP; + + if (fm->fm_extent_count > FIEMAP_MAX_EXTENTS) + return -EINVAL; + + rc = fiemap_check_ranges(inode, fm->fm_start, fm->fm_length, &len); + if (rc) + return rc; + + fieinfo.fi_flags = fm->fm_flags; + fieinfo.fi_extents_max = fm->fm_extent_count; + fieinfo.fi_extents_start = fm->fm_extents; + + if (fieinfo.fi_flags & FIEMAP_FLAG_SYNC) + filemap_write_and_wait(inode->i_mapping); + + rc = inode->i_op->fiemap(inode, &fieinfo, fm->fm_start, len); + fm->fm_flags = fieinfo.fi_flags; + fm->fm_mapped_extents = fieinfo.fi_extents_mapped; + + return rc; +} + +static int osd_ladvise(const struct lu_env *env, struct dt_object *dt, + __u64 start, __u64 end, enum lu_ladvise_type advice) +{ + int rc = 0; + struct inode *inode = osd_dt_obj(dt)->oo_inode; + ENTRY; + + switch (advice) { + case LU_LADVISE_DONTNEED: + if (end == 0) + break; + invalidate_mapping_pages(inode->i_mapping, + start >> PAGE_CACHE_SHIFT, + (end - 1) >> PAGE_CACHE_SHIFT); + break; + default: + rc = -ENOTSUPP; + break; + } + + RETURN(rc); } /* @@ -1717,17 +2016,17 @@ const struct dt_body_operations osd_body_ops_new = { }; const struct dt_body_operations osd_body_ops = { - .dbo_read = osd_read, - .dbo_declare_write = osd_declare_write, - .dbo_write = osd_write, - .dbo_bufs_get = osd_bufs_get, - .dbo_bufs_put = osd_bufs_put, - .dbo_write_prep = osd_write_prep, - .dbo_declare_write_commit = osd_declare_write_commit, - .dbo_write_commit = osd_write_commit, - .dbo_read_prep = osd_read_prep, - .dbo_declare_punch = osd_declare_punch, - .dbo_punch = osd_punch, - .dbo_fiemap_get = osd_fiemap_get, + .dbo_read = osd_read, + .dbo_declare_write = osd_declare_write, + .dbo_write = osd_write, + .dbo_bufs_get = osd_bufs_get, + .dbo_bufs_put = osd_bufs_put, + .dbo_write_prep = osd_write_prep, + .dbo_declare_write_commit = osd_declare_write_commit, + .dbo_write_commit = osd_write_commit, + .dbo_read_prep = osd_read_prep, + .dbo_declare_punch = osd_declare_punch, + .dbo_punch = osd_punch, + .dbo_fiemap_get = osd_fiemap_get, + .dbo_ladvise = osd_ladvise, }; -