X-Git-Url: https://git.whamcloud.com/?a=blobdiff_plain;f=lib%2Fext2fs%2Funix_io.c;h=353d85afa7fe032db418681226e105c0f7e294d8;hb=0e0c7537eb5fdcef9a90b1a3ed8365c3878392a1;hp=21a273db6e4f6a47558a98a21da6d270c9925a8e;hpb=c859cb1de0d624caa0779fb17d1a53766143136e;p=tools%2Fe2fsprogs.git diff --git a/lib/ext2fs/unix_io.c b/lib/ext2fs/unix_io.c index 21a273d..353d85a 100644 --- a/lib/ext2fs/unix_io.c +++ b/lib/ext2fs/unix_io.c @@ -1,13 +1,13 @@ /* * unix_io.c --- This is the Unix (well, really POSIX) implementation - * of the I/O manager. + * of the I/O manager. * * Implements a one-block write-through cache. * * Includes support for Windows NT support under Cygwin. * * Copyright (C) 1993, 1994, 1995, 1996, 1997, 1998, 1999, 2000, 2001, - * 2002 by Theodore Ts'o. + * 2002 by Theodore Ts'o. * * %Begin-Header% * This file may be redistributed under the terms of the GNU Library @@ -15,12 +15,22 @@ * %End-Header% */ +#if !defined(__FreeBSD__) && !defined(__NetBSD__) && !defined(__OpenBSD__) +#define _XOPEN_SOURCE 600 +#define _DARWIN_C_SOURCE +#define _FILE_OFFSET_BITS 64 +#ifndef _LARGEFILE_SOURCE #define _LARGEFILE_SOURCE +#endif +#ifndef _LARGEFILE64_SOURCE #define _LARGEFILE64_SOURCE +#endif #ifndef _GNU_SOURCE #define _GNU_SOURCE #endif +#endif +#include "config.h" #include #include #if HAVE_UNISTD_H @@ -34,34 +44,42 @@ #ifdef __linux__ #include #endif +#if HAVE_SYS_TYPES_H +#include +#endif #ifdef HAVE_SYS_IOCTL_H #include #endif #ifdef HAVE_SYS_MOUNT_H #include #endif +#ifdef HAVE_SYS_PRCTL_H +#include +#else +#define PR_GET_DUMPABLE 3 +#endif #if HAVE_SYS_STAT_H #include #endif -#if HAVE_SYS_TYPES_H -#include -#endif #if HAVE_SYS_RESOURCE_H #include #endif +#if HAVE_LINUX_FALLOC_H +#include +#endif +#ifdef HAVE_PTHREAD +#include +#endif #if defined(__linux__) && defined(_IO) && !defined(BLKROGET) #define BLKROGET _IO(0x12, 94) /* Get read-only status (0 = read_write). */ #endif -#if defined(__linux__) && defined(_IO) && !defined(BLKSSZGET) -#define BLKSSZGET _IO(0x12,104)/* get block device sector size */ -#endif - #undef ALIGN_DEBUG #include "ext2_fs.h" #include "ext2fs.h" +#include "ext2fsP.h" /* * For checking structure magic numbers... @@ -71,11 +89,12 @@ if ((struct)->magic != (code)) return (code) struct unix_cache { - char *buf; - unsigned long block; - int access_time; - unsigned dirty:1; - unsigned in_use:1; + char *buf; + unsigned long long block; + int access_time; + unsigned dirty:1; + unsigned in_use:1; + unsigned write_err:1; }; #define CACHE_SIZE 8 @@ -92,56 +111,61 @@ struct unix_private_data { struct unix_cache cache[CACHE_SIZE]; void *bounce; struct struct_io_stats io_stats; +#ifdef HAVE_PTHREAD + pthread_mutex_t cache_mutex; + pthread_mutex_t bounce_mutex; + pthread_mutex_t stats_mutex; +#endif }; -#define IS_ALIGNED(n, align) ((((unsigned long) n) & \ - ((unsigned long) ((align)-1))) == 0) +#define IS_ALIGNED(n, align) ((((uintptr_t) n) & \ + ((uintptr_t) ((align)-1))) == 0) -static errcode_t unix_open(const char *name, int flags, io_channel *channel); -static errcode_t unix_close(io_channel channel); -static errcode_t unix_set_blksize(io_channel channel, int blksize); -static errcode_t unix_read_blk(io_channel channel, unsigned long block, - int count, void *data); -static errcode_t unix_write_blk(io_channel channel, unsigned long block, - int count, const void *data); -static errcode_t unix_flush(io_channel channel); -static errcode_t unix_write_byte(io_channel channel, unsigned long offset, - int size, const void *data); -static errcode_t unix_set_option(io_channel channel, const char *option, - const char *arg); -static errcode_t unix_get_stats(io_channel channel, io_stats *stats) -; -static void reuse_cache(io_channel channel, struct unix_private_data *data, - struct unix_cache *cache, unsigned long long block); -static errcode_t unix_read_blk64(io_channel channel, unsigned long long block, - int count, void *data); -static errcode_t unix_write_blk64(io_channel channel, unsigned long long block, - int count, const void *data); -static errcode_t unix_discard(io_channel channel, unsigned long long block, - unsigned long long count); +typedef enum lock_kind { + CACHE_MTX, BOUNCE_MTX, STATS_MTX +} kind_t; -static struct struct_io_manager struct_unix_manager = { - EXT2_ET_MAGIC_IO_MANAGER, - "Unix I/O Manager", - unix_open, - unix_close, - unix_set_blksize, - unix_read_blk, - unix_write_blk, - unix_flush, - unix_write_byte, - unix_set_option, - unix_get_stats, - unix_read_blk64, - unix_write_blk64, - unix_discard, -}; +#ifdef HAVE_PTHREAD +static inline pthread_mutex_t *get_mutex(struct unix_private_data *data, + kind_t kind) +{ + if (data->flags & IO_FLAG_THREADS) { + switch (kind) { + case CACHE_MTX: + return &data->cache_mutex; + case BOUNCE_MTX: + return &data->bounce_mutex; + case STATS_MTX: + return &data->stats_mutex; + } + } + return NULL; +} +#endif -io_manager unix_io_manager = &struct_unix_manager; +static inline void mutex_lock(struct unix_private_data *data, kind_t kind) +{ +#ifdef HAVE_PTHREAD + pthread_mutex_t *mtx = get_mutex(data,kind); + + if (mtx) + pthread_mutex_lock(mtx); +#endif +} + +static inline void mutex_unlock(struct unix_private_data *data, kind_t kind) +{ +#ifdef HAVE_PTHREAD + pthread_mutex_t *mtx = get_mutex(data,kind); + + if (mtx) + pthread_mutex_unlock(mtx); +#endif +} static errcode_t unix_get_stats(io_channel channel, io_stats *stats) { - errcode_t retval = 0; + errcode_t retval = 0; struct unix_private_data *data; @@ -149,12 +173,38 @@ static errcode_t unix_get_stats(io_channel channel, io_stats *stats) data = (struct unix_private_data *) channel->private_data; EXT2_CHECK_MAGIC(data, EXT2_ET_MAGIC_UNIX_IO_CHANNEL); - if (stats) + if (stats) { + mutex_lock(data, STATS_MTX); *stats = &data->io_stats; + mutex_unlock(data, STATS_MTX); + } return retval; } +static char *safe_getenv(const char *arg) +{ + if ((getuid() != geteuid()) || (getgid() != getegid())) + return NULL; +#ifdef HAVE_PRCTL + if (prctl(PR_GET_DUMPABLE, 0, 0, 0, 0) == 0) + return NULL; +#else +#if (defined(linux) && defined(SYS_prctl)) + if (syscall(SYS_prctl, PR_GET_DUMPABLE, 0, 0, 0, 0) == 0) + return NULL; +#endif +#endif + +#if defined(HAVE_SECURE_GETENV) + return secure_getenv(arg); +#elif defined(HAVE___SECURE_GETENV) + return __secure_getenv(arg); +#else + return getenv(arg); +#endif +} + /* * Here are the raw I/O functions */ @@ -168,25 +218,64 @@ static errcode_t raw_read_blk(io_channel channel, ext2_loff_t location; int actual = 0; unsigned char *buf = bufv; + ssize_t really_read = 0; + unsigned long long aligned_blk; + int align_size, offset; - size = (count < 0) ? -count : count * channel->block_size; + size = (count < 0) ? -count : (ext2_loff_t) count * channel->block_size; + mutex_lock(data, STATS_MTX); data->io_stats.bytes_read += size; + mutex_unlock(data, STATS_MTX); location = ((ext2_loff_t) block * channel->block_size) + data->offset; - if (ext2fs_llseek(data->dev, location, SEEK_SET) != location) { - retval = errno ? errno : EXT2_ET_LLSEEK_FAILED; - goto error_out; + + if (data->flags & IO_FLAG_FORCE_BOUNCE) + goto bounce_read; + +#ifdef HAVE_PREAD64 + /* Try an aligned pread */ + if ((channel->align == 0) || + (IS_ALIGNED(buf, channel->align) && + IS_ALIGNED(location, channel->align) && + IS_ALIGNED(size, channel->align))) { + actual = pread64(data->dev, buf, size, location); + if (actual == size) + return 0; + actual = 0; + } +#elif HAVE_PREAD + /* Try an aligned pread */ + if ((sizeof(off_t) >= sizeof(ext2_loff_t)) && + ((channel->align == 0) || + (IS_ALIGNED(buf, channel->align) && + IS_ALIGNED(location, channel->align) && + IS_ALIGNED(size, channel->align)))) { + actual = pread(data->dev, buf, size, location); + if (actual == size) + return 0; + actual = 0; } - if ((data->align == 0) || - ((IS_ALIGNED(buf, data->align)) && IS_ALIGNED(size, data->align))) { +#endif /* HAVE_PREAD */ + + if ((channel->align == 0) || + (IS_ALIGNED(buf, channel->align) && + IS_ALIGNED(location, channel->align) && + IS_ALIGNED(size, channel->align))) { + mutex_lock(data, BOUNCE_MTX); + if (ext2fs_llseek(data->dev, location, SEEK_SET) < 0) { + retval = errno ? errno : EXT2_ET_LLSEEK_FAILED; + goto error_unlock; + } actual = read(data->dev, buf, size); if (actual != size) { short_read: - if (actual < 0) + if (actual < 0) { + retval = errno; actual = 0; - retval = EXT2_ET_SHORT_READ; - goto error_out; + } else + retval = EXT2_ET_SHORT_READ; + goto error_unlock; } - return 0; + goto success_unlock; } #ifdef ALIGN_DEBUG @@ -198,37 +287,72 @@ static errcode_t raw_read_blk(io_channel channel, * The buffer or size which we're trying to read isn't aligned * to the O_DIRECT rules, so we need to do this the hard way... */ +bounce_read: + if (channel->align == 0) + channel->align = 1; + if ((channel->block_size > channel->align) && + (channel->block_size % channel->align) == 0) + align_size = channel->block_size; + else + align_size = channel->align; + aligned_blk = location / align_size; + offset = location % align_size; + + mutex_lock(data, BOUNCE_MTX); + if (ext2fs_llseek(data->dev, aligned_blk * align_size, SEEK_SET) < 0) { + retval = errno ? errno : EXT2_ET_LLSEEK_FAILED; + goto error_unlock; + } while (size > 0) { - actual = read(data->dev, data->bounce, channel->block_size); - if (actual != channel->block_size) + actual = read(data->dev, data->bounce, align_size); + if (actual != align_size) { + mutex_unlock(data, BOUNCE_MTX); + actual = really_read; + buf -= really_read; + size += really_read; goto short_read; - actual = size; - if (size > channel->block_size) - actual = channel->block_size; - memcpy(buf, data->bounce, actual); + } + if ((actual + offset) > align_size) + actual = align_size - offset; + if (actual > size) + actual = size; + memcpy(buf, (char *)data->bounce + offset, actual); + + really_read += actual; size -= actual; buf += actual; + offset = 0; + aligned_blk++; } +success_unlock: + mutex_unlock(data, BOUNCE_MTX); return 0; -error_out: - memset((char *) buf+actual, 0, size-actual); +error_unlock: + mutex_unlock(data, BOUNCE_MTX); + if (actual >= 0 && actual < size) + memset((char *) buf+actual, 0, size-actual); if (channel->read_error) retval = (channel->read_error)(channel, block, count, buf, size, actual, retval); return retval; } +#define RAW_WRITE_NO_HANDLER 1 + static errcode_t raw_write_blk(io_channel channel, struct unix_private_data *data, unsigned long long block, - int count, const void *bufv) + int count, const void *bufv, + int flags) { ssize_t size; ext2_loff_t location; int actual = 0; errcode_t retval; const unsigned char *buf = bufv; + unsigned long long aligned_blk; + int align_size, offset; if (count == 1) size = channel->block_size; @@ -236,19 +360,55 @@ static errcode_t raw_write_blk(io_channel channel, if (count < 0) size = -count; else - size = count * channel->block_size; + size = (ext2_loff_t) count * channel->block_size; } + mutex_lock(data, STATS_MTX); data->io_stats.bytes_written += size; + mutex_unlock(data, STATS_MTX); location = ((ext2_loff_t) block * channel->block_size) + data->offset; - if (ext2fs_llseek(data->dev, location, SEEK_SET) != location) { - retval = errno ? errno : EXT2_ET_LLSEEK_FAILED; - goto error_out; - } - if ((data->align == 0) || - ((IS_ALIGNED(buf, data->align)) && IS_ALIGNED(size, data->align))) { + if (data->flags & IO_FLAG_FORCE_BOUNCE) + goto bounce_write; + +#ifdef HAVE_PWRITE64 + /* Try an aligned pwrite */ + if ((channel->align == 0) || + (IS_ALIGNED(buf, channel->align) && + IS_ALIGNED(location, channel->align) && + IS_ALIGNED(size, channel->align))) { + actual = pwrite64(data->dev, buf, size, location); + if (actual == size) + return 0; + } +#elif HAVE_PWRITE + /* Try an aligned pwrite */ + if ((sizeof(off_t) >= sizeof(ext2_loff_t)) && + ((channel->align == 0) || + (IS_ALIGNED(buf, channel->align) && + IS_ALIGNED(location, channel->align) && + IS_ALIGNED(size, channel->align)))) { + actual = pwrite(data->dev, buf, size, location); + if (actual == size) + return 0; + } +#endif /* HAVE_PWRITE */ + + if ((channel->align == 0) || + (IS_ALIGNED(buf, channel->align) && + IS_ALIGNED(location, channel->align) && + IS_ALIGNED(size, channel->align))) { + mutex_lock(data, BOUNCE_MTX); + if (ext2fs_llseek(data->dev, location, SEEK_SET) < 0) { + retval = errno ? errno : EXT2_ET_LLSEEK_FAILED; + goto error_unlock; + } actual = write(data->dev, buf, size); + mutex_unlock(data, BOUNCE_MTX); + if (actual < 0) { + retval = errno; + goto error_out; + } if (actual != size) { short_write: retval = EXT2_ET_SHORT_WRITE; @@ -265,29 +425,68 @@ static errcode_t raw_write_blk(io_channel channel, * The buffer or size which we're trying to write isn't aligned * to the O_DIRECT rules, so we need to do this the hard way... */ +bounce_write: + if (channel->align == 0) + channel->align = 1; + if ((channel->block_size > channel->align) && + (channel->block_size % channel->align) == 0) + align_size = channel->block_size; + else + align_size = channel->align; + aligned_blk = location / align_size; + offset = location % align_size; + while (size > 0) { - if (size < channel->block_size) { + int actual_w; + + mutex_lock(data, BOUNCE_MTX); + if (size < align_size || offset) { + if (ext2fs_llseek(data->dev, aligned_blk * align_size, + SEEK_SET) < 0) { + retval = errno ? errno : EXT2_ET_LLSEEK_FAILED; + goto error_unlock; + } actual = read(data->dev, data->bounce, - channel->block_size); - if (actual != channel->block_size) { - retval = EXT2_ET_SHORT_READ; - goto error_out; + align_size); + if (actual != align_size) { + if (actual < 0) { + retval = errno; + goto error_unlock; + } + memset((char *) data->bounce + actual, 0, + align_size - actual); } } actual = size; - if (size > channel->block_size) - actual = channel->block_size; - memcpy(data->bounce, buf, actual); - actual = write(data->dev, data->bounce, channel->block_size); - if (actual != channel->block_size) + if ((actual + offset) > align_size) + actual = align_size - offset; + if (actual > size) + actual = size; + memcpy(((char *)data->bounce) + offset, buf, actual); + if (ext2fs_llseek(data->dev, aligned_blk * align_size, SEEK_SET) < 0) { + retval = errno ? errno : EXT2_ET_LLSEEK_FAILED; + goto error_unlock; + } + actual_w = write(data->dev, data->bounce, align_size); + mutex_unlock(data, BOUNCE_MTX); + if (actual_w < 0) { + retval = errno; + goto error_out; + } + if (actual_w != align_size) goto short_write; size -= actual; buf += actual; + location += actual; + aligned_blk++; + offset = 0; } return 0; +error_unlock: + mutex_unlock(data, BOUNCE_MTX); error_out: - if (channel->write_error) + if (((flags & RAW_WRITE_NO_HANDLER) == 0) && channel->write_error) retval = (channel->write_error)(channel, block, count, buf, size, actual, retval); return retval; @@ -314,16 +513,14 @@ static errcode_t alloc_cache(io_channel channel, cache->in_use = 0; if (cache->buf) ext2fs_free_mem(&cache->buf); - retval = ext2fs_get_memalign(channel->block_size, - data->align, &cache->buf); + retval = io_channel_alloc_buf(channel, 0, &cache->buf); if (retval) return retval; } - if (data->align) { + if (channel->align || data->flags & IO_FLAG_FORCE_BOUNCE) { if (data->bounce) ext2fs_free_mem(&data->bounce); - retval = ext2fs_get_memalign(channel->block_size, data->align, - &data->bounce); + retval = io_channel_alloc_buf(channel, 0, &data->bounce); } return retval; } @@ -383,47 +580,99 @@ static struct unix_cache *find_cached_block(struct unix_private_data *data, /* * Reuse a particular cache entry for another block. */ -static void reuse_cache(io_channel channel, struct unix_private_data *data, - struct unix_cache *cache, unsigned long long block) +static errcode_t reuse_cache(io_channel channel, + struct unix_private_data *data, struct unix_cache *cache, + unsigned long long block) { - if (cache->dirty && cache->in_use) - raw_write_blk(channel, data, cache->block, 1, cache->buf); + if (cache->dirty && cache->in_use) { + errcode_t retval; + + retval = raw_write_blk(channel, data, cache->block, 1, + cache->buf, RAW_WRITE_NO_HANDLER); + if (retval) { + cache->write_err = 1; + return retval; + } + } cache->in_use = 1; cache->dirty = 0; + cache->write_err = 0; cache->block = block; cache->access_time = ++data->access_time; + return 0; } +#define FLUSH_INVALIDATE 0x01 +#define FLUSH_NOLOCK 0x02 + /* * Flush all of the blocks in the cache */ static errcode_t flush_cached_blocks(io_channel channel, struct unix_private_data *data, - int invalidate) - + int flags) { struct unix_cache *cache; - errcode_t retval, retval2; + errcode_t retval, retval2 = 0; int i; + int errors_found = 0; - retval2 = 0; + if ((flags & FLUSH_NOLOCK) == 0) + mutex_lock(data, CACHE_MTX); for (i=0, cache = data->cache; i < CACHE_SIZE; i++, cache++) { - if (!cache->in_use) + if (!cache->in_use || !cache->dirty) continue; - - if (invalidate) - cache->in_use = 0; - - if (!cache->dirty) - continue; - retval = raw_write_blk(channel, data, - cache->block, 1, cache->buf); - if (retval) + cache->block, 1, cache->buf, + RAW_WRITE_NO_HANDLER); + if (retval) { + cache->write_err = 1; + errors_found = 1; retval2 = retval; - else + } else { cache->dirty = 0; + cache->write_err = 0; + if (flags & FLUSH_INVALIDATE) + cache->in_use = 0; + } + } + if ((flags & FLUSH_NOLOCK) == 0) + mutex_unlock(data, CACHE_MTX); +retry: + while (errors_found) { + if ((flags & FLUSH_NOLOCK) == 0) + mutex_lock(data, CACHE_MTX); + errors_found = 0; + for (i=0, cache = data->cache; i < CACHE_SIZE; i++, cache++) { + if (!cache->in_use || !cache->write_err) + continue; + errors_found = 1; + if (cache->write_err && channel->write_error) { + char *err_buf = NULL; + unsigned long long err_block = cache->block; + + cache->dirty = 0; + cache->in_use = 0; + cache->write_err = 0; + if (io_channel_alloc_buf(channel, 0, + &err_buf)) + err_buf = NULL; + else + memcpy(err_buf, cache->buf, + channel->block_size); + mutex_unlock(data, CACHE_MTX); + (channel->write_error)(channel, err_block, + 1, err_buf, channel->block_size, -1, + retval2); + if (err_buf) + ext2fs_free_mem(&err_buf); + goto retry; + } else + cache->write_err = 0; + } + if ((flags & FLUSH_NOLOCK) == 0) + mutex_unlock(data, CACHE_MTX); } return retval2; } @@ -435,29 +684,72 @@ static errcode_t flush_cached_blocks(io_channel channel, #endif #endif -static errcode_t unix_open(const char *name, int flags, io_channel *channel) +int ext2fs_open_file(const char *pathname, int flags, mode_t mode) +{ + if (mode) +#if defined(HAVE_OPEN64) && !defined(__OSX_AVAILABLE_BUT_DEPRECATED) + return open64(pathname, flags, mode); + else + return open64(pathname, flags); +#else + return open(pathname, flags, mode); + else + return open(pathname, flags); +#endif +} + +int ext2fs_stat(const char *path, ext2fs_struct_stat *buf) +{ +#if defined(HAVE_FSTAT64) && !defined(__OSX_AVAILABLE_BUT_DEPRECATED) + return stat64(path, buf); +#else + return stat(path, buf); +#endif +} + +int ext2fs_fstat(int fd, ext2fs_struct_stat *buf) +{ +#if defined(HAVE_FSTAT64) && !defined(__OSX_AVAILABLE_BUT_DEPRECATED) + return fstat64(fd, buf); +#else + return fstat(fd, buf); +#endif +} + + +static errcode_t unix_open_channel(const char *name, int fd, + int flags, io_channel *channel, + io_manager io_mgr) { io_channel io = NULL; struct unix_private_data *data = NULL; errcode_t retval; - int open_flags, zeroes = 0; ext2fs_struct_stat st; #ifdef __linux__ - struct utsname ut; + struct utsname ut; +#endif + + if (safe_getenv("UNIX_IO_FORCE_BOUNCE")) + flags |= IO_FLAG_FORCE_BOUNCE; + +#ifdef __linux__ + /* + * We need to make sure any previous errors in the block + * device are thrown away, sigh. + */ + (void) fsync(fd); #endif - if (name == 0) - return EXT2_ET_BAD_DEVICE_NAME; retval = ext2fs_get_mem(sizeof(struct struct_io_channel), &io); if (retval) - return retval; + goto cleanup; memset(io, 0, sizeof(struct struct_io_channel)); io->magic = EXT2_ET_MAGIC_IO_CHANNEL; retval = ext2fs_get_mem(sizeof(struct unix_private_data), &data); if (retval) goto cleanup; - io->manager = unix_io_manager; + io->manager = io_mgr; retval = ext2fs_get_mem(strlen(name)+1, &io->name); if (retval) goto cleanup; @@ -468,47 +760,62 @@ static errcode_t unix_open(const char *name, int flags, io_channel *channel) io->read_error = 0; io->write_error = 0; io->refcount = 1; + io->flags = 0; memset(data, 0, sizeof(struct unix_private_data)); data->magic = EXT2_ET_MAGIC_UNIX_IO_CHANNEL; data->io_stats.num_fields = 2; + data->flags = flags; + data->dev = fd; - open_flags = (flags & IO_FLAG_RW) ? O_RDWR : O_RDONLY; - if (flags & IO_FLAG_EXCLUSIVE) - open_flags |= O_EXCL; -#ifdef O_DIRECT +#if defined(O_DIRECT) if (flags & IO_FLAG_DIRECT_IO) - open_flags |= O_DIRECT; + io->align = ext2fs_get_dio_alignment(data->dev); +#elif defined(F_NOCACHE) + if (flags & IO_FLAG_DIRECT_IO) + io->align = 4096; #endif - data->flags = flags; - data->dev = ext2fs_open_file(io->name, open_flags); - if (data->dev < 0) { - retval = errno; - goto cleanup; - } - -#ifdef BLKSSZGET - if (flags & IO_FLAG_DIRECT_IO) { - if (ioctl(data->dev, BLKSSZGET, &data->align) != 0) - data->align = io->block_size; + /* + * If the device is really a block device, then set the + * appropriate flag, otherwise we can set DISCARD_ZEROES flag + * because we are going to use punch hole instead of discard + * and if it succeed, subsequent read from sparse area returns + * zero. + */ + if (ext2fs_fstat(data->dev, &st) == 0) { + if (ext2fsP_is_disk_device(st.st_mode)) + io->flags |= CHANNEL_FLAGS_BLOCK_DEVICE; + else + io->flags |= CHANNEL_FLAGS_DISCARD_ZEROES; } -#endif #ifdef BLKDISCARDZEROES - ioctl(data->dev, BLKDISCARDZEROES, &zeroes); - if (zeroes) - io->flags |= CHANNEL_FLAGS_DISCARD_ZEROES; + { + int zeroes = 0; + if (ioctl(data->dev, BLKDISCARDZEROES, &zeroes) == 0 && + zeroes) + io->flags |= CHANNEL_FLAGS_DISCARD_ZEROES; + } #endif -#if defined(__CYGWIN__) || defined(__FreeBSD__) || defined(__FreeBSD_kernel__) +#if defined(__CYGWIN__) /* * Some operating systems require that the buffers be aligned, * regardless of O_DIRECT */ - data->align = 512; + if (!io->align) + io->align = 512; #endif +#if defined(__FreeBSD__) || defined(__FreeBSD_kernel__) + if (io->flags & CHANNEL_FLAGS_BLOCK_DEVICE) { + int dio_align = ext2fs_get_dio_alignment(fd); + + if (io->align < dio_align) + io->align = dio_align; + } +#endif if ((retval = alloc_cache(io, data))) goto cleanup; @@ -521,7 +828,6 @@ static errcode_t unix_open(const char *name, int flags, io_channel *channel) /* Is the block device actually writable? */ error = ioctl(data->dev, BLKROGET, &readonly); if (!error && readonly) { - close(data->dev); retval = EPERM; goto cleanup; } @@ -548,8 +854,8 @@ static errcode_t unix_open(const char *name, int flags, io_channel *channel) (ut.release[2] == '4') && (ut.release[3] == '.') && (ut.release[4] == '1') && (ut.release[5] >= '0') && (ut.release[5] < '8')) && - (ext2fs_stat(io->name, &st) == 0) && - (S_ISBLK(st.st_mode))) { + (ext2fs_fstat(data->dev, &st) == 0) && + (ext2fsP_is_disk_device(st.st_mode))) { struct rlimit rlim; rlim.rlim_cur = rlim.rlim_max = (unsigned long) RLIM_INFINITY; @@ -562,19 +868,98 @@ static errcode_t unix_open(const char *name, int flags, io_channel *channel) } } #endif +#ifdef HAVE_PTHREAD + if (flags & IO_FLAG_THREADS) { + io->flags |= CHANNEL_FLAGS_THREADS; + retval = pthread_mutex_init(&data->cache_mutex, NULL); + if (retval) + goto cleanup; + retval = pthread_mutex_init(&data->bounce_mutex, NULL); + if (retval) { + pthread_mutex_destroy(&data->cache_mutex); + goto cleanup; + } + retval = pthread_mutex_init(&data->stats_mutex, NULL); + if (retval) { + pthread_mutex_destroy(&data->cache_mutex); + pthread_mutex_destroy(&data->bounce_mutex); + goto cleanup; + } + } +#endif *channel = io; return 0; cleanup: if (data) { + if (data->dev >= 0) + close(data->dev); free_cache(data); ext2fs_free_mem(&data); } - if (io) + if (io) { + if (io->name) { + ext2fs_free_mem(&io->name); + } ext2fs_free_mem(&io); + } return retval; } +static errcode_t unixfd_open(const char *str_fd, int flags, + io_channel *channel) +{ + int fd; + int fd_flags; + + fd = atoi(str_fd); +#if defined(HAVE_FCNTL) + fd_flags = fcntl(fd, F_GETFD); + if (fd_flags == -1) + return EBADF; + + flags = 0; + if (fd_flags & O_RDWR) + flags |= IO_FLAG_RW; + if (fd_flags & O_EXCL) + flags |= IO_FLAG_EXCLUSIVE; +#if defined(O_DIRECT) + if (fd_flags & O_DIRECT) + flags |= IO_FLAG_DIRECT_IO; +#endif +#endif /* HAVE_FCNTL */ + + return unix_open_channel(str_fd, fd, flags, channel, unixfd_io_manager); +} + +static errcode_t unix_open(const char *name, int flags, + io_channel *channel) +{ + int fd = -1; + int open_flags; + + if (name == 0) + return EXT2_ET_BAD_DEVICE_NAME; + + open_flags = (flags & IO_FLAG_RW) ? O_RDWR : O_RDONLY; + if (flags & IO_FLAG_EXCLUSIVE) + open_flags |= O_EXCL; +#if defined(O_DIRECT) + if (flags & IO_FLAG_DIRECT_IO) + open_flags |= O_DIRECT; +#endif + fd = ext2fs_open_file(name, open_flags, 0); + if (fd < 0) + return errno; +#if defined(F_NOCACHE) && !defined(IO_DIRECT) + if (flags & IO_FLAG_DIRECT_IO) { + if (fcntl(fd, F_NOCACHE, 1) < 0) + return errno; + } +#endif + return unix_open_channel(name, fd, flags, channel, unix_io_manager); +} + static errcode_t unix_close(io_channel channel) { struct unix_private_data *data; @@ -594,6 +979,13 @@ static errcode_t unix_close(io_channel channel) if (close(data->dev) < 0) retval = errno; free_cache(data); +#ifdef HAVE_PTHREAD + if (data->flags & IO_FLAG_THREADS) { + pthread_mutex_destroy(&data->cache_mutex); + pthread_mutex_destroy(&data->bounce_mutex); + pthread_mutex_destroy(&data->stats_mutex); + } +#endif ext2fs_free_mem(&channel->private_data); if (channel->name) @@ -605,32 +997,37 @@ static errcode_t unix_close(io_channel channel) static errcode_t unix_set_blksize(io_channel channel, int blksize) { struct unix_private_data *data; - errcode_t retval; + errcode_t retval = 0; EXT2_CHECK_MAGIC(channel, EXT2_ET_MAGIC_IO_CHANNEL); data = (struct unix_private_data *) channel->private_data; EXT2_CHECK_MAGIC(data, EXT2_ET_MAGIC_UNIX_IO_CHANNEL); if (channel->block_size != blksize) { + mutex_lock(data, CACHE_MTX); + mutex_lock(data, BOUNCE_MTX); #ifndef NO_IO_CACHE - if ((retval = flush_cached_blocks(channel, data, 0))) + if ((retval = flush_cached_blocks(channel, data, FLUSH_NOLOCK))){ + mutex_unlock(data, BOUNCE_MTX); + mutex_unlock(data, CACHE_MTX); return retval; + } #endif channel->block_size = blksize; free_cache(data); - if ((retval = alloc_cache(channel, data))) - return retval; + retval = alloc_cache(channel, data); + mutex_unlock(data, BOUNCE_MTX); + mutex_unlock(data, CACHE_MTX); } - return 0; + return retval; } - static errcode_t unix_read_blk64(io_channel channel, unsigned long long block, int count, void *buf) { struct unix_private_data *data; - struct unix_cache *cache, *reuse[READ_DIRECT_SIZE]; + struct unix_cache *cache; errcode_t retval; char *cp; int i, j; @@ -642,6 +1039,8 @@ static errcode_t unix_read_blk64(io_channel channel, unsigned long long block, #ifdef NO_IO_CACHE return raw_read_blk(channel, data, block, count, buf); #else + if (data->flags & IO_FLAG_NOCACHE) + return raw_read_blk(channel, data, block, count, buf); /* * If we're doing an odd-sized read or a very large read, * flush out the cache and then do a direct read. @@ -653,9 +1052,10 @@ static errcode_t unix_read_blk64(io_channel channel, unsigned long long block, } cp = buf; + mutex_lock(data, CACHE_MTX); while (count > 0) { /* If it's in the cache, use it! */ - if ((cache = find_cached_block(data, block, &reuse[0]))) { + if ((cache = find_cached_block(data, block, NULL))) { #ifdef DEBUG printf("Using cached block %lu\n", block); #endif @@ -665,45 +1065,60 @@ static errcode_t unix_read_blk64(io_channel channel, unsigned long long block, cp += channel->block_size; continue; } - if (count == 1) { - /* - * Special case where we read directly into the - * cache buffer; important in the O_DIRECT case - */ - cache = reuse[0]; - reuse_cache(channel, data, cache, block); - if ((retval = raw_read_blk(channel, data, block, 1, - cache->buf))) { - cache->in_use = 0; - return retval; - } - memcpy(cp, cache->buf, channel->block_size); - return 0; - } /* * Find the number of uncached blocks so we can do a * single read request */ for (i=1; i < count; i++) - if (find_cached_block(data, block+i, &reuse[i])) + if (find_cached_block(data, block+i, NULL)) break; #ifdef DEBUG printf("Reading %d blocks starting at %lu\n", i, block); #endif + mutex_unlock(data, CACHE_MTX); if ((retval = raw_read_blk(channel, data, block, i, cp))) return retval; + mutex_lock(data, CACHE_MTX); /* Save the results in the cache */ for (j=0; j < i; j++) { + if (!find_cached_block(data, block, &cache)) { + retval = reuse_cache(channel, data, + cache, block); + if (retval) + goto call_write_handler; + memcpy(cache->buf, cp, channel->block_size); + } count--; - cache = reuse[j]; - reuse_cache(channel, data, cache, block++); - memcpy(cache->buf, cp, channel->block_size); + block++; cp += channel->block_size; } } + mutex_unlock(data, CACHE_MTX); return 0; + +call_write_handler: + if (cache->write_err && channel->write_error) { + char *err_buf = NULL; + unsigned long long err_block = cache->block; + + cache->dirty = 0; + cache->in_use = 0; + cache->write_err = 0; + if (io_channel_alloc_buf(channel, 0, &err_buf)) + err_buf = NULL; + else + memcpy(err_buf, cache->buf, channel->block_size); + mutex_unlock(data, CACHE_MTX); + (channel->write_error)(channel, err_block, 1, err_buf, + channel->block_size, -1, + retval); + if (err_buf) + ext2fs_free_mem(&err_buf); + } else + mutex_unlock(data, CACHE_MTX); + return retval; #endif /* NO_IO_CACHE */ } @@ -727,16 +1142,19 @@ static errcode_t unix_write_blk64(io_channel channel, unsigned long long block, EXT2_CHECK_MAGIC(data, EXT2_ET_MAGIC_UNIX_IO_CHANNEL); #ifdef NO_IO_CACHE - return raw_write_blk(channel, data, block, count, buf); + return raw_write_blk(channel, data, block, count, buf, 0); #else + if (data->flags & IO_FLAG_NOCACHE) + return raw_write_blk(channel, data, block, count, buf, 0); /* * If we're doing an odd-sized write or a very large write, * flush out the cache completely and then do a direct write. */ if (count < 0 || count > WRITE_DIRECT_SIZE) { - if ((retval = flush_cached_blocks(channel, data, 1))) + if ((retval = flush_cached_blocks(channel, data, + FLUSH_INVALIDATE))) return retval; - return raw_write_blk(channel, data, block, count, buf); + return raw_write_blk(channel, data, block, count, buf, 0); } /* @@ -746,25 +1164,72 @@ static errcode_t unix_write_blk64(io_channel channel, unsigned long long block, */ writethrough = channel->flags & CHANNEL_FLAGS_WRITETHROUGH; if (writethrough) - retval = raw_write_blk(channel, data, block, count, buf); + retval = raw_write_blk(channel, data, block, count, buf, 0); cp = buf; + mutex_lock(data, CACHE_MTX); while (count > 0) { cache = find_cached_block(data, block, &reuse); if (!cache) { + errcode_t err; + cache = reuse; - reuse_cache(channel, data, cache, block); + err = reuse_cache(channel, data, cache, block); + if (err) + goto call_write_handler; } - memcpy(cache->buf, cp, channel->block_size); + if (cache->buf != cp) + memcpy(cache->buf, cp, channel->block_size); cache->dirty = !writethrough; count--; block++; cp += channel->block_size; } + mutex_unlock(data, CACHE_MTX); + return retval; + +call_write_handler: + if (cache->write_err && channel->write_error) { + char *err_buf = NULL; + unsigned long long err_block = cache->block; + + cache->dirty = 0; + cache->in_use = 0; + cache->write_err = 0; + if (io_channel_alloc_buf(channel, 0, &err_buf)) + err_buf = NULL; + else + memcpy(err_buf, cache->buf, channel->block_size); + mutex_unlock(data, CACHE_MTX); + (channel->write_error)(channel, err_block, 1, err_buf, + channel->block_size, -1, + retval); + if (err_buf) + ext2fs_free_mem(&err_buf); + } else + mutex_unlock(data, CACHE_MTX); return retval; #endif /* NO_IO_CACHE */ } +static errcode_t unix_cache_readahead(io_channel channel, + unsigned long long block, + unsigned long long count) +{ +#ifdef POSIX_FADV_WILLNEED + struct unix_private_data *data; + + data = (struct unix_private_data *)channel->private_data; + EXT2_CHECK_MAGIC(data, EXT2_ET_MAGIC_UNIX_IO_CHANNEL); + return posix_fadvise(data->dev, + (ext2_loff_t)block * channel->block_size + data->offset, + (ext2_loff_t)count * channel->block_size, + POSIX_FADV_WILLNEED); +#else + return EXT2_ET_OP_NOT_SUPPORTED; +#endif +} + static errcode_t unix_write_blk(io_channel channel, unsigned long block, int count, const void *buf) { @@ -782,7 +1247,7 @@ static errcode_t unix_write_byte(io_channel channel, unsigned long offset, data = (struct unix_private_data *) channel->private_data; EXT2_CHECK_MAGIC(data, EXT2_ET_MAGIC_UNIX_IO_CHANNEL); - if (data->align != 0) { + if (channel->align != 0) { #ifdef ALIGN_DEBUG printf("unix_write_byte: O_DIRECT fallback\n"); #endif @@ -793,7 +1258,7 @@ static errcode_t unix_write_byte(io_channel channel, unsigned long offset, /* * Flush out the cache completely */ - if ((retval = flush_cached_blocks(channel, data, 1))) + if ((retval = flush_cached_blocks(channel, data, FLUSH_INVALIDATE))) return retval; #endif @@ -801,6 +1266,8 @@ static errcode_t unix_write_byte(io_channel channel, unsigned long offset, return errno; actual = write(data->dev, buf, size); + if (actual < 0) + return errno; if (actual != size) return EXT2_ET_SHORT_WRITE; @@ -822,7 +1289,10 @@ static errcode_t unix_flush(io_channel channel) #ifndef NO_IO_CACHE retval = flush_cached_blocks(channel, data, 0); #endif - fsync(data->dev); +#ifdef HAVE_FSYNC + if (!retval && fsync(data->dev) != 0) + return errno; +#endif return retval; } @@ -831,6 +1301,7 @@ static errcode_t unix_set_option(io_channel channel, const char *option, { struct unix_private_data *data; unsigned long long tmp; + errcode_t retval; char *end; EXT2_CHECK_MAGIC(channel, EXT2_ET_MAGIC_IO_CHANNEL); @@ -849,33 +1320,192 @@ static errcode_t unix_set_option(io_channel channel, const char *option, return EXT2_ET_INVALID_ARGUMENT; return 0; } + if (!strcmp(option, "cache")) { + if (!arg) + return EXT2_ET_INVALID_ARGUMENT; + if (!strcmp(arg, "on")) { + data->flags &= ~IO_FLAG_NOCACHE; + return 0; + } + if (!strcmp(arg, "off")) { + retval = flush_cached_blocks(channel, data, 0); + data->flags |= IO_FLAG_NOCACHE; + return retval; + } + return EXT2_ET_INVALID_ARGUMENT; + } return EXT2_ET_INVALID_ARGUMENT; } #if defined(__linux__) && !defined(BLKDISCARD) -#define BLKDISCARD _IO(0x12,119) +#define BLKDISCARD _IO(0x12,119) #endif static errcode_t unix_discard(io_channel channel, unsigned long long block, unsigned long long count) { -#ifdef BLKDISCARD struct unix_private_data *data; - __uint64_t range[2]; int ret; EXT2_CHECK_MAGIC(channel, EXT2_ET_MAGIC_IO_CHANNEL); data = (struct unix_private_data *) channel->private_data; EXT2_CHECK_MAGIC(data, EXT2_ET_MAGIC_UNIX_IO_CHANNEL); - range[0] = (__uint64_t)(block) * channel->block_size; - range[1] = (__uint64_t)(count) * channel->block_size; + if (channel->flags & CHANNEL_FLAGS_BLOCK_DEVICE) { +#ifdef BLKDISCARD + __u64 range[2]; + + range[0] = (__u64)(block) * channel->block_size + data->offset; + range[1] = (__u64)(count) * channel->block_size; - ret = ioctl(data->dev, BLKDISCARD, &range); - if (ret < 0) + ret = ioctl(data->dev, BLKDISCARD, &range); +#else + goto unimplemented; +#endif + } else { +#if defined(HAVE_FALLOCATE) && defined(FALLOC_FL_PUNCH_HOLE) + /* + * If we are not on block device, try to use punch hole + * to reclaim free space. + */ + ret = fallocate(data->dev, + FALLOC_FL_PUNCH_HOLE | FALLOC_FL_KEEP_SIZE, + (off_t)(block) * channel->block_size + data->offset, + (off_t)(count) * channel->block_size); +#else + goto unimplemented; +#endif + } + if (ret < 0) { + if (errno == EOPNOTSUPP) + goto unimplemented; return errno; + } return 0; -#else +unimplemented: return EXT2_ET_UNIMPLEMENTED; +} + +/* + * If we know about ZERO_RANGE, try that before we try PUNCH_HOLE because + * ZERO_RANGE doesn't unmap preallocated blocks. We prefer fallocate because + * it always invalidates page cache, and libext2fs requires that reads after + * ZERO_RANGE return zeroes. + */ +static int __unix_zeroout(int fd, off_t offset, off_t len) +{ + int ret = -1; + +#if defined(HAVE_FALLOCATE) && defined(FALLOC_FL_ZERO_RANGE) + ret = fallocate(fd, FALLOC_FL_ZERO_RANGE, offset, len); + if (ret == 0) + return 0; #endif +#if defined(HAVE_FALLOCATE) && defined(FALLOC_FL_PUNCH_HOLE) && defined(FALLOC_FL_KEEP_SIZE) + ret = fallocate(fd, FALLOC_FL_PUNCH_HOLE | FALLOC_FL_KEEP_SIZE, + offset, len); + if (ret == 0) + return 0; +#endif + errno = EOPNOTSUPP; + return ret; } + +/* parameters might not be used if OS doesn't support zeroout */ +#if __GNUC_PREREQ (4, 6) +#pragma GCC diagnostic push +#pragma GCC diagnostic ignored "-Wunused-parameter" +#endif +static errcode_t unix_zeroout(io_channel channel, unsigned long long block, + unsigned long long count) +{ + struct unix_private_data *data; + int ret; + + EXT2_CHECK_MAGIC(channel, EXT2_ET_MAGIC_IO_CHANNEL); + data = (struct unix_private_data *) channel->private_data; + EXT2_CHECK_MAGIC(data, EXT2_ET_MAGIC_UNIX_IO_CHANNEL); + + if (safe_getenv("UNIX_IO_NOZEROOUT")) + goto unimplemented; + + if (!(channel->flags & CHANNEL_FLAGS_BLOCK_DEVICE)) { + /* Regular file, try to use truncate/punch/zero. */ + struct stat statbuf; + + if (count == 0) + return 0; + /* + * If we're trying to zero a range past the end of the file, + * extend the file size, then truncate everything. + */ + ret = fstat(data->dev, &statbuf); + if (ret) + goto err; + if ((unsigned long long) statbuf.st_size < + (block + count) * channel->block_size + data->offset) { + ret = ftruncate(data->dev, + (block + count) * channel->block_size + data->offset); + if (ret) + goto err; + } + } + + ret = __unix_zeroout(data->dev, + (off_t)(block) * channel->block_size + data->offset, + (off_t)(count) * channel->block_size); +err: + if (ret < 0) { + if (errno == EOPNOTSUPP) + goto unimplemented; + return errno; + } + return 0; +unimplemented: + return EXT2_ET_UNIMPLEMENTED; +} +#if __GNUC_PREREQ (4, 6) +#pragma GCC diagnostic pop +#endif + +static struct struct_io_manager struct_unix_manager = { + .magic = EXT2_ET_MAGIC_IO_MANAGER, + .name = "Unix I/O Manager", + .open = unix_open, + .close = unix_close, + .set_blksize = unix_set_blksize, + .read_blk = unix_read_blk, + .write_blk = unix_write_blk, + .flush = unix_flush, + .write_byte = unix_write_byte, + .set_option = unix_set_option, + .get_stats = unix_get_stats, + .read_blk64 = unix_read_blk64, + .write_blk64 = unix_write_blk64, + .discard = unix_discard, + .cache_readahead = unix_cache_readahead, + .zeroout = unix_zeroout, +}; + +io_manager unix_io_manager = &struct_unix_manager; + +static struct struct_io_manager struct_unixfd_manager = { + .magic = EXT2_ET_MAGIC_IO_MANAGER, + .name = "Unix fd I/O Manager", + .open = unixfd_open, + .close = unix_close, + .set_blksize = unix_set_blksize, + .read_blk = unix_read_blk, + .write_blk = unix_write_blk, + .flush = unix_flush, + .write_byte = unix_write_byte, + .set_option = unix_set_option, + .get_stats = unix_get_stats, + .read_blk64 = unix_read_blk64, + .write_blk64 = unix_write_blk64, + .discard = unix_discard, + .cache_readahead = unix_cache_readahead, + .zeroout = unix_zeroout, +}; + +io_manager unixfd_io_manager = &struct_unixfd_manager;