From 105cdfe5a04608da3858e8b57e4032507bbac5fa Mon Sep 17 00:00:00 2001 From: Adrien Schildknecht Date: Thu, 10 Nov 2016 22:34:32 -0800 Subject: [PATCH] AOSP: libext2fs: android sparse io manager Add a new io manager to directly output sparse images. Test: mke2fs [...] -E sparse_file; simg2img system.img system.ext4; e2fsck system.ext4 Change-Id: I41cf8c1b33d359be4f104e03fb4041863214843c From AOSP commit: f9e0f1d4a7cf32c4091eee7d2a1676cac1d17438 Signed-off-by: Theodore Ts'o --- lib/ext2fs/Android.mk | 6 +- lib/ext2fs/Makefile.in | 8 + lib/ext2fs/ext2_io.h | 6 + lib/ext2fs/sparse_io.c | 499 +++++++++++++++++++++++++++++++++++++++++++++++++ misc/mke2fs.c | 23 ++- util/android_config.h | 2 + 6 files changed, 540 insertions(+), 4 deletions(-) create mode 100644 lib/ext2fs/sparse_io.c diff --git a/lib/ext2fs/Android.mk b/lib/ext2fs/Android.mk index 356232f..89b1db9 100644 --- a/lib/ext2fs/Android.mk +++ b/lib/ext2fs/Android.mk @@ -76,6 +76,7 @@ libext2fs_src_files := \ symlink.c \ undo_io.c \ unix_io.c \ + sparse_io.c \ unlink.c \ valid_blk.c \ version.c @@ -107,7 +108,7 @@ include $(CLEAR_VARS) LOCAL_SRC_FILES := $(libext2fs_src_files) LOCAL_SYSTEM_SHARED_LIBRARIES := $(libext2fs_system_shared_libraries) -LOCAL_SHARED_LIBRARIES := $(libext2fs_shared_libraries) +LOCAL_SHARED_LIBRARIES := $(libext2fs_shared_libraries) libsparse libz LOCAL_C_INCLUDES := $(libext2fs_c_includes) LOCAL_EXPORT_C_INCLUDE_DIRS := $(libext2fs_c_includes) LOCAL_CFLAGS := $(libext2fs_cflags) @@ -119,7 +120,7 @@ include $(BUILD_SHARED_LIBRARY) include $(CLEAR_VARS) LOCAL_SRC_FILES := $(libext2fs_src_files) -LOCAL_STATIC_LIBRARIES := $(libext2fs_static_libraries) $(libext2fs_system_static_libraries) +LOCAL_STATIC_LIBRARIES := $(libext2fs_static_libraries) $(libext2fs_system_static_libraries) libsparse_static libz LOCAL_C_INCLUDES := $(libext2fs_c_includes) LOCAL_EXPORT_C_INCLUDE_DIRS := $(libext2fs_c_includes) LOCAL_CFLAGS := $(libext2fs_cflags) $(libext2fs_cflags_linux) @@ -132,6 +133,7 @@ include $(CLEAR_VARS) LOCAL_SRC_FILES := $(libext2fs_src_files) LOCAL_SHARED_LIBRARIES := $(addsuffix -host, $(libext2fs_shared_libraries)) +LOCAL_STATIC_LIBRARIES := libsparse_host libz LOCAL_C_INCLUDES := $(libext2fs_c_includes) LOCAL_EXPORT_C_INCLUDE_DIRS := $(libext2fs_c_includes) LOCAL_CFLAGS := $(libext2fs_cflags) diff --git a/lib/ext2fs/Makefile.in b/lib/ext2fs/Makefile.in index b23bf88..b6b0f85 100644 --- a/lib/ext2fs/Makefile.in +++ b/lib/ext2fs/Makefile.in @@ -125,6 +125,7 @@ OBJS= $(DEBUGFS_LIB_OBJS) $(RESIZE_LIB_OBJS) $(E2IMAGE_LIB_OBJS) \ $(TDB_OBJ) \ undo_io.o \ unix_io.o \ + sparse_io.o \ unlink.o \ valid_blk.o \ version.o \ @@ -212,6 +213,7 @@ SRCS= ext2_err.c \ $(srcdir)/tst_iscan.c \ $(srcdir)/undo_io.c \ $(srcdir)/unix_io.c \ + $(srcdir)/sparse_io.c \ $(srcdir)/unlink.c \ $(srcdir)/valid_blk.c \ $(srcdir)/version.c \ @@ -1108,6 +1110,12 @@ unix_io.o: $(srcdir)/unix_io.c $(top_builddir)/lib/config.h \ $(srcdir)/ext2_fs.h $(srcdir)/ext3_extents.h $(top_srcdir)/lib/et/com_err.h \ $(srcdir)/ext2_io.h $(top_builddir)/lib/ext2fs/ext2_err.h \ $(srcdir)/ext2_ext_attr.h $(srcdir)/bitops.h +sparse_io.o: $(srcdir)/sparse_io.c $(top_builddir)/lib/config.h \ + $(top_builddir)/lib/dirpaths.h $(srcdir)/ext2_fs.h \ + $(top_builddir)/lib/ext2fs/ext2_types.h $(srcdir)/ext2fs.h \ + $(srcdir)/ext2_fs.h $(srcdir)/ext3_extents.h $(top_srcdir)/lib/et/com_err.h \ + $(srcdir)/ext2_io.h $(top_builddir)/lib/ext2fs/ext2_err.h \ + $(srcdir)/ext2_ext_attr.h $(srcdir)/bitops.h unlink.o: $(srcdir)/unlink.c $(top_builddir)/lib/config.h \ $(top_builddir)/lib/dirpaths.h $(srcdir)/ext2_fs.h \ $(top_builddir)/lib/ext2fs/ext2_types.h $(srcdir)/ext2fs.h \ diff --git a/lib/ext2fs/ext2_io.h b/lib/ext2fs/ext2_io.h index 6b7e977..5540900 100644 --- a/lib/ext2fs/ext2_io.h +++ b/lib/ext2fs/ext2_io.h @@ -12,6 +12,8 @@ #ifndef _EXT2FS_EXT2_IO_H #define _EXT2FS_EXT2_IO_H +#include + /* * ext2_loff_t is defined here since unix_io.c needs it. */ @@ -141,6 +143,10 @@ extern errcode_t io_channel_cache_readahead(io_channel io, extern io_manager unix_io_manager; extern io_manager unixfd_io_manager; +/* sparse_io.c */ +extern io_manager sparse_io_manager; +extern io_manager sparsefd_io_manager; + /* undo_io.c */ extern io_manager undo_io_manager; extern errcode_t set_undo_io_backing_manager(io_manager manager); diff --git a/lib/ext2fs/sparse_io.c b/lib/ext2fs/sparse_io.c new file mode 100644 index 0000000..77bc421 --- /dev/null +++ b/lib/ext2fs/sparse_io.c @@ -0,0 +1,499 @@ +#include "config.h" +#include +#include +#include +#include +#include +#include "ext2_fs.h" +#include "ext2fs.h" + +#if !defined(ENABLE_LIBSPARSE) +static errcode_t sparse_open(const char *name EXT2FS_ATTR((unused)), + int flags EXT2FS_ATTR((unused)), + io_channel *channel EXT2FS_ATTR((unused))) +{ + return EXT2_ET_UNIMPLEMENTED; +} +static errcode_t sparse_close(io_channel channel EXT2FS_ATTR((unused))) +{ + return EXT2_ET_UNIMPLEMENTED; +} +static struct struct_io_manager struct_sparse_manager = { + .magic = EXT2_ET_MAGIC_IO_MANAGER, + .name = "Android sparse I/O Manager", + .open = sparse_open, + .close = sparse_close, +}; +static struct struct_io_manager struct_sparsefd_manager = { + .magic = EXT2_ET_MAGIC_IO_MANAGER, + .name = "Android sparse fd I/O Manager", + .open = sparse_open, + .close = sparse_close, +}; +#else +#include + +struct sparse_map { + int fd; + char **blocks; + int block_size; + uint64_t blocks_count; + char *file; + struct sparse_file *sparse_file; + io_channel channel; +}; + +struct sparse_io_params { + int fd; + char *file; + uint64_t blocks_count; + unsigned int block_size; +}; + +static errcode_t sparse_write_blk(io_channel channel, unsigned long block, + int count, const void *buf); + +static void free_sparse_blocks(struct sparse_map *sm) +{ + uint64_t i; + + for (i = 0; i < sm->blocks_count; ++i) + free(sm->blocks[i]); + free(sm->blocks); + sm->blocks = NULL; +} + +static int sparse_import_segment(void *priv, const void *data, int len, + unsigned int block, unsigned int nr_blocks) +{ + struct sparse_map *sm = priv; + + /* Ignore chunk headers, only write the data */ + if (!nr_blocks || len % sm->block_size) + return 0; + + return sparse_write_blk(sm->channel, block, nr_blocks, data); +} + +static errcode_t io_manager_import_sparse(struct sparse_io_params *params, + struct sparse_map *sm, io_channel io) +{ + int fd; + errcode_t retval; + struct sparse_file *sparse_file; + + if (params->fd < 0) { + fd = open(params->file, O_RDONLY); + if (fd < 0) { + retval = -1; + goto err_open; + } + } else + fd = params->fd; + sparse_file = sparse_file_import(fd, false, false); + if (!sparse_file) { + retval = -1; + goto err_sparse; + } + + sm->block_size = sparse_file_block_size(sparse_file); + sm->blocks_count = (sparse_file_len(sparse_file, 0, 0) - 1) + / sm->block_size + 1; + sm->blocks = calloc(sm->blocks_count, sizeof(char*)); + if (!sm->blocks) { + retval = -1; + goto err_alloc; + } + io->block_size = sm->block_size; + + retval = sparse_file_foreach_chunk(sparse_file, true, false, + sparse_import_segment, sm); + + if (retval) + free_sparse_blocks(sm); +err_alloc: + sparse_file_destroy(sparse_file); +err_sparse: + close(fd); +err_open: + return retval; +} + +static errcode_t io_manager_configure(struct sparse_io_params *params, + int flags, io_channel io) +{ + errcode_t retval; + uint64_t img_size; + struct sparse_map *sm = calloc(1, sizeof(*sm)); + if (!sm) + return EXT2_ET_NO_MEMORY; + + sm->file = params->file; + sm->channel = io; + io->private_data = sm; + retval = io_manager_import_sparse(params, sm, io); + if (retval) { + if (!params->block_size || !params->blocks_count) { + retval = -EINVAL; + goto err_params; + } + sm->block_size = params->block_size; + sm->blocks_count = params->blocks_count; + sm->blocks = calloc(params->blocks_count, sizeof(void*)); + if (!sm->blocks) { + retval = EXT2_ET_NO_MEMORY; + goto err_alloc; + } + } + io->block_size = sm->block_size; + img_size = (uint64_t)sm->block_size * sm->blocks_count; + + if (flags & IO_FLAG_RW) { + sm->sparse_file = sparse_file_new(sm->block_size, img_size); + if (!sm->sparse_file) { + retval = EXT2_ET_NO_MEMORY; + goto err_alloc; + } + if (params->fd < 0) { + sm->fd = open(params->file, O_CREAT | O_RDWR | O_TRUNC, + 0644); + if (sm->fd < 0) { + retval = errno; + goto err_open; + } + } else + sm->fd = params->fd; + } else { + sm->fd = -1; + sm->sparse_file = NULL; + } + return 0; + +err_open: + sparse_file_destroy(sm->sparse_file); +err_alloc: + free_sparse_blocks(sm); +err_params: + free(sm); + return retval; +} + +static errcode_t sparse_open_channel(struct sparse_io_params *sparse_params, + int flags, io_channel *channel) +{ + io_channel io; + + io = calloc(1, sizeof(struct struct_io_channel)); + io->magic = EXT2_ET_MAGIC_IO_CHANNEL; + io->block_size = 0; + io->refcount = 1; + *channel = io; + return io_manager_configure(sparse_params, flags, io); +} + +static errcode_t read_sparse_argv(const char *name, bool is_fd, + struct sparse_io_params *sparse_params) +{ + int ret; + sparse_params->fd = -1; + sparse_params->file = NULL; + sparse_params->block_size = 0; + sparse_params->blocks_count = 0; + + if (is_fd) { + ret = sscanf(name, "%d:%llu:%u", &sparse_params->fd, + (unsigned long long *)&sparse_params->blocks_count, + &sparse_params->block_size); + } else { + ret = sscanf(name, "%m[^:]:%llu%*[:]%u", &sparse_params->file, + (unsigned long long *)&sparse_params->blocks_count, + &sparse_params->block_size); + } + + if (ret < 1) { + free(sparse_params->file); + return -EINVAL; + } + return 0; +} + +static errcode_t sparse_open(const char *name, int flags, io_channel *channel) +{ + errcode_t retval; + struct sparse_io_params sparse_params; + + retval = read_sparse_argv(name, false, &sparse_params); + if (retval) + return EXT2_ET_BAD_DEVICE_NAME; + + retval = sparse_open_channel(&sparse_params, flags, channel); + if (retval) + return retval; + (*channel)->manager = sparse_io_manager; + + return retval; +} + +static errcode_t sparsefd_open(const char *name, int flags, io_channel *channel) +{ + errcode_t retval; + struct sparse_io_params sparse_params; + + retval = read_sparse_argv(name, true, &sparse_params); + if (retval) + return EXT2_ET_BAD_DEVICE_NAME; + + retval = sparse_open_channel(&sparse_params, flags, channel); + if (retval) + return retval; + (*channel)->manager = sparsefd_io_manager; + + return retval; +} + +static errcode_t sparse_close_channel(io_channel channel) +{ + uint64_t i; + errcode_t retval = 0; + struct sparse_map *sm = channel->private_data; + + if (sm->sparse_file) { + for (i = 0; i < sm->blocks_count; ++i) { + if (!sm->blocks[i]) + continue; + retval = sparse_file_add_data(sm->sparse_file, + sm->blocks[i], + sm->block_size, i); + if (retval) + goto ret; + } + retval = sparse_file_write(sm->sparse_file, sm->fd, + /*gzip*/0, /*sparse*/1, /*crc*/0); + } + +ret: + if (sm->sparse_file) + sparse_file_destroy(sm->sparse_file); + free_sparse_blocks(sm); + free(sm->file); + free(sm); + free(channel); + return retval; +} + +static errcode_t sparse_close(io_channel channel) +{ + errcode_t retval; + struct sparse_map *sm = channel->private_data; + int fd = sm->fd; + + retval = sparse_close_channel(channel); + if (fd >= 0) + close(fd); + + return retval; +} + +static errcode_t sparse_set_blksize(io_channel channel, int blksize) +{ + channel->block_size = blksize; + return 0; +} + +static blk64_t block_to_sparse_block(blk64_t block, blk64_t *offset, + io_channel channel, struct sparse_map *sm) +{ + int ratio; + blk64_t ret = block; + + ratio = sm->block_size / channel->block_size; + ret /= ratio; + *offset = (block % ratio) * channel->block_size; + + return ret; +} + +static errcode_t check_block_size(io_channel channel, struct sparse_map *sm) +{ + if (sm->block_size >= channel->block_size) + return 0; + return EXT2_ET_UNEXPECTED_BLOCK_SIZE; +} + +static errcode_t sparse_read_blk64(io_channel channel, blk64_t block, + int count, void *buf) +{ + int i; + char *out = buf; + blk64_t offset = 0, cur_block; + struct sparse_map *sm = channel->private_data; + + if (check_block_size(channel, sm)) + return EXT2_ET_UNEXPECTED_BLOCK_SIZE; + + if (count < 0) { //partial read + count = -count; + cur_block = block_to_sparse_block(block, &offset, channel, sm); + if (sm->blocks[cur_block]) + memcpy(out, (sm->blocks[cur_block]) + offset, count); + else + memset(out, 0, count); + } else { + for (i = 0; i < count; ++i) { + cur_block = block_to_sparse_block(block + i, &offset, + channel, sm); + if (sm->blocks[cur_block]) + memcpy(out + (i * channel->block_size), + sm->blocks[cur_block] + offset, + channel->block_size); + else if (sm->blocks) + memset(out + (i * channel->block_size), 0, + channel->block_size); + } + } + return 0; +} + +static errcode_t sparse_read_blk(io_channel channel, unsigned long block, + int count, void *buf) +{ + return sparse_read_blk64(channel, block, count, buf); +} + +static errcode_t sparse_write_blk64(io_channel channel, blk64_t block, + int count, const void *buf) +{ + int i; + blk64_t offset = 0, cur_block; + const char *in = buf; + struct sparse_map *sm = channel->private_data; + + if (check_block_size(channel, sm)) + return EXT2_ET_UNEXPECTED_BLOCK_SIZE; + + if (count < 0) { //partial write + count = -count; + cur_block = block_to_sparse_block(block, &offset, channel, + sm); + if (!sm->blocks[cur_block]) { + sm->blocks[cur_block] = calloc(1, sm->block_size); + if (!sm->blocks[cur_block]) + return EXT2_ET_NO_MEMORY; + } + memcpy(sm->blocks[cur_block] + offset, in, count); + } else { + for (i = 0; i < count; ++i) { + if (block + i >= sm->blocks_count) + return 0; + cur_block = block_to_sparse_block(block + i, &offset, + channel, sm); + if (!sm->blocks[cur_block]) { + sm->blocks[cur_block] = + calloc(1, sm->block_size); + if (!sm->blocks[cur_block]) + return EXT2_ET_NO_MEMORY; + } + memcpy(sm->blocks[cur_block] + offset, + in + (i * channel->block_size), + channel->block_size); + } + } + return 0; +} + +static errcode_t sparse_write_blk(io_channel channel, unsigned long block, + int count, const void *buf) +{ + return sparse_write_blk64(channel, block, count, buf); +} + +static errcode_t sparse_discard(io_channel channel __attribute__((unused)), + blk64_t blk, unsigned long long count) +{ + blk64_t cur_block, offset; + struct sparse_map *sm = channel->private_data; + + if (check_block_size(channel, sm)) + return EXT2_ET_UNEXPECTED_BLOCK_SIZE; + + for (unsigned long long i = 0; i < count; ++i) { + if (blk + i >= sm->blocks_count) + return 0; + cur_block = block_to_sparse_block(blk + i, &offset, channel, + sm); + if (!sm->blocks[cur_block]) + continue; + free(sm->blocks[cur_block]); + sm->blocks[cur_block] = NULL; + } + return 0; +} + +static errcode_t sparse_zeroout(io_channel channel, blk64_t blk, + unsigned long long count) +{ + return sparse_discard(channel, blk, count); +} + +static errcode_t sparse_flush(io_channel channel __attribute__((unused))) +{ + return 0; +} + +static errcode_t sparse_set_option(io_channel channel __attribute__((unused)), + const char *option __attribute__((unused)), + const char *arg __attribute__((unused))) +{ + return 0; +} + +static errcode_t sparse_cache_readahead( + io_channel channel __attribute__((unused)), + blk64_t blk __attribute__((unused)), + unsigned long long count __attribute__((unused))) +{ + return 0; +} + +static struct struct_io_manager struct_sparse_manager = { + .magic = EXT2_ET_MAGIC_IO_MANAGER, + .name = "Android sparse I/O Manager", + .open = sparse_open, + .close = sparse_close, + .set_blksize = sparse_set_blksize, + .read_blk = sparse_read_blk, + .write_blk = sparse_write_blk, + .flush = sparse_flush, + .write_byte = NULL, + .set_option = sparse_set_option, + .get_stats = NULL, + .read_blk64 = sparse_read_blk64, + .write_blk64 = sparse_write_blk64, + .discard = sparse_discard, + .cache_readahead = sparse_cache_readahead, + .zeroout = sparse_zeroout, +}; + +static struct struct_io_manager struct_sparsefd_manager = { + .magic = EXT2_ET_MAGIC_IO_MANAGER, + .name = "Android sparse fd I/O Manager", + .open = sparsefd_open, + .close = sparse_close, + .set_blksize = sparse_set_blksize, + .read_blk = sparse_read_blk, + .write_blk = sparse_write_blk, + .flush = sparse_flush, + .write_byte = NULL, + .set_option = sparse_set_option, + .get_stats = NULL, + .read_blk64 = sparse_read_blk64, + .write_blk64 = sparse_write_blk64, + .discard = sparse_discard, + .cache_readahead = sparse_cache_readahead, + .zeroout = sparse_zeroout, +}; + +#endif + +io_manager sparse_io_manager = &struct_sparse_manager; +io_manager sparsefd_io_manager = &struct_sparsefd_manager; diff --git a/misc/mke2fs.c b/misc/mke2fs.c index b157006..a7bab52 100644 --- a/misc/mke2fs.c +++ b/misc/mke2fs.c @@ -113,6 +113,9 @@ char **fs_types; const char *src_root_dir; /* Copy files from the specified directory */ static char *undo_file; +static int android_sparse_file; /* -E android_sparse */ +static char *android_sparse_params; + static profile_t profile; static int sys_page_size = 4096; @@ -553,7 +556,7 @@ static void zap_sector(ext2_filsys fs, int sect, int nsect) int retval; unsigned int *magic; - buf = malloc(512*nsect); + buf = calloc(512, nsect); if (!buf) { printf(_("Out of memory erasing sectors %d-%d\n"), sect, sect + nsect - 1); @@ -1026,6 +1029,8 @@ static void parse_extended_opts(struct ext2_super_block *param, badopt = token; continue; } + } else if (!strcmp(token, "android_sparse")) { + android_sparse_file = 1; } else { r_usage++; badopt = token; @@ -2828,7 +2833,21 @@ int main (int argc, char *argv[]) */ if (!quiet) flags |= EXT2_FLAG_PRINT_PROGRESS; - retval = ext2fs_initialize(device_name, flags, &fs_param, io_ptr, &fs); + if (android_sparse_file) { + android_sparse_params = malloc(PATH_MAX + 32); + if (!android_sparse_params) { + com_err(program_name, ENOMEM, "%s", + _("in malloc for android_sparse_params")); + exit(1); + } + snprintf(android_sparse_params, PATH_MAX + 32, "%s:%u:%u", + device_name, fs_param.s_blocks_count, + 1024 << fs_param.s_log_block_size); + retval = ext2fs_initialize(android_sparse_params, flags, + &fs_param, sparse_io_manager, &fs); + } else + retval = ext2fs_initialize(device_name, flags, &fs_param, + io_ptr, &fs); if (retval) { com_err(device_name, retval, "%s", _("while setting up superblock")); diff --git a/util/android_config.h b/util/android_config.h index b3fd304..af370ff 100644 --- a/util/android_config.h +++ b/util/android_config.h @@ -6,6 +6,8 @@ #define ROOT_SYSCONFDIR "/etc" +#define ENABLE_LIBSPARSE 1 + #define DISABLE_BACKTRACE 1 #define HAVE_DIRENT_H 1 #define HAVE_ERRNO_H 1 -- 1.8.3.1