From: Rahul Deshmukh Date: Thu, 4 Mar 2010 01:03:21 +0000 (-0800) Subject: b=20492 allow llverdev/llverfs to handle media errors better X-Git-Tag: 1.10.0.38~5 X-Git-Url: https://git.whamcloud.com/?p=fs%2Flustre-release.git;a=commitdiff_plain;h=4d6c19b8604ca3fb5c28056f55ba7f4026d84d4a b=20492 allow llverdev/llverfs to handle media errors better i=adilger i=rread --- diff --git a/lustre/tests/conf-sanity.sh b/lustre/tests/conf-sanity.sh index d12d663..fbe96bb 100644 --- a/lustre/tests/conf-sanity.sh +++ b/lustre/tests/conf-sanity.sh @@ -2320,6 +2320,34 @@ if ! combined_mgs_mds ; then stop mgs fi +run_llverfs() +{ + local dir=$1 + local partial_arg="" + local size=$(df -B G $dir | tail -1 | awk '{print $2}' | sed 's/G//') # Gb + + # Run in partial (fast) mode if the size + # of a partition > 10 GB + [ $size -gt 10 ] && partial_arg="-p" + + llverfs $partial_arg $dir +} + +test_54a() { + do_rpc_nodes $(facet_host ost1) run_llverdev $(ostdevname 1) + [ $? -eq 0 ] || error "llverdev failed!" + reformat_and_config +} +run_test 54a "llverdev" + +test_54b() { + setup + run_llverfs $MOUNT + [ $? -eq 0 ] || error "llverfs failed!" + cleanup +} +run_test 54b "llverfs" + cleanup_gss equals_msg `basename $0`: test complete [ -f "$TESTSUITELOG" ] && cat $TESTSUITELOG && grep -q FAIL $TESTSUITELOG && exit 1 || true diff --git a/lustre/tests/test-framework.sh b/lustre/tests/test-framework.sh index 71aa529..36d6bb1 100644 --- a/lustre/tests/test-framework.sh +++ b/lustre/tests/test-framework.sh @@ -3915,3 +3915,17 @@ log_sub_test() { yml_log_sub_test $@ >> $YAML_LOG } +run_llverdev() +{ + local dev=$1 + local devname=$(basename $1) + local size=$(grep "$devname"$ /proc/partitions | awk '{print $3}') + size=$(($size / 1024 / 1024)) # Gb + + local partial_arg="" + # Run in partial (fast) mode if the size + # of a partition > 10 GB + [ $size -gt 10 ] && partial_arg="-p" + + llverdev --force $partial_arg $dev +} diff --git a/lustre/utils/llverdev.c b/lustre/utils/llverdev.c index 085b551..58641f2 100644 --- a/lustre/utils/llverdev.c +++ b/lustre/utils/llverdev.c @@ -26,7 +26,7 @@ * GPL HEADER END */ /* - * Copyright 2008 Sun Microsystems, Inc. All rights reserved + * Copyright 2009 Sun Microsystems, Inc. All rights reserved * Use is subject to license terms. */ /* @@ -41,14 +41,17 @@ * This tool have two working modes * 1. full mode * 2. fast mode - * The full mode is basic mode in which program writes the test pattern - * on entire disk. The test pattern (device offset and timestamp) is written - * at the beginning of each 4kB block. When the whole device is full then - * read operation is performed to verify that the test pattern is correct. - * In the fast mode the program writes data at the critical locations + * + * In full mode, the program writes a test pattern on the entire disk. + * The test pattern (device offset and timestamp) is written at the + * beginning of each 4kB block. When the whole device is full the read + * operation is performed to verify that the test pattern is correct. + * + * In fast mode, the program writes data at the critical locations * of the device such as start of the device, before and after multiple of 1GB * offset and at the end. - * A chunk buffer with default size of 1MB is used to write and read test + * + * A chunk buffer with default size of 1MB is used to write and read test * pattern in bulk. */ @@ -97,8 +100,9 @@ /* Structure for writting test pattern */ struct block_data { - long long bd_offset; - time_t bd_time; + unsigned long long bd_offset; + unsigned long long bd_time; + unsigned long long bd_inode; }; static char *progname; /* name by which this program was run. */ static unsigned verbose = 1; /* prints offset in kB, operation rate */ @@ -106,7 +110,7 @@ static int readoption; /* run test in read-only (verify) mode */ static int writeoption; /* run test in write_only mode */ const char *devname; /* name of device to be tested. */ static unsigned full = 1; /* flag to full check */ -static int fd; +static int error_count; /* number of IO errors hit during run */ static int isatty_flag; static struct option const longopts[] = @@ -122,6 +126,7 @@ static struct option const longopts[] = { "verbose", no_argument, 0, 'v' }, { "write", no_argument, 0, 'w' }, { "long", no_argument, 0, 'l' }, + { "full", no_argument, 0, 'l' }, { 0, 0, 0, 0} }; @@ -132,22 +137,22 @@ static struct option const longopts[] = void usage(int status) { if (status != 0) { - printf("\nUsage: %s [OPTION]... ...\n", - progname); - printf("Block device verification tool.\n" - "\t-t {seconds}, --timestamp, " - "set test time (default=current time())\n" - "\t-o {offset}, --offset, " - "offset in kB of start of test, default=0\n" - "\t-r, --read run test in verify mode\n" - "\t-w, --write run test in test-pattern mode, default=rw\n" - "\t-v, --verbose\n" - "\t-q, --quiet\n" - "\t-l, --long, full check of device\n" - "\t-p, --partial, for partial check (1GB steps)\n" - "\t-c, --chunksize, IO chunk size, default=1048576\n" - "\t-f, --force, force test to run without confirmation\n" - "\t-h, --help display this help and exit\n"); + printf("\nUsage: %s [OPTION]... ...\n", + progname); + printf("Block device verification tool.\n" + "\t-t {seconds}, --timestamp, " + "set test time (default=current time())\n" + "\t-o {offset}, --offset, " + "offset in kB of start of test, default=0\n" + "\t-r, --read, run in verify mode\n" + "\t-w, --write, run in test-pattern mode, default=rw\n" + "\t-v, --verbose\n" + "\t-q, --quiet\n" + "\t-l, --long, --full check of device\n" + "\t-p, --partial, for partial check (1GB steps)\n" + "\t-c {bytes}, --chunksize, IO size, default=1048576\n" + "\t-f, --force, force test to run without confirmation\n" + "\t-h, --help, display this help and exit\n"); } exit(status); } @@ -157,9 +162,10 @@ void usage(int status) */ static int open_dev(const char *devname, int mode) { + int fd; #ifdef HAVE_EXT2FS_EXT2FS_H - int mount_flags; - char mountpt[80] = ""; + int mount_flags; + char mountpt[80] = ""; if (ext2fs_check_mount_point(devname, &mount_flags, mountpt, sizeof(mountpt))) { @@ -178,10 +184,9 @@ static int open_dev(const char *devname, int mode) fprintf(stderr, "%s: Open failed: %s",progname,strerror(errno)); exit(3); } - return (fd); + return fd; } -#undef HAVE_BLKID_BLKID_H /* sigh, RHEL3 systems do not have libblkid.so.1 */ #ifdef HAVE_BLKID_BLKID_H #include #endif @@ -224,7 +229,7 @@ static loff_t sizeof_dev(int fd) } } fprintf(stderr, "%s: unable to determine size of %s\n", - progname, devname); + progname, devname); return 0; #endif @@ -241,8 +246,9 @@ out: * Verify_chunk: Verifies test pattern in each 4kB (BLOCKSIZE) is correct. * Returns 0 if test offset and timestamp is correct otherwise 1. */ -int verify_chunk(char *chunk_buf, size_t chunksize, - unsigned long long chunk_off, time_t time_st) +int verify_chunk(char *chunk_buf, const size_t chunksize, + unsigned long long chunk_off, const unsigned long long time_st, + const unsigned long long inode_st, const char *file) { struct block_data *bd; char *chunk_end; @@ -251,12 +257,15 @@ int verify_chunk(char *chunk_buf, size_t chunksize, (char *)chunk_buf < chunk_end; chunk_buf += BLOCKSIZE, chunk_off += BLOCKSIZE) { bd = (struct block_data *)chunk_buf; - if ((bd->bd_offset == chunk_off) && (bd->bd_time == time_st)) + if ((bd->bd_offset == chunk_off) && (bd->bd_time == time_st) && + (bd->bd_inode == inode_st)) continue; - fprintf(stderr, "\n%s: verify failed at offset/timestamp " - "%llu/%lu: found %llu/%lu instead\n", progname, - chunk_off, time_st, bd->bd_offset, bd->bd_time); + fprintf(stderr, "\n%s: verify %s failed offset/timestamp/inode " + "%llu/%llu/%llu: found %llu/%llu/%llu instead\n", + progname, file, chunk_off, time_st, inode_st, + bd->bd_offset, bd->bd_time, bd->bd_inode); + error_count++; return 1; } return 0; @@ -268,7 +277,7 @@ int verify_chunk(char *chunk_buf, size_t chunksize, * each 4kB(BLOCKSIZE) blocks in chunk_buf. */ void fill_chunk(char *chunk_buf, size_t chunksize, loff_t chunk_off, - time_t time_st) + const time_t time_st, const ino_t inode_st) { struct block_data *bd; char *chunk_end; @@ -279,6 +288,7 @@ void fill_chunk(char *chunk_buf, size_t chunksize, loff_t chunk_off, bd = (struct block_data *)chunk_buf; bd->bd_offset = chunk_off; bd->bd_time = time_st; + bd->bd_inode = inode_st; } } @@ -309,57 +319,101 @@ void show_rate(char *op, unsigned long long offset, unsigned long long *count) } /* - * write_chunk: write the chunk_buf on the device. The number of write + * Write a chunk to disk, handling errors, interrupted writes, etc. + * + * If there is an IO error hit during the write, it is possible that + * this will just show up as a short write, and a subsequent write + * will return the actual error. We want to continue in the face of + * minor media errors so that we can validate the whole device if + * possible, but if there are many errors we don't want to loop forever. + * + * The error count will be returned upon exit to ensure that the + * media errors are detected even if nobody is looking at the output. + * + * Returns 0 on success, or -ve errno on failure. + */ +size_t write_retry(int fd, const char *chunk_buf, size_t nrequested, + unsigned long long offset, const char *file) +{ + long nwritten; + +retry: + nwritten = write(fd, chunk_buf, nrequested); + if (nwritten < 0) { + if (errno != ENOSPC) { + fprintf(stderr, "\n%s: write %s@%llu+%zi failed: %s\n", + progname, file, offset, nrequested, + strerror(errno)); + if (error_count++ < 100) + return 0; + } + return -errno; + } + if (nwritten < nrequested) { + fprintf(stderr, "\n%s: write %s@%llu+%zi short: %ld written\n", + progname, file, offset, nrequested, nwritten); + offset += nwritten; + nrequested -= nwritten; + goto retry; + } + + return 0; +} + +/* + * write_chunks: write the chunk_buf on the device. The number of write * operations are based on the parameters write_end, offset, and chunksize. + * + * Returns 0 on success, or -ve error number on failure. */ -int write_chunks(unsigned long long offset, unsigned long long write_end, - char *chunk_buf, size_t chunksize, time_t time_st) +int write_chunks(int fd, unsigned long long offset,unsigned long long write_end, + char *chunk_buf, size_t chunksize, const time_t time_st, + const ino_t inode_st, const char *file) { unsigned long long stride, count = 0; stride = full ? chunksize : (ONE_GB - chunksize); - for (offset = offset & ~(chunksize - 1); offset < write_end; offset += stride) { + int ret; + if (lseek64(fd, offset, SEEK_SET) == -1) { - fprintf(stderr, "\n%s: lseek64(%llu) failed: %s\n", - progname, offset, strerror(errno)); - return 1; + fprintf(stderr, "\n%s: lseek64(%s+%llu) failed: %s\n", + progname, file, offset, strerror(errno)); + return -errno; } if (offset + chunksize > write_end) chunksize = write_end - offset; - if (!full && offset > chunksize) { - fill_chunk(chunk_buf, chunksize, offset, time_st); - if (write(fd, chunk_buf, chunksize) < 0) { - fprintf(stderr, "\n%s: write %llu failed: %s\n", - progname, offset, strerror(errno)); - return 1; - } + fill_chunk(chunk_buf, chunksize, offset, time_st, + inode_st); + ret = write_retry(fd, chunk_buf, chunksize, + offset, file); + if (ret < 0) + return ret; offset += chunksize; + count += chunksize; if (offset + chunksize > write_end) chunksize = write_end - offset; } - - fill_chunk(chunk_buf, chunksize, offset, time_st); - if (write(fd, chunk_buf, chunksize) < 0) { - fprintf(stderr, "\n%s: write %llu failed: %s\n", - progname, offset, strerror(errno)); - return 1; - } + fill_chunk(chunk_buf, chunksize, offset, time_st, inode_st); + ret = write_retry(fd, chunk_buf, chunksize, offset, file); + if (ret < 0) + return ret; count += chunksize; if (verbose > 1) show_rate("write", offset, &count); } + if (verbose > 1) { show_rate("write", offset, &count); printf("\nwrite complete\n"); } if (fsync(fd) == -1) { - fprintf(stderr, "%s: fsync faild: %s\n", progname, + fprintf(stderr, "%s: fsync failed: %s\n", progname, strerror(errno)); - return 1; + return -errno; } return 0; } @@ -368,19 +422,21 @@ int write_chunks(unsigned long long offset, unsigned long long write_end, * read_chunk: reads the chunk_buf from the device. The number of read * operations are based on the parameters read_end, offset, and chunksize. */ -int read_chunks(unsigned long long offset, unsigned long long read_end, - char *chunk_buf, size_t chunksize, time_t time_st) +int read_chunks(int fd, unsigned long long offset, unsigned long long read_end, + char *chunk_buf, size_t chunksize, const time_t time_st, + const ino_t inode_st, const char *file) { unsigned long long stride, count = 0; - stride = full ? chunksize : (ONE_GB - chunksize); - if (ioctl(fd, BLKFLSBUF, 0) < 0 && verbose) fprintf(stderr, "%s: ioctl BLKFLSBUF failed: %s (ignoring)\n", progname, strerror(errno)); + stride = full ? chunksize : (ONE_GB - chunksize); for (offset = offset & ~(chunksize - 1); offset < read_end; offset += stride) { + ssize_t nread; + if (lseek64(fd, offset, SEEK_SET) == -1) { fprintf(stderr, "\n%s: lseek64(%llu) failed: %s\n", progname, offset, strerror(errno)); @@ -390,26 +446,54 @@ int read_chunks(unsigned long long offset, unsigned long long read_end, chunksize = read_end - offset; if (!full && offset > chunksize) { - if (read (fd, chunk_buf, chunksize) < 0) { - fprintf(stderr, "\n%s: read %llu failed: %s\n", - progname, offset, strerror(errno)); + nread = read(fd, chunk_buf, chunksize); + if (nread < 0) { + fprintf(stderr,"\n%s: read %s@%llu+%zi failed: " + "%s\n", progname, file, offset, + chunksize, strerror(errno)); + error_count++; return 1; } - if (verify_chunk(chunk_buf, chunksize, offset, - time_st) != 0) + if (nread < chunksize) { + fprintf(stderr, "\n%s: read %s@%llu+%zi short: " + "%zi read\n", progname, file, offset, + chunksize, nread); + error_count++; + } + if (verify_chunk(chunk_buf, nread, offset, time_st, + inode_st, file) != 0) return 1; offset += chunksize; + count += chunksize; + + /* Need to reset position after read error */ + if (nread < chunksize && + lseek64(fd, offset, SEEK_SET) == -1) { + fprintf(stderr, + "\n%s: lseek64(%s@%llu) failed: %s\n", + progname, file, offset,strerror(errno)); + return 1; + } if (offset + chunksize >= read_end) chunksize = read_end - offset; } - if (read(fd, chunk_buf, chunksize) < 0) { + nread = read(fd, chunk_buf, chunksize); + if (nread < 0) { fprintf(stderr, "\n%s: read failed: %s\n", progname, strerror(errno)); + error_count++; return 1; } + if (nread < chunksize) { + fprintf(stderr, "\n%s: read %s@%llu+%zi short: " + "%zi read\n", progname, file, offset, + chunksize, nread); + error_count++; + } - if (verify_chunk(chunk_buf, chunksize, offset, time_st) != 0) + if (verify_chunk(chunk_buf, nread, offset, time_st, + inode_st, file) != 0) return 1; count += chunksize; @@ -433,6 +517,7 @@ int main(int argc, char **argv) unsigned long long dev_size = 0; char yesno[4]; int mode = O_RDWR; /* mode which device should be opened */ + int fd; int error = 0, c; progname = strrchr(argv[0], '/') == NULL ? @@ -549,16 +634,18 @@ int main(int argc, char **argv) goto close_dev; } if (writeoption) { - if (write_chunks(offset, dev_size, chunk_buf, chunksize, - time_st)) { + c = write_chunks(fd, offset, dev_size, chunk_buf, chunksize, + time_st, 0, devname); + if (c < 0 && c != -ENOSPC) { error = 3; goto chunk_buf; } - if (!full) { /* end of device aligned to a block */ + if (!full) { /* end of device aligned to a block */ offset = ((dev_size - chunksize + BLOCKSIZE - 1) & ~(BLOCKSIZE - 1)); - if (write_chunks(offset, dev_size, chunk_buf, chunksize, - time_st)) { + c = write_chunks(fd, offset, dev_size, chunk_buf, + chunksize, time_st, 0, devname); + if (c < 0 && c != -ENOSPC) { error = 3; goto chunk_buf; } @@ -566,16 +653,16 @@ int main(int argc, char **argv) offset = offset_orig; } if (readoption) { - if (read_chunks(offset, dev_size, chunk_buf, chunksize, - time_st)) { + if (read_chunks(fd, offset, dev_size, chunk_buf, chunksize, + time_st, 0, devname)) { error = 2; goto chunk_buf; } if (!full) { /* end of device aligned to a block */ offset = ((dev_size - chunksize + BLOCKSIZE - 1) & ~(BLOCKSIZE - 1)); - if (read_chunks(offset, dev_size, chunk_buf, chunksize, - time_st)) { + if (read_chunks(fd, offset, dev_size, chunk_buf, + chunksize, time_st, 0, devname)) { error = 2; goto chunk_buf; } @@ -583,7 +670,7 @@ int main(int argc, char **argv) if (verbose) printf("\n%s: data verified successfully\n", progname); } - error = 0; + error = error_count; chunk_buf: free(chunk_buf); close_dev: diff --git a/lustre/utils/llverfs.c b/lustre/utils/llverfs.c index 3f65d55..6d7a85d 100644 --- a/lustre/utils/llverfs.c +++ b/lustre/utils/llverfs.c @@ -26,7 +26,7 @@ * GPL HEADER END */ /* - * Copyright 2008 Sun Microsystems, Inc. All rights reserved + * Copyright 2009 Sun Microsystems, Inc. All rights reserved * Use is subject to license terms. */ /* @@ -35,22 +35,27 @@ * * lustre/utils/llverfs.c * - * ext3 Filesystem Verification Tool. - * This program tests the correct operation of ext3 filesystem. + * Filesystem Verification Tool. + * This program tests the correct operation of large filesystems and + * the underlying block storage device(s). * This tool have two working modes * 1. full mode * 2. fast mode - * The full mode is basic mode in which program creates a subdirectory - * in the test fileysytem, writes n(files_in_dir, default=16) large(4GB) files - * to the directory with the test pattern at the start of each 4kb block. - * The test pattern contains timestamp, relative file offset and per file - * unique idenfifier(inode number). this continues until whole filesystem is - * full and then this tooll verifies that the data in all of the test files - * is correct. - * In the fast mode the tool creates a test directories with - * EXT3_TOPDIR_FL flag set. the number of directories equals to the number - * of block groups in the filesystem(e.g. 65536 directories for 8TB filesystem) - * and then writes a single 1MB file in each directory. The tool then verifies + * + * In full mode, the program creates a subdirectory in the test + * fileysytem, writes n(files_in_dir, default=16) large(4GB) files to + * the directory with the test pattern at the start of each 4kb block. + * The test pattern contains timestamp, relative file offset and per + * file unique idenfifier(inode number). This continues until the + * whole filesystem is full and then the tool verifies that the data + * in all of the test files is correct. + * + * In fast mode, the tool creates test directories with the + * EXT3_TOPDIR_FL flag set (if supported) to spread the directory data + * around the block device instead of localizing it in a single place. + * The number of directories equals to the number of block groups in the + * filesystem (e.g. 65536 directories for 8TB ext3/ext4 filesystem) and + * then writes a single 1MB file in each directory. The tool then verifies * that the data in each file is correct. */ @@ -108,13 +113,13 @@ static int readoption; /* run test in read-only (verify) mode */ static int writeoption; /* run test in write_only mode */ char *testdir; /* name of device to be tested. */ static unsigned full = 1; /* flag to full check */ -static int errno_local; /* local copy of errno */ -static unsigned long num_files; /* Total number of files for read/write */ +static int error_count; /* number of IO errors hit during run */ +char filecount[PATH_MAX]; /* file with total number of files written*/ +static unsigned long num_files; /* Total number of files for read/write */ static loff_t file_size = 4*ONE_GB; /* Size of each file */ static unsigned files_in_dir = 32; /* number of files in each directioy */ static unsigned num_dirs = 30000; /* total number of directories */ const int dirmode = S_IRWXU | S_IRGRP | S_IXGRP | S_IROTH | S_IXOTH; -static int fd = -1; static int isatty_flag; static int perms = S_IRWXU | S_IRGRP | S_IROTH; @@ -124,9 +129,11 @@ static struct option const longopts[] = { "help", no_argument, 0, 'h' }, { "offset", required_argument, 0, 'o' }, { "long", no_argument, 0, 'l' }, + { "full", no_argument, 0, 'l' }, { "partial", required_argument, 0, 'p' }, { "quiet", required_argument, 0, 'q' }, { "read", no_argument, 0, 'r' }, + { "filesize", no_argument, 0, 's' }, { "timestamp", required_argument, 0, 't' }, { "verbose", no_argument, 0, 'v' }, { "write", no_argument, 0, 'w' }, @@ -139,23 +146,22 @@ static struct option const longopts[] = */ void usage(int status) { - if (status != 0) - { - printf("\nUsage: %s [OPTION]... ...\n", - progname); - printf("ext3 filesystem verification tool.\n" - "\t-t {seconds} for --timestamp, set test time" - "(default=current time())\n" - "\t-o {offset} for --offset, directory starting offset" - " from which tests should start\n" - "\t-r run test in read (verify) mode\n" - "\t-w run test in write (test-pattern) mode (default=r&w)\n" - "\t-v for verbose\n" - "\t-p for --partial, for partial check (1MB files)\n" - "\t-l for --long, full check (4GB file with 4k blocks)\n" - "\t-c for --chunksize, IO chunk size (default=1048576)\n" - "\t-h display this help and exit\n" - "\t--help display this help and exit\n"); + if (status != 0) { + printf("\nUsage: %s [OPTION]... ...\n", + progname); + printf("ext3 filesystem verification tool.\n" + "\t-t {seconds}, --timestamp, set test time" + "(default=current time())\n" + "\t-o {offset}, --offset, directory starting offset" + " from which tests should start\n" + "\t-r, --read, run in verify mode\n" + "\t-w, --write, run in test-pattern mode, default=rw\n" + "\t-v, --verbose\n" + "\t-p, --partial, for partial check (1MB files)\n" + "\t-l, --long, --full check (4GB file with 4k blocks)\n" + "\t-c, --chunksize, IO chunk size in MB (default=1)\n" + "\t-s, --filesize, file size in MB (default=4096)\n" + "\t-h, --help, display this help and exit\n"); } exit(status); } @@ -165,7 +171,7 @@ void usage(int status) */ static int open_file(const char *file, int flag) { - fd = open(file, flag, perms); + int fd = open(file, flag, perms); if (fd < 0) { fprintf(stderr, "\n%s: Open '%s' failed:%s\n", progname, file, strerror(errno)); @@ -178,9 +184,9 @@ static int open_file(const char *file, int flag) * Verify_chunk: Verifies test pattern in each 4kB (BLOCKSIZE) is correct. * Returns 0 if test offset and timestamp is correct otherwise 1. */ -int verify_chunk(char *chunk_buf, size_t chunksize,unsigned long long chunk_off, - unsigned long long time_st, unsigned long long inode_st, - char *file) +int verify_chunk(char *chunk_buf, const size_t chunksize, + unsigned long long chunk_off, const unsigned long long time_st, + const unsigned long long inode_st, const char *file) { struct block_data *bd; char *chunk_end; @@ -192,7 +198,7 @@ int verify_chunk(char *chunk_buf, size_t chunksize,unsigned long long chunk_off, if ((bd->bd_offset == chunk_off) && (bd->bd_time == time_st) && (bd->bd_inode == inode_st)) continue; - fprintf(stderr,"\n%s: verify %s failed offset/timestamp/inode " + fprintf(stderr, "\n%s: verify %s failed offset/timestamp/inode " "%llu/%llu/%llu: found %llu/%llu/%llu instead\n", progname, file, chunk_off, time_st, inode_st, bd->bd_offset, bd->bd_time, bd->bd_inode); @@ -207,7 +213,7 @@ int verify_chunk(char *chunk_buf, size_t chunksize,unsigned long long chunk_off, * each 4kB(BLOCKSIZE) blocks in chunk_buf. */ void fill_chunk(char *chunk_buf, size_t chunksize, loff_t chunk_off, - time_t time_st, ino_t inode_st) + const time_t time_st, const ino_t inode_st) { struct block_data *bd; char *chunk_end; @@ -223,52 +229,85 @@ void fill_chunk(char *chunk_buf, size_t chunksize, loff_t chunk_off, } /* - * write_chunk: write the chunk_buf on the device. The number of write + * Write a chunk to disk, handling errors, interrupted writes, etc. + * + * If there is an IO error hit during the write, it is possible that + * this will just show up as a short write, and a subsequent write + * will return the actual error. We want to continue in the face of + * minor media errors so that we can validate the whole device if + * possible, but if there are many errors we don't want to loop forever. + * + * The error count will be returned upon exit to ensure that the + * media errors are detected even if nobody is looking at the output. + * + * Returns 0 on success, or -ve errno on failure. + */ +int write_retry(int fd, const char *chunk_buf, size_t nrequested, + unsigned long long offset, const char *file) +{ + long nwritten; + +retry: + nwritten = write(fd, chunk_buf, nrequested); + if (nwritten < 0) { + if (errno != ENOSPC) { + fprintf(stderr, "\n%s: write %s@%llu+%zi failed: %s\n", + progname, file, offset, nrequested, + strerror(errno)); + if (error_count++ < 100) + return 0; + } + return -errno; + } + if (nwritten < nrequested) { + fprintf(stderr, "\n%s: write %s@%llu+%zi short: %ld written\n", + progname, file, offset, nrequested, nwritten); + offset += nwritten; + nrequested -= nwritten; + goto retry; + } + + return 0; +} + +/* + * write_chunks: write the chunk_buf on the device. The number of write * operations are based on the parameters write_end, offset, and chunksize. + * + * Returns 0 on success, or -ve error number on failure. */ int write_chunks(int fd, unsigned long long offset,unsigned long long write_end, - char *chunk_buf, size_t chunksize, time_t time_st, - ino_t inode_st, const char *file) + char *chunk_buf, size_t chunksize, const time_t time_st, + const ino_t inode_st, const char *file) { unsigned long long stride; stride = full ? chunksize : (ONE_GB - chunksize); for (offset = offset & ~(chunksize - 1); offset < write_end; offset += stride) { + int ret; + if (lseek64(fd, offset, SEEK_SET) == -1) { fprintf(stderr, "\n%s: lseek64(%s+%llu) failed: %s\n", progname, file, offset, strerror(errno)); - return 1; + return -errno; } if (offset + chunksize > write_end) chunksize = write_end - offset; if (!full && offset > chunksize) { fill_chunk(chunk_buf, chunksize, offset, time_st, - inode_st); - if (write(fd, chunk_buf, chunksize) < 0) { - if (errno == ENOSPC) { - errno_local = errno; - return 0; - } - fprintf(stderr, - "\n%s: write %s+%llu failed: %s\n", - progname, file, offset,strerror(errno)); - return errno; - } + inode_st); + ret = write_retry(fd, chunk_buf, chunksize,offset,file); + if (ret < 0) + return ret; offset += chunksize; if (offset + chunksize > write_end) chunksize = write_end - offset; } fill_chunk(chunk_buf, chunksize, offset, time_st, inode_st); - if (write(fd, (char *) chunk_buf, chunksize) < 0) { - if (errno == ENOSPC) { - errno_local = errno; - return 0; - } - fprintf(stderr, "\n%s: write %s+%llu failed: %s\n", - progname, file, offset, strerror(errno)); - return 1; - } + ret = write_retry(fd, chunk_buf, chunksize, offset, file); + if (ret < 0) + return ret; } return 0; } @@ -278,14 +317,16 @@ int write_chunks(int fd, unsigned long long offset,unsigned long long write_end, * operations are based on the parameters read_end, offset, and chunksize. */ int read_chunks(int fd, unsigned long long offset, unsigned long long read_end, - char *chunk_buf, size_t chunksize, time_t time_st, - ino_t inode_st, char *file) + char *chunk_buf, size_t chunksize, const time_t time_st, + const ino_t inode_st, const char *file) { unsigned long long stride; stride = full ? chunksize : (ONE_GB - chunksize); for (offset = offset & ~(chunksize - 1); offset < read_end; offset += stride) { + ssize_t nread; + if (lseek64(fd, offset, SEEK_SET) == -1) { fprintf(stderr, "\n%s: lseek64(%s+%llu) failed: %s\n", progname, file, offset, strerror(errno)); @@ -293,28 +334,58 @@ int read_chunks(int fd, unsigned long long offset, unsigned long long read_end, } if (offset + chunksize > read_end) chunksize = read_end - offset; + if (!full && offset > chunksize) { - if (read(fd, chunk_buf, chunksize) < 0) { - fprintf(stderr, - "\n%s: read %s+%llu failed: %s\n", - progname, file, offset,strerror(errno)); + nread = read(fd, chunk_buf, chunksize); + if (nread < 0) { + fprintf(stderr,"\n%s: read %s@%llu+%zi failed: " + "%s\n", progname, file, offset, + chunksize, strerror(errno)); + error_count++; return 1; } - if (verify_chunk(chunk_buf, chunksize, offset, - time_st, inode_st, file) != 0) + if (nread < chunksize) { + fprintf(stderr, "\n%s: read %s@%llu+%zi short: " + "%zi read\n", progname, file, offset, + chunksize, nread); + error_count++; + } + if (verify_chunk(chunk_buf, nread, offset, time_st, + inode_st, file) != 0) { return 1; + } offset += chunksize; + + /* Need to reset position after read error */ + if (nread < chunksize && + lseek64(fd, offset, SEEK_SET) == -1) { + fprintf(stderr, + "\n%s: lseek64(%s@%llu) failed: %s\n", + progname, file, offset,strerror(errno)); + return 1; + } if (offset + chunksize >= read_end) chunksize = read_end - offset; } - if (read(fd, chunk_buf, chunksize) < 0) { - fprintf(stderr, "\n%s: read %s+%llu failed: %s\n", - progname, file, offset, strerror(errno)); + nread = read(fd, chunk_buf, chunksize); + if (nread < 0) { + fprintf(stderr, "\n%s: read %s@%llu+%zi failed: %s\n", + progname, file, offset, chunksize, + strerror(errno)); + error_count++; return 1; } - if (verify_chunk(chunk_buf, chunksize, offset, time_st, - inode_st, file) != 0) + if (nread < chunksize) { + fprintf(stderr, "\n%s: read %s@%llu+%zi short: " + "%zi read\n", progname, file, offset, + chunksize, nread); + error_count++; + } + + if (verify_chunk(chunk_buf, nread, offset, time_st, + inode_st, file) != 0) { return 1; + } } return 0; } @@ -369,6 +440,7 @@ static int dir_write(char *chunk_buf, size_t chunksize, { char tempfile[PATH_MAX]; char tempdir[PATH_MAX]; + FILE *countfile; struct stat64 file; int file_num = 999999999; ino_t inode_st = 0; @@ -379,7 +451,22 @@ static int dir_write(char *chunk_buf, size_t chunksize, "\n%s: can't set TOPDIR_FL on %s: %s (ignoring)", progname, testdir, strerror(errno)); #endif + countfile = fopen(filecount, "w"); + if (countfile == NULL) { + fprintf(stderr, "\n%s: creating %s failed :%s\n", + progname, filecount, strerror(errno)); + return 5; + } + /* reserve space for the countfile */ + if (fprintf(countfile, "%lu", num_files) < 1 || + fflush(countfile) != 0) { + fprintf(stderr, "\n%s: writing %s failed :%s\n", + progname, filecount, strerror(errno)); + return 6; + } for (; dir_num < num_dirs; num_files++, file_num++) { + int fd, ret; + if (file_num >= files_in_dir) { if (dir_num == num_dirs - 1) break; @@ -411,16 +498,23 @@ static int dir_write(char *chunk_buf, size_t chunksize, if (verbose > 1) show_filename("write", tempfile); - if (write_chunks(fd, 0, file_size, chunk_buf, chunksize, - time_st, inode_st, tempfile)) { - close(fd); - return 1; - } + ret = write_chunks(fd, 0, file_size, chunk_buf, chunksize, + time_st, inode_st, tempfile); close(fd); - - if (errno_local == ENOSPC) + if (ret < 0) { + if (ret != -ENOSPC) + return 1; break; + } + + fseek(countfile, 0, SEEK_SET); + if (fprintf(countfile, "%lu", num_files) < 1 || + fflush(countfile) != 0) { + fprintf(stderr, "\n%s: writing %s failed :%s\n", + progname, filecount, strerror(errno)); + } } + fclose(countfile); if (verbose) { verbose++; @@ -447,6 +541,8 @@ static int dir_read(char *chunk_buf, size_t chunksize, ino_t inode_st = 0; for (count = 0; count < num_files && dir_num < num_dirs; count++) { + int fd, ret; + if (file_num == 0) { if (dir_num == num_dirs - 1) break; @@ -470,12 +566,11 @@ static int dir_read(char *chunk_buf, size_t chunksize, if (count == num_files) file_size = file.st_size; - if (read_chunks(fd, 0, file_size, chunk_buf, chunksize, - time_st, inode_st, tempfile)) { - close(fd); - return 1; - } + ret = read_chunks(fd, 0, file_size, chunk_buf, chunksize, + time_st, inode_st, tempfile); close(fd); + if (ret) + return 1; if (++file_num >= files_in_dir) file_num = 0; @@ -496,19 +591,18 @@ int main(int argc, char **argv) char *chunk_buf; /* chunk buffer */ int error = 0; FILE *countfile = NULL; - char filecount[PATH_MAX]; unsigned long dir_num = 0, dir_num_orig = 0;/* starting directory */ int c; progname = strrchr(argv[0], '/') ? strrchr(argv[0], '/') + 1 : argv[0]; - while ((c = getopt_long(argc, argv, "t:rwvplo:h", + while ((c = getopt_long(argc, argv, "c:hlo:pqrs:t:vw", longopts, NULL)) != -1) { switch (c) { case 'c': - chunksize = (strtoul(optarg, NULL, 0) * ONE_MB); - if (!chunksize) { - fprintf(stderr, "%s: Chunk size value should be" - "a multiple of 1MB\n", progname); + chunksize = strtoul(optarg, NULL, 0) * ONE_MB; + if (chunksize == 0) { + fprintf(stderr, "%s: bad chunk size '%s'\n", + optarg, progname); return -1; } break; @@ -519,6 +613,9 @@ int main(int argc, char **argv) dir_num = strtoul(optarg, NULL, 0); break; case 'p': + file_size = ONE_MB; + chunksize = ONE_MB; + files_in_dir = 1; full = 0; break; case 'q': @@ -527,15 +624,23 @@ int main(int argc, char **argv) case 'r': readoption = 1; break; + case 's': + file_size = strtoul(optarg, NULL, 0) * ONE_MB; + if (file_size == 0) { + fprintf(stderr, "%s: bad file size '%s'\n", + optarg, progname); + return -1; + } + break; case 't': time_st = (time_t)strtoul(optarg, NULL, 0); break; - case 'w': - writeoption = 1; - break; case 'v': verbose++; break; + case 'w': + writeoption = 1; + break; case 'h': default: @@ -603,11 +708,11 @@ int main(int argc, char **argv) fs->super->s_blocks_per_group); ext2fs_close(fs); #else - goto guess; + goto guess; #endif if (0) { /* ugh */ struct statfs64 statbuf; - guess: +guess: if (statfs64(testdir, &statbuf) == 0) { num_dirs = (long long)statbuf.f_blocks * statbuf.f_bsize / (128ULL << 20); @@ -622,10 +727,6 @@ int main(int argc, char **argv) printf("dirs: %u\n", num_dirs); } } - - file_size = ONE_MB; - chunksize = ONE_MB; - files_in_dir = 1; } chunk_buf = (char *)calloc(chunksize, 1); if (chunk_buf == NULL) { @@ -647,22 +748,14 @@ int main(int argc, char **argv) error = 3; goto out; } - countfile = fopen(filecount, "w"); - if (countfile != NULL) { - if (fprintf(countfile, "%lu", num_files) < 1 || - fflush(countfile) != 0) { - fprintf(stderr, "\n%s: writing %s failed :%s\n", - progname, filecount, strerror(errno)); - } - fclose(countfile); - } dir_num = dir_num_orig; } if (readoption) { if (!writeoption) { countfile = fopen(filecount, "r"); if (countfile == NULL || - fscanf(countfile, "%lu", &num_files) != 1) { + fscanf(countfile, "%lu", &num_files) != 1 || + num_files == 0) { fprintf(stderr, "\n%s: reading %s failed :%s\n", progname, filecount, strerror(errno)); num_files = num_dirs * files_in_dir; @@ -679,7 +772,7 @@ int main(int argc, char **argv) goto out; } } - error = 0; + error = error_count; out: free(chunk_buf); return error;