1 // SPDX-License-Identifier: GPL-2.0
4 * Copyright (c) 2007, 2010, Oracle and/or its affiliates. All rights reserved.
5 * Use is subject to license terms.
7 * Copyright (c) 2011, Intel Corporation.
12 * This file is part of Lustre, http://www.lustre.org/
14 * lustre/utils/llverfs.c
16 * Filesystem Verification Tool: this program tests the correct operation of
17 * large filesystems and the underlying block storage device(s). For more
18 * information, see the llverfs.8 man page.
28 #ifndef _LARGEFILE64_SOURCE
29 #define _LARGEFILE64_SOURCE
31 #ifndef _FILE_OFFSET_BITS
32 #define _FILE_OFFSET_BITS 64
49 #include <sys/types.h>
53 #include <gnu/stubs.h>
54 #include <gnu/stubs.h>
56 #ifdef HAVE_EXT2FS_EXT2FS_H
58 # include <ext2fs/ext2fs.h>
60 # ifndef EXT2_TOPDIR_FL
61 # define EXT2_TOPDIR_FL 0x00020000 /* Top of directory tree */
63 static int fsetflags(const char *path, unsigned int flag)
65 char cmd[PATH_MAX + 128];
68 if (flag != EXT2_TOPDIR_FL) {
73 snprintf(cmd, sizeof(cmd), "chattr +T %s", path);
86 #define ONE_MB (1024 * 1024)
87 #define ONE_GB ((unsigned long long)(1024 * 1024 * 1024))
88 #define BLOCKSIZE 4096
90 /* Structure for writing test pattern */
92 unsigned long long bd_offset;
93 unsigned long long bd_time;
94 unsigned long long bd_inode;
96 static char *progname; /* name by which this program was run. */
97 static unsigned verbose = 1; /* prints offset in kB, operation rate */
98 static int readoption; /* run test in read-only (verify) mode */
99 static int writeoption; /* run test in write_only mode */
100 char *testdir; /* name of device to be tested. */
101 static unsigned full = 1; /* flag to full check */
102 static int error_count; /* number of IO errors hit during run */
103 char filecount[PATH_MAX]; /* file with total number of files written*/
104 static unsigned long num_files; /* Total number of files for read/write */
105 static loff_t file_size = 4*ONE_GB; /* Size of each file */
106 static unsigned files_in_dir = 32; /* number of files in each directioy */
107 static unsigned int num_dirs; /* total number of directories */
108 const int dirmode = S_IRWXU | S_IRGRP | S_IXGRP | S_IROTH | S_IXOTH;
109 static int isatty_flag;
110 static int perms = S_IRWXU | S_IRGRP | S_IROTH;
112 static struct option const long_opts[] = {
113 { .val = 'c', .name = "chunksize", .has_arg = required_argument },
114 { .val = 'h', .name = "help", .has_arg = no_argument },
115 { .val = 'l', .name = "long", .has_arg = no_argument },
116 { .val = 'l', .name = "full", .has_arg = no_argument },
117 { .val = 'n', .name = "num_dirs", .has_arg = required_argument },
118 { .val = 'o', .name = "offset", .has_arg = required_argument },
119 { .val = 'p', .name = "partial", .has_arg = required_argument },
120 { .val = 'q', .name = "quiet", .has_arg = required_argument },
121 { .val = 'r', .name = "read", .has_arg = no_argument },
122 { .val = 's', .name = "filesize", .has_arg = no_argument },
123 { .val = 't', .name = "timestamp", .has_arg = required_argument },
124 { .val = 'v', .name = "verbose", .has_arg = no_argument },
125 { .val = 'w', .name = "write", .has_arg = no_argument },
129 * Usages: displays help information, whenever user supply --help option in
130 * command or enters incorrect command line.
132 static void usage(int status)
135 printf("\nUsage: %s [OPTION]... <filesystem path> ...\n",
137 printf("Filesystem verification tool.\n"
138 "\t-t {seconds}, --timestamp, set test time"
139 "(default=current time())\n"
140 "\t-o {offset}, --offset, directory starting offset"
141 " from which tests should start\n"
142 "\t-r, --read, run in verify mode\n"
143 "\t-w, --write, run in test-pattern mode, default=rw\n"
145 "\t-p, --partial, for partial check (1MB files)\n"
146 "\t-l, --long, --full check (4GB file with 4k blocks)\n"
147 "\t-n, --num_dirs, number of directories to create\n"
148 "\t-c, --chunksize, IO chunk size in MB (default=1)\n"
149 "\t-s, --filesize, file size in MB (default=4096)\n"
150 "\t-h, --help, display this help and exit\n");
156 * open_file: Opens file in specified mode and returns fd.
158 static int open_file(const char *file, int flag)
160 int fd = open(file, flag, perms);
162 fprintf(stderr, "\n%s: Open '%s' failed:%s\n",
163 progname, file, strerror(errno));
169 * Verify_chunk: Verifies test pattern in each 4kB (BLOCKSIZE) is correct.
170 * Returns 0 if test offset and timestamp is correct otherwise 1.
172 static int verify_chunk(char *chunk_buf, const size_t chunksize,
173 unsigned long long chunk_off,
174 const unsigned long long time_st,
175 const unsigned long long inode_st,
178 struct block_data *bd;
181 for (chunk_end = chunk_buf + chunksize - sizeof(*bd);
182 (char *)chunk_buf < chunk_end;
183 chunk_buf += BLOCKSIZE, chunk_off += BLOCKSIZE) {
184 bd = (struct block_data *)chunk_buf;
185 if ((bd->bd_offset == chunk_off) && (bd->bd_time == time_st) &&
186 (bd->bd_inode == inode_st))
188 fprintf(stderr, "\n%s: verify %s failed offset/timestamp/inode "
189 "%llu/%llu/%llu: found %llu/%llu/%llu instead\n",
190 progname, file, chunk_off, time_st, inode_st,
191 bd->bd_offset, bd->bd_time, bd->bd_inode);
198 * fill_chunk: Fills the chunk with current or user specified timestamp
199 * and offset. The test pattern is filled at the beginning of
200 * each 4kB(BLOCKSIZE) blocks in chunk_buf.
202 static void fill_chunk(char *chunk_buf, size_t chunksize, loff_t chunk_off,
203 const time_t time_st, const ino_t inode_st)
205 struct block_data *bd;
208 for (chunk_end = chunk_buf + chunksize - sizeof(*bd);
209 (char *)chunk_buf < chunk_end;
210 chunk_buf += BLOCKSIZE, chunk_off += BLOCKSIZE) {
211 bd = (struct block_data *)chunk_buf;
212 bd->bd_offset = chunk_off;
213 bd->bd_time = time_st;
214 bd->bd_inode = inode_st;
219 * Write a chunk to disk, handling errors, interrupted writes, etc.
221 * If there is an IO error hit during the write, it is possible that
222 * this will just show up as a short write, and a subsequent write
223 * will return the actual error. We want to continue in the face of
224 * minor media errors so that we can validate the whole device if
225 * possible, but if there are many errors we don't want to loop forever.
227 * The error count will be returned upon exit to ensure that the
228 * media errors are detected even if nobody is looking at the output.
230 * Returns 0 on success, or -ve errno on failure.
232 static int write_retry(int fd, const char *chunk_buf, size_t nrequested,
233 unsigned long long offset, const char *file)
238 nwritten = write(fd, chunk_buf, nrequested);
240 if (errno != ENOSPC) {
241 fprintf(stderr, "\n%s: write %s@%llu+%zi failed: %s\n",
242 progname, file, offset, nrequested,
244 if (error_count++ < 100)
249 if (nwritten < nrequested) {
250 fprintf(stderr, "\n%s: write %s@%llu+%zi short: %ld written\n",
251 progname, file, offset, nrequested, nwritten);
253 chunk_buf += nwritten;
254 nrequested -= nwritten;
262 * write_chunks: write the chunk_buf on the device. The number of write
263 * operations are based on the parameters write_end, offset, and chunksize.
265 * Returns 0 on success, or -ve error number on failure.
267 static int write_chunks(int fd, unsigned long long offset,
268 unsigned long long write_end, char *chunk_buf,
269 size_t chunksize, const time_t time_st,
270 const ino_t inode_st, const char *file)
272 unsigned long long stride;
274 stride = full ? chunksize : (ONE_GB - chunksize);
275 for (offset = offset & ~(chunksize - 1); offset < write_end;
279 if (stride != chunksize && lseek64(fd, offset, SEEK_SET) < 0) {
280 fprintf(stderr, "\n%s: lseek66(%s+%llu) failed: %s\n",
281 progname, file, offset, strerror(errno));
284 if (offset + chunksize > write_end)
285 chunksize = write_end - offset;
286 if (!full && offset > chunksize) {
287 fill_chunk(chunk_buf, chunksize, offset, time_st,
289 ret = write_retry(fd, chunk_buf, chunksize,offset,file);
293 if (offset + chunksize > write_end)
294 chunksize = write_end - offset;
296 fill_chunk(chunk_buf, chunksize, offset, time_st, inode_st);
297 ret = write_retry(fd, chunk_buf, chunksize, offset, file);
305 * read_chunk: reads the chunk_buf from the device. The number of read
306 * operations are based on the parameters read_end, offset, and chunksize.
308 static int read_chunks(int fd, unsigned long long offset,
309 unsigned long long read_end, char *chunk_buf,
310 size_t chunksize, const time_t time_st,
311 const ino_t inode_st, const char *file)
313 unsigned long long stride;
315 stride = full ? chunksize : (ONE_GB - chunksize);
316 for (offset = offset & ~(chunksize - 1); offset < read_end;
320 if (stride != chunksize && lseek64(fd, offset, SEEK_SET) < 0) {
321 fprintf(stderr, "\n%s: lseek64(%s+%llu) failed: %s\n",
322 progname, file, offset, strerror(errno));
325 if (offset + chunksize > read_end)
326 chunksize = read_end - offset;
328 if (!full && offset > chunksize) {
329 nread = read(fd, chunk_buf, chunksize);
331 fprintf(stderr,"\n%s: read %s@%llu+%zi failed: "
332 "%s\n", progname, file, offset,
333 chunksize, strerror(errno));
337 if (nread < chunksize) {
338 fprintf(stderr, "\n%s: read %s@%llu+%zi short: "
339 "%zi read\n", progname, file, offset,
343 if (verify_chunk(chunk_buf, nread, offset, time_st,
344 inode_st, file) != 0) {
349 /* Need to reset position after read error */
350 if (nread < chunksize &&
351 lseek64(fd, offset, SEEK_SET) == -1) {
353 "\n%s: lseek64(%s@%llu) failed: %s\n",
354 progname, file, offset,strerror(errno));
357 if (offset + chunksize >= read_end)
358 chunksize = read_end - offset;
360 nread = read(fd, chunk_buf, chunksize);
362 fprintf(stderr, "\n%s: read %s@%llu+%zi failed: %s\n",
363 progname, file, offset, chunksize,
368 if (nread < chunksize) {
369 fprintf(stderr, "\n%s: read %s@%llu+%zi short: "
370 "%zi read\n", progname, file, offset,
375 if (verify_chunk(chunk_buf, nread, offset, time_st,
376 inode_st, file) != 0) {
384 * new_file: prepares new filename using file counter and current dir.
386 static char *new_file(char *tempfile, char *cur_dir, int file_num)
390 rc = snprintf(tempfile, PATH_MAX, "%s/file%03d", cur_dir, file_num);
391 if (rc >= PATH_MAX || rc < 0)
398 * new_dir: prepares new dir name using dir counters.
400 static char *new_dir(char *tempdir, int dir_num)
404 rc = snprintf(tempdir, PATH_MAX, "%s/llverfs_dir%05d", testdir, dir_num);
405 if (rc >= PATH_MAX || rc < 0)
412 * calc_total_bytes: calculates total bytes that need to be
413 * written into or read from the filesystem.
415 static unsigned long long calc_total_bytes(const char *op)
417 unsigned long long total_bytes = 0;
418 struct statfs64 statbuf;
421 if (statfs64(testdir, &statbuf) == 0) {
422 if (strcmp(op, "write") == 0)
423 total_bytes = (unsigned long long)
424 (statbuf.f_bavail * statbuf.f_bsize);
425 else if (strcmp(op, "read") == 0)
426 total_bytes = (unsigned long long)
427 (statbuf.f_blocks * statbuf.f_bsize);
429 fprintf(stderr, "\n%s: invalid operation: %s\n",
434 fprintf(stderr, "\n%s: unable to stat %s: %s\n",
435 progname, testdir, strerror(errno));
439 total_bytes = num_dirs * files_in_dir * file_size;
446 * show_rate: displays the current read/write file name and performance,
447 * along with an estimate of how long the whole read/write operation
450 static void show_rate(char *op, char *filename,
451 const struct timeval *start_time,
452 const unsigned long long total_bytes,
453 const unsigned long long curr_bytes)
455 static struct timeval last_time;
456 static unsigned long long last_bytes;
458 struct timeval curr_time;
459 double curr_delta, overall_delta, curr_rate, overall_rate;
461 int remain_hours, remain_minutes, remain_seconds;
463 if (last_op != op[0]) {
465 last_time = *start_time;
469 gettimeofday(&curr_time, NULL);
471 curr_delta = (curr_time.tv_sec - last_time.tv_sec) +
472 (double)(curr_time.tv_usec - last_time.tv_usec) / 1000000;
474 overall_delta = (curr_time.tv_sec - start_time->tv_sec) +
475 (double)(curr_time.tv_usec - start_time->tv_usec) / 1000000;
477 curr_rate = (curr_bytes - last_bytes) / curr_delta;
478 overall_rate = curr_bytes / overall_delta;
480 if (curr_rate == 0) {
481 last_time = curr_time;
484 remain_time = (total_bytes - curr_bytes) / curr_rate;
486 remain_hours = remain_time / 3600;
487 remain_minutes = (remain_time - remain_hours * 3600) / 60;
488 remain_seconds = (remain_time - remain_hours * 3600 -
489 remain_minutes * 60);
491 if (curr_delta > 4 || verbose > 2) {
495 printf("%s: %s, current: %5g MB/s, overall: %5g MB/s, "
496 "ETA: %u:%02u:%02u", op, filename,
497 curr_rate / ONE_MB, overall_rate / ONE_MB,
498 remain_hours, remain_minutes, remain_seconds);
505 last_time = curr_time;
506 last_bytes = curr_bytes;
511 * dir_write: This function writes directories and files on device.
512 * it works for both full and partial modes.
514 static int dir_write(char *chunk_buf, size_t chunksize,
515 time_t time_st, unsigned long dir_num)
517 char tempfile[PATH_MAX];
518 char tempdir[PATH_MAX];
521 int file_num = 999999999;
523 struct timeval start_time;
524 unsigned long long total_bytes;
525 unsigned long long curr_bytes = 0;
528 if (!full && fsetflags(testdir, EXT2_TOPDIR_FL))
530 "\n%s: can't set TOPDIR_FL on %s: %s (ignoring)",
531 progname, testdir, strerror(errno));
533 countfile = fopen(filecount, "w");
534 if (countfile == NULL) {
535 fprintf(stderr, "\n%s: creating %s failed :%s\n",
536 progname, filecount, strerror(errno));
539 /* reserve space for the countfile */
540 if (fprintf(countfile, "%lu", num_files) < 1 ||
541 fflush(countfile) != 0) {
542 fprintf(stderr, "\n%s: writing %s failed :%s\n",
543 progname, filecount, strerror(errno));
548 /* calculate total bytes that need to be written */
549 total_bytes = calc_total_bytes("write");
550 if (total_bytes <= 0) {
551 fprintf(stderr, "\n%s: unable to calculate total bytes\n",
557 if (!full && (dir_num != 0))
558 total_bytes -= dir_num * files_in_dir * file_size;
560 gettimeofday(&start_time, NULL);
561 for (; dir_num < num_dirs; num_files++, file_num++) {
564 if (file_num >= files_in_dir) {
566 if (mkdir(new_dir(tempdir, dir_num), dirmode) < 0) {
569 if (errno != EEXIST) {
570 fprintf(stderr, "\n%s: mkdir %s : %s\n",
580 fd = open_file(new_file(tempfile, tempdir, file_num),
581 O_WRONLY | O_CREAT | O_TRUNC | O_LARGEFILE);
583 if (fstat64(fd, &file) == 0) {
584 inode_st = file.st_ino;
586 fprintf(stderr, "\n%s: write stat '%s': %s",
587 progname, tempfile, strerror(errno));
595 ret = write_chunks(fd, 0, file_size, chunk_buf, chunksize,
596 time_st, inode_st, tempfile);
599 if (ret != -ENOSPC) {
603 curr_bytes = total_bytes;
607 curr_bytes += file_size;
609 show_rate("write", tempfile, &start_time,
610 total_bytes, curr_bytes);
612 fseek(countfile, 0, SEEK_SET);
613 if (fprintf(countfile, "%lu", num_files) < 1 ||
614 fflush(countfile) != 0) {
615 fprintf(stderr, "\n%s: writing %s failed :%s\n",
616 progname, filecount, strerror(errno));
621 show_rate("write_done", tempfile, &start_time, total_bytes, curr_bytes);
632 * dir_read: This function reads directories and files on device.
633 * it works for both full and partial modes.
635 static int dir_read(char *chunk_buf, size_t chunksize,
636 time_t time_st, unsigned long dir_num)
638 char tempfile[PATH_MAX];
639 char tempdir[PATH_MAX];
640 unsigned long count = 0;
644 struct timeval start_time;
645 unsigned long long total_bytes;
646 unsigned long long curr_bytes = 0;
648 /* calculate total bytes that need to be read */
649 total_bytes = calc_total_bytes("read");
650 if (total_bytes <= 0) {
651 fprintf(stderr, "\n%s: unable to calculate total bytes\n",
657 total_bytes -= dir_num * files_in_dir * file_size;
659 gettimeofday(&start_time, NULL);
660 for (count = 0; count < num_files && dir_num < num_dirs; count++) {
664 new_dir(tempdir, dir_num);
668 fd = open_file(new_file(tempfile, tempdir, file_num),
669 O_RDONLY | O_LARGEFILE);
671 if (fstat64(fd, &file) == 0) {
672 inode_st = file.st_ino;
674 fprintf(stderr, "\n%s: read stat '%s': %s\n",
675 progname, tempfile, strerror(errno));
683 if (count == num_files)
684 file_size = file.st_size;
685 ret = read_chunks(fd, 0, file_size, chunk_buf, chunksize,
686 time_st, inode_st, tempfile);
691 curr_bytes += file_size;
693 show_rate("read", tempfile, &start_time,
694 total_bytes, curr_bytes);
696 if (++file_num >= files_in_dir)
700 show_rate("read_done", tempfile, &start_time, total_bytes, curr_bytes);
707 int main(int argc, char **argv)
709 time_t time_st = 0; /* Default timestamp */
710 size_t chunksize = ONE_MB; /* IO chunk size(defailt=1MB) */
711 char *chunk_buf; /* chunk buffer */
713 FILE *countfile = NULL;
714 unsigned long dir_num = 0, dir_num_orig = 0;/* starting directory */
717 progname = strrchr(argv[0], '/') ? strrchr(argv[0], '/') + 1 : argv[0];
718 while ((c = getopt_long(argc, argv, "c:hln:o:pqrs:t:vw",
719 long_opts, NULL)) != -1) {
720 unsigned long val; /* Staging value for num_dirs */
724 chunksize = strtoul(optarg, NULL, 0) * ONE_MB;
725 if (chunksize == 0) {
726 fprintf(stderr, "%s: bad chunk size '%s'\n",
735 /* num_dirs cannot be negative */
736 if (optarg[0] == '-')
738 val = strtoul(optarg, NULL, 0);
739 if (val == 0 || val > INT_MAX || errno == ERANGE)
744 fprintf(stderr, "%s: bad directory count '%s'", optarg,
748 case 'o': /* offset */
749 dir_num = strtoul(optarg, NULL, 0);
764 file_size = strtoul(optarg, NULL, 0) * ONE_MB;
765 if (file_size == 0) {
766 fprintf(stderr, "%s: bad file size '%s'\n",
772 time_st = (time_t)strtoul(optarg, NULL, 0);
787 testdir = argv[optind];
790 fprintf(stderr, "%s: pathname not given\n", progname);
794 if (!readoption && !writeoption) {
799 (void) time(&time_st);
800 printf("Timestamp: %lu\n", (unsigned long )time_st);
801 isatty_flag = isatty(STDOUT_FILENO);
803 if (!full && !num_dirs) {
804 #ifdef HAVE_EXT2FS_EXT2FS_H
805 struct mntent *tempmnt;
809 if ((fp = setmntent("/etc/mtab", "r")) == NULL) {
810 fprintf(stderr, "%s: fail to open /etc/mtab in read mode :%s\n",
811 progname, strerror(errno));
815 /* find device name using filesystem */
816 while ((tempmnt = getmntent(fp)) != NULL) {
817 if (strcmp(tempmnt->mnt_dir, testdir) == 0)
821 if (tempmnt == NULL) {
822 fprintf(stderr, "%s: no device found for '%s'\n",
828 if (ext2fs_open(tempmnt->mnt_fsname, 0, 0, 0,
829 unix_io_manager, &fs)) {
830 fprintf(stderr, "%s: unable to open ext3 fs on '%s'\n",
837 num_dirs = (fs->super->s_blocks_count +
838 fs->super->s_blocks_per_group - 1) /
839 fs->super->s_blocks_per_group;
841 printf("ext3 block groups: %u, fs blocks: %u "
842 "blocks per group: %u\n",
843 num_dirs, fs->super->s_blocks_count,
844 fs->super->s_blocks_per_group);
851 struct statfs64 statbuf;
854 * Most extN filesystems are formatted with 128MB/group
855 * (32k bitmap = 4KB blocksize * 8 bits/block) * 4KB,
856 * so this is a relatively safe default (somewhat more
857 * or less doesn't make a huge difference for testing).
859 * We want to create one directory per group, together
860 * with the "TOPDIR" feature, so that the directories
861 * are spread across the whole block device.
863 if (statfs64(testdir, &statbuf) == 0) {
864 num_dirs = 1 + (long long)statbuf.f_blocks *
866 (full ? files_in_dir * file_size : 128*ONE_MB);
868 printf("dirs: %u, fs blocks: %llu\n",
869 num_dirs, (long long)statbuf.f_blocks);
871 fprintf(stderr, "%s: unable to stat '%s': %s\n",
872 progname, testdir, strerror(errno));
876 printf("dirs: %u\n", num_dirs);
879 chunk_buf = (char *)calloc(chunksize, 1);
880 if (chunk_buf == NULL) {
881 fprintf(stderr, "Memory allocation failed for chunk_buf\n");
884 snprintf(filecount, sizeof(filecount), "%s/%s.filecount",
887 (void)mkdir(testdir, dirmode);
891 num_files = dir_num * files_in_dir;
893 printf("\n%s: %lu files already written\n",
894 progname, num_files);
896 if (dir_write(chunk_buf, chunksize, time_st, dir_num)) {
900 dir_num = dir_num_orig;
904 countfile = fopen(filecount, "r");
905 if (countfile == NULL ||
906 fscanf(countfile, "%lu", &num_files) != 1 ||
908 fprintf(stderr, "\n%s: reading %s failed :%s\n",
909 progname, filecount, strerror(errno));
910 num_files = num_dirs * files_in_dir;
912 num_files -= (dir_num * files_in_dir);
917 if (dir_read(chunk_buf, chunksize, time_st, dir_num)) {
918 fprintf(stderr, "\n%s: Data verification failed\n",