lustre/utils/llverfs.c

   1 /* -*- mode: c; c-basic-offset: 8; indent-tabs-mode: nil; -*-
   2  * vim:expandtab:shiftwidth=8:tabstop=8:
   3  *
   4  * GPL HEADER START
   5  *
   6  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
   7  *
   8  * This program is free software; you can redistribute it and/or modify
   9  * it under the terms of the GNU General Public License version 2 only,
  10  * as published by the Free Software Foundation.
  11  *
  12  * This program is distributed in the hope that it will be useful, but
  13  * WITHOUT ANY WARRANTY; without even the implied warranty of
  14  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  15  * General Public License version 2 for more details (a copy is included
  16  * in the LICENSE file that accompanied this code).
  17  *
  18  * You should have received a copy of the GNU General Public License
  19  * version 2 along with this program; If not, see
  20  * http://www.sun.com/software/products/lustre/docs/GPLv2.pdf
  21  *
  22  * Please contact Sun Microsystems, Inc., 4150 Network Circle, Santa Clara,
  23  * CA 95054 USA or visit www.sun.com if you need additional information or
  24  * have any questions.
  25  *
  26  * GPL HEADER END
  27  */
  28 /*
  29  * Copyright (c) 2007, 2010, Oracle and/or its affiliates. All rights reserved.
  30  * Use is subject to license terms.
  31  *
  32  * Copyright (c) 2011, Whamcloud, Inc.
  33  */
  34 /*
  35  * This file is part of Lustre, http://www.lustre.org/
  36  * Lustre is a trademark of Sun Microsystems, Inc.
  37  *
  38  * lustre/utils/llverfs.c
  39  *
  40  * Filesystem Verification Tool.
  41  * This program tests the correct operation of large filesystems and
  42  * the underlying block storage device(s).
  43  * This tool have two working modes
  44  * 1. full mode
  45  * 2. partial mode
  46  *
  47  * In full mode, the program creates a subdirectory in the test
  48  * filesystem, writes n(files_in_dir, default=32) large(4GB) files to
  49  * the directory with the test pattern at the start of each 4kb block.
  50  * The test pattern contains timestamp, relative file offset and per
  51  * file unique identifier(inode number).  This continues until the
  52  * whole filesystem is full and then the tool verifies that the data
  53  * in all of the test files is correct.
  54  *
  55  * In partial mode, the tool creates test directories with the
  56  * EXT3_TOPDIR_FL flag set (if supported) to spread the directory data
  57  * around the block device instead of localizing it in a single place.
  58  * The number of directories equals to the number of block groups in the
  59  * filesystem (e.g. 65536 directories for 8TB ext3/ext4 filesystem) and
  60  * then writes a single 1MB file in each directory. The tool then verifies
  61  * that the data in each file is correct.
  62  */
  63
  64 #ifndef _GNU_SOURCE
  65 #define _GNU_SOURCE
  66 #endif
  67 #ifndef LUSTRE_UTILS
  68 #define LUSTRE_UTILS
  69 #endif
  70 #ifndef _LARGEFILE64_SOURCE
  71 #define _LARGEFILE64_SOURCE
  72 #endif
  73 #ifndef _FILE_OFFSET_BITS
  74 #define _FILE_OFFSET_BITS 64
  75 #endif
  76
  77 #include <features.h>
  78 #include <stdlib.h>
  79 #include <stdio.h>
  80 #include <string.h>
  81 #include <ctype.h>
  82 #include <fcntl.h>
  83 #include <unistd.h>
  84 #include <limits.h>
  85 #include <errno.h>
  86 #include <fcntl.h>
  87 #include <getopt.h>
  88 #include <time.h>
  89 #include <dirent.h>
  90 #include <mntent.h>
  91 #include <sys/types.h>
  92 #include <sys/stat.h>
  93 #include <sys/vfs.h>
  94 #include <sys/time.h>
  95 #include <gnu/stubs.h>
  96 #include <gnu/stubs.h>
  97
  98 #ifdef HAVE_EXT2FS_EXT2FS_H
  99 #  include <e2p/e2p.h>
 100 #  include <ext2fs/ext2fs.h>
 101 #endif
 102
 103 #define ONE_MB (1024 * 1024)
 104 #define ONE_GB ((unsigned long long)(1024 * 1024 * 1024))
 105 #define BLOCKSIZE 4096
 106
 107 /* Structure for writing test pattern */
 108 struct block_data {
 109         unsigned long long bd_offset;
 110         unsigned long long bd_time;
 111         unsigned long long bd_inode;
 112 };
 113 static char *progname;              /* name by which this program was run. */
 114 static unsigned verbose = 1;        /* prints offset in kB, operation rate */
 115 static int readoption;              /* run test in read-only (verify) mode */
 116 static int writeoption;             /* run test in write_only mode */
 117 char *testdir;                      /* name of device to be tested. */
 118 static unsigned full = 1;           /* flag to full check */
 119 static int error_count;             /* number of IO errors hit during run */
 120 char filecount[PATH_MAX];           /* file with total number of files written*/
 121 static unsigned long num_files;     /* Total number of files for read/write */
 122 static loff_t file_size = 4*ONE_GB; /* Size of each file */
 123 static unsigned files_in_dir = 32;  /* number of files in each directioy */
 124 static unsigned num_dirs = 30000;   /* total number of directories */
 125 const int dirmode = S_IRWXU | S_IRGRP | S_IXGRP | S_IROTH | S_IXOTH;
 126 static int isatty_flag;
 127 static int perms =  S_IRWXU | S_IRGRP | S_IROTH;
 128
 129 static struct option const longopts[] =
 130 {
 131         { "chunksize", required_argument, 0, 'c' },
 132         { "help", no_argument, 0, 'h' },
 133         { "offset", required_argument, 0, 'o' },
 134         { "long", no_argument, 0, 'l' },
 135         { "full", no_argument, 0, 'l' },
 136         { "partial", required_argument, 0, 'p' },
 137         { "quiet", required_argument, 0, 'q' },
 138         { "read", no_argument, 0, 'r' },
 139         { "filesize", no_argument, 0, 's' },
 140         { "timestamp", required_argument, 0, 't' },
 141         { "verbose", no_argument, 0, 'v' },
 142         { "write", no_argument, 0, 'w' },
 143         { 0, 0, 0, 0}
 144 };
 145
 146 /*
 147  * Usages: displays help information, whenever user supply --help option in
 148  * command or enters incorrect command line.
 149  */
 150 void usage(int status)
 151 {
 152         if (status != 0) {
 153                 printf("\nUsage: %s [OPTION]... <filesystem path> ...\n",
 154                        progname);
 155                 printf("Filesystem verification tool.\n"
 156                        "\t-t {seconds}, --timestamp,  set test time"
 157                        "(default=current time())\n"
 158                        "\t-o {offset}, --offset, directory starting offset"
 159                        " from which tests should start\n"
 160                        "\t-r, --read, run in verify mode\n"
 161                        "\t-w, --write, run in test-pattern mode, default=rw\n"
 162                        "\t-v, --verbose\n"
 163                        "\t-p, --partial, for partial check (1MB files)\n"
 164                        "\t-l, --long, --full check (4GB file with 4k blocks)\n"
 165                        "\t-c, --chunksize, IO chunk size in MB (default=1)\n"
 166                        "\t-s, --filesize, file size in MB (default=4096)\n"
 167                        "\t-h, --help, display this help and exit\n");
 168         }
 169         exit(status);
 170 }
 171
 172 /*
 173  * open_file: Opens file in specified mode and returns fd.
 174  */
 175 static int open_file(const char *file, int flag)
 176 {
 177         int fd = open(file, flag, perms);
 178         if (fd < 0) {
 179                 fprintf(stderr, "\n%s: Open '%s' failed:%s\n",
 180                         progname, file, strerror(errno));
 181         }
 182         return (fd);
 183 }
 184
 185 /*
 186  * Verify_chunk: Verifies test pattern in each 4kB (BLOCKSIZE) is correct.
 187  * Returns 0 if test offset and timestamp is correct otherwise 1.
 188  */
 189 int verify_chunk(char *chunk_buf, const size_t chunksize,
 190                  unsigned long long chunk_off, const unsigned long long time_st,
 191                  const unsigned long long inode_st, const char *file)
 192 {
 193         struct block_data *bd;
 194         char *chunk_end;
 195
 196         for (chunk_end = chunk_buf + chunksize - sizeof(*bd);
 197              (char *)chunk_buf < chunk_end;
 198              chunk_buf += BLOCKSIZE, chunk_off += BLOCKSIZE) {
 199                 bd = (struct block_data *)chunk_buf;
 200                 if ((bd->bd_offset == chunk_off) && (bd->bd_time == time_st) &&
 201                     (bd->bd_inode == inode_st))
 202                         continue;
 203                 fprintf(stderr, "\n%s: verify %s failed offset/timestamp/inode "
 204                         "%llu/%llu/%llu: found %llu/%llu/%llu instead\n",
 205                         progname, file, chunk_off, time_st, inode_st,
 206                         bd->bd_offset, bd->bd_time, bd->bd_inode);
 207                 return 1;
 208         }
 209         return 0;
 210 }
 211
 212 /*
 213  * fill_chunk: Fills the chunk with current or user specified timestamp
 214  * and offset. The test pattern is filled at the beginning of
 215  * each 4kB(BLOCKSIZE) blocks in chunk_buf.
 216  */
 217 void fill_chunk(char *chunk_buf, size_t chunksize, loff_t chunk_off,
 218                 const time_t time_st, const ino_t inode_st)
 219 {
 220         struct block_data *bd;
 221         char *chunk_end;
 222
 223         for (chunk_end = chunk_buf + chunksize - sizeof(*bd);
 224              (char *)chunk_buf < chunk_end;
 225              chunk_buf += BLOCKSIZE, chunk_off += BLOCKSIZE) {
 226                 bd = (struct block_data *)chunk_buf;
 227                 bd->bd_offset = chunk_off;
 228                 bd->bd_time = time_st;
 229                 bd->bd_inode = inode_st;
 230         }
 231 }
 232
 233 /*
 234  * Write a chunk to disk, handling errors, interrupted writes, etc.
 235  *
 236  * If there is an IO error hit during the write, it is possible that
 237  * this will just show up as a short write, and a subsequent write
 238  * will return the actual error.  We want to continue in the face of
 239  * minor media errors so that we can validate the whole device if
 240  * possible, but if there are many errors we don't want to loop forever.
 241  *
 242  * The error count will be returned upon exit to ensure that the
 243  * media errors are detected even if nobody is looking at the output.
 244  *
 245  * Returns 0 on success, or -ve errno on failure.
 246  */
 247 int write_retry(int fd, const char *chunk_buf, size_t nrequested,
 248                 unsigned long long offset, const char *file)
 249 {
 250         long nwritten;
 251
 252 retry:
 253         nwritten = write(fd, chunk_buf, nrequested);
 254         if (nwritten < 0) {
 255                 if (errno != ENOSPC) {
 256                         fprintf(stderr, "\n%s: write %s@%llu+%zi failed: %s\n",
 257                                 progname, file, offset, nrequested,
 258                                 strerror(errno));
 259                         if (error_count++ < 100)
 260                                 return 0;
 261                 }
 262                 return -errno;
 263         }
 264         if (nwritten < nrequested) {
 265                 fprintf(stderr, "\n%s: write %s@%llu+%zi short: %ld written\n",
 266                         progname, file, offset, nrequested, nwritten);
 267                 offset += nwritten;
 268                 chunk_buf += nwritten;
 269                 nrequested -= nwritten;
 270                 goto retry;
 271         }
 272
 273         return 0;
 274 }
 275
 276 /*
 277  * write_chunks: write the chunk_buf on the device. The number of write
 278  * operations are based on the parameters write_end, offset, and chunksize.
 279  *
 280  * Returns 0 on success, or -ve error number on failure.
 281  */
 282 int write_chunks(int fd, unsigned long long offset,unsigned long long write_end,
 283                  char *chunk_buf, size_t chunksize, const time_t time_st,
 284                  const ino_t inode_st, const char *file)
 285 {
 286         unsigned long long stride;
 287
 288         stride = full ? chunksize : (ONE_GB - chunksize);
 289         for (offset = offset & ~(chunksize - 1); offset < write_end;
 290              offset += stride) {
 291                 int ret;
 292
 293                 if (lseek64(fd, offset, SEEK_SET) == -1) {
 294                         fprintf(stderr, "\n%s: lseek64(%s+%llu) failed: %s\n",
 295                                 progname, file, offset, strerror(errno));
 296                         return -errno;
 297                 }
 298                 if (offset + chunksize > write_end)
 299                         chunksize = write_end - offset;
 300                 if (!full && offset > chunksize) {
 301                         fill_chunk(chunk_buf, chunksize, offset, time_st,
 302                                    inode_st);
 303                         ret = write_retry(fd, chunk_buf, chunksize,offset,file);
 304                         if (ret < 0)
 305                                 return ret;
 306                         offset += chunksize;
 307                         if (offset + chunksize > write_end)
 308                                 chunksize = write_end - offset;
 309                 }
 310                 fill_chunk(chunk_buf, chunksize, offset, time_st, inode_st);
 311                 ret = write_retry(fd, chunk_buf, chunksize, offset, file);
 312                 if (ret < 0)
 313                         return ret;
 314         }
 315         return 0;
 316 }
 317
 318 /*
 319  * read_chunk: reads the chunk_buf from the device. The number of read
 320  * operations are based on the parameters read_end, offset, and chunksize.
 321  */
 322 int read_chunks(int fd, unsigned long long offset, unsigned long long read_end,
 323                 char *chunk_buf, size_t chunksize, const time_t time_st,
 324                 const ino_t inode_st, const char *file)
 325 {
 326         unsigned long long stride;
 327
 328         stride = full ? chunksize : (ONE_GB - chunksize);
 329         for (offset = offset & ~(chunksize - 1); offset < read_end;
 330              offset += stride) {
 331                 ssize_t nread;
 332
 333                 if (lseek64(fd, offset, SEEK_SET) == -1) {
 334                         fprintf(stderr, "\n%s: lseek64(%s+%llu) failed: %s\n",
 335                                 progname, file, offset, strerror(errno));
 336                         return 1;
 337                 }
 338                 if (offset + chunksize > read_end)
 339                         chunksize = read_end - offset;
 340
 341                 if (!full && offset > chunksize) {
 342                         nread = read(fd, chunk_buf, chunksize);
 343                         if (nread < 0) {
 344                                 fprintf(stderr,"\n%s: read %s@%llu+%zi failed: "
 345                                         "%s\n", progname, file, offset,
 346                                         chunksize, strerror(errno));
 347                                 error_count++;
 348                                 return 1;
 349                         }
 350                         if (nread < chunksize) {
 351                                 fprintf(stderr, "\n%s: read %s@%llu+%zi short: "
 352                                         "%zi read\n", progname, file, offset,
 353                                         chunksize, nread);
 354                                 error_count++;
 355                         }
 356                         if (verify_chunk(chunk_buf, nread, offset, time_st,
 357                                          inode_st, file) != 0) {
 358                                 return 1;
 359                         }
 360                         offset += chunksize;
 361
 362                         /* Need to reset position after read error */
 363                         if (nread < chunksize &&
 364                             lseek64(fd, offset, SEEK_SET) == -1) {
 365                                 fprintf(stderr,
 366                                         "\n%s: lseek64(%s@%llu) failed: %s\n",
 367                                         progname, file, offset,strerror(errno));
 368                                 return 1;
 369                         }
 370                         if (offset + chunksize >= read_end)
 371                                 chunksize = read_end - offset;
 372                 }
 373                 nread = read(fd, chunk_buf, chunksize);
 374                 if (nread < 0) {
 375                         fprintf(stderr, "\n%s: read %s@%llu+%zi failed: %s\n",
 376                                 progname, file, offset, chunksize,
 377                                 strerror(errno));
 378                         error_count++;
 379                         return 1;
 380                 }
 381                 if (nread < chunksize) {
 382                         fprintf(stderr, "\n%s: read %s@%llu+%zi short: "
 383                                 "%zi read\n", progname, file, offset,
 384                                 chunksize, nread);
 385                         error_count++;
 386                 }
 387
 388                 if (verify_chunk(chunk_buf, nread, offset, time_st,
 389                                  inode_st, file) != 0) {
 390                         return 1;
 391                 }
 392         }
 393         return 0;
 394 }
 395
 396 /*
 397  * new_file: prepares new filename using file counter and current dir.
 398  */
 399 char *new_file(char *tempfile, char *cur_dir, int file_num)
 400 {
 401         sprintf(tempfile, "%s/file%03d", cur_dir, file_num);
 402         return tempfile;
 403 }
 404
 405 /*
 406  * new_dir: prepares new dir name using dir counters.
 407  */
 408 char *new_dir(char *tempdir, int dir_num)
 409 {
 410         sprintf(tempdir, "%s/dir%05d", testdir, dir_num);
 411         return tempdir;
 412 }
 413
 414 /*
 415  * calc_total_bytes: calculates total bytes that need to be
 416  * written into or read from the filesystem.
 417  */
 418 static unsigned long long calc_total_bytes(const char *op)
 419 {
 420         unsigned long long total_bytes = 0;
 421         struct statfs64 statbuf;
 422
 423         if (full) {
 424                 if (statfs64(testdir, &statbuf) == 0) {
 425                         if (strcmp(op, "write") == 0)
 426                                 total_bytes = (unsigned long long)
 427                                         (statbuf.f_bavail * statbuf.f_bsize);
 428                         else if (strcmp(op, "read") == 0)
 429                                 total_bytes = (unsigned long long)
 430                                         (statbuf.f_blocks * statbuf.f_bsize);
 431                         else {
 432                                 fprintf(stderr, "\n%s: invalid operation: %s\n",
 433                                         progname, op);
 434                                 return -1;
 435                         }
 436                 } else {
 437                         fprintf(stderr, "\n%s: unable to stat %s: %s\n",
 438                                 progname, testdir, strerror(errno));
 439                         return -errno;
 440                 }
 441         } else {
 442                 total_bytes = num_dirs * files_in_dir * file_size;
 443         }
 444
 445         return total_bytes;
 446 }
 447
 448 /*
 449  * show_rate: displays the current read/write file name and performance,
 450  * along with an estimate of how long the whole read/write operation
 451  * will continue.
 452  */
 453 void show_rate(char *op, char *filename, const struct timeval *start_time,
 454                const unsigned long long total_bytes,
 455                const unsigned long long curr_bytes)
 456 {
 457         static struct timeval last_time;
 458         static unsigned long long last_bytes;
 459         static char last_op;
 460         struct timeval curr_time;
 461         double curr_delta, overall_delta, curr_rate, overall_rate;
 462         double total_time, remain_time;
 463         int remain_hours, remain_minutes, remain_seconds;
 464
 465         if (last_time.tv_sec == 0)
 466                 last_time = *start_time;
 467
 468         if (last_op != op[0]) {
 469                 last_bytes = 0;
 470                 last_time = *start_time;
 471                 last_op = op[0];
 472         }
 473
 474         gettimeofday(&curr_time, NULL);
 475
 476         curr_delta = (curr_time.tv_sec - last_time.tv_sec) +
 477                 (double)(curr_time.tv_usec - last_time.tv_usec) / 1000000;
 478
 479         overall_delta = (curr_time.tv_sec - start_time->tv_sec) +
 480                 (double)(curr_time.tv_usec - start_time->tv_usec) / 1000000;
 481
 482         curr_rate = (curr_bytes - last_bytes) / curr_delta;
 483         overall_rate = curr_bytes / overall_delta;
 484
 485         if (curr_rate == 0) {
 486                 last_time = curr_time;
 487                 return;
 488         }
 489         total_time = total_bytes / curr_rate;
 490         remain_time = total_time - overall_delta;
 491
 492         remain_hours = remain_time / 3600;
 493         remain_minutes = (remain_time - remain_hours * 3600) / 60;
 494         remain_seconds = (remain_time - remain_hours * 3600 -
 495                 remain_minutes * 60);
 496
 497         if (curr_delta > 4 || verbose > 2) {
 498                 if (isatty_flag)
 499                         printf("\r");
 500
 501                 printf("%s filename: %s, current %5g MB/s, overall %5g MB/s, "
 502                        "est %u:%u:%u left", op, filename,
 503                        curr_rate / ONE_MB, overall_rate / ONE_MB,
 504                        remain_hours, remain_minutes, remain_seconds);
 505
 506                 if (isatty_flag)
 507                         fflush(stdout);
 508                 else
 509                         printf("\n");
 510         }
 511
 512         last_time = curr_time;
 513         last_bytes = curr_bytes;
 514 }
 515
 516 /*
 517  * dir_write: This function writes directories and files on device.
 518  * it works for both full and partial modes.
 519  */
 520 static int dir_write(char *chunk_buf, size_t chunksize,
 521                      time_t time_st, unsigned long dir_num)
 522 {
 523         char tempfile[PATH_MAX];
 524         char tempdir[PATH_MAX];
 525         FILE *countfile;
 526         struct stat64 file;
 527         int file_num = 999999999;
 528         ino_t inode_st = 0;
 529         struct timeval start_time;
 530         unsigned long long total_bytes;
 531         unsigned long long curr_bytes = 0;
 532
 533 #ifdef HAVE_EXT2FS_EXT2FS_H
 534         if (!full && fsetflags(testdir, EXT2_TOPDIR_FL))
 535                 fprintf(stderr,
 536                         "\n%s: can't set TOPDIR_FL on %s: %s (ignoring)",
 537                         progname, testdir, strerror(errno));
 538 #endif
 539         countfile = fopen(filecount, "w");
 540         if (countfile == NULL) {
 541                 fprintf(stderr, "\n%s: creating %s failed :%s\n",
 542                         progname, filecount, strerror(errno));
 543                 return 5;
 544         }
 545         /* reserve space for the countfile */
 546         if (fprintf(countfile, "%lu", num_files) < 1 ||
 547             fflush(countfile) != 0) {
 548                 fprintf(stderr, "\n%s: writing %s failed :%s\n",
 549                         progname, filecount, strerror(errno));
 550                 return 6;
 551         }
 552
 553         /* calculate total bytes that need to be written */
 554         total_bytes = calc_total_bytes("write");
 555         if (total_bytes <= 0) {
 556                 fprintf(stderr, "\n%s: unable to calculate total bytes\n",
 557                         progname);
 558                 return 7;
 559         }
 560
 561         if (!full && (dir_num != 0))
 562                 total_bytes -= dir_num * files_in_dir * file_size;
 563
 564         gettimeofday(&start_time, NULL);
 565         for (; dir_num < num_dirs; num_files++, file_num++) {
 566                 int fd, ret;
 567
 568                 if (file_num >= files_in_dir) {
 569                         if (dir_num == num_dirs - 1)
 570                                 break;
 571
 572                         file_num = 0;
 573                         if (mkdir(new_dir(tempdir, dir_num), dirmode) < 0) {
 574                                 if (errno == ENOSPC)
 575                                         break;
 576                                 if (errno != EEXIST) {
 577                                         fprintf(stderr, "\n%s: mkdir %s : %s\n",
 578                                                 progname, tempdir,
 579                                                 strerror(errno));
 580                                         return 1;
 581                                 }
 582                         }
 583                         dir_num++;
 584                 }
 585
 586                 fd = open_file(new_file(tempfile, tempdir, file_num),
 587                                O_WRONLY | O_CREAT | O_TRUNC | O_LARGEFILE);
 588                 if (fd >= 0) {
 589                         if (fstat64(fd, &file) == 0) {
 590                                 inode_st = file.st_ino;
 591                         } else {
 592                                 fprintf(stderr, "\n%s: write stat '%s': %s",
 593                                         progname, tempfile, strerror(errno));
 594                                 close(fd);
 595                                 break;
 596                         }
 597                 } else {
 598                         break;
 599                 }
 600
 601                 ret = write_chunks(fd, 0, file_size, chunk_buf, chunksize,
 602                                    time_st, inode_st, tempfile);
 603                 close(fd);
 604                 if (ret < 0) {
 605                         if (ret != -ENOSPC)
 606                                 return 1;
 607                         curr_bytes = total_bytes;
 608                         break;
 609                 }
 610
 611                 curr_bytes += file_size;
 612                 if (verbose > 1)
 613                         show_rate("write", tempfile, &start_time,
 614                                   total_bytes, curr_bytes);
 615
 616                 fseek(countfile, 0, SEEK_SET);
 617                 if (fprintf(countfile, "%lu", num_files) < 1 ||
 618                     fflush(countfile) != 0) {
 619                         fprintf(stderr, "\n%s: writing %s failed :%s\n",
 620                                 progname, filecount, strerror(errno));
 621                 }
 622         }
 623         fclose(countfile);
 624
 625         if (verbose) {
 626                 verbose++;
 627                 show_rate("write", tempfile, &start_time,
 628                           total_bytes, curr_bytes);
 629                 printf("\nwrite complete\n");
 630                 verbose--;
 631         }
 632
 633         return 0;
 634 }
 635
 636 /*
 637  * dir_read: This function reads directories and files on device.
 638  * it works for both full and partial modes.
 639  */
 640 static int dir_read(char *chunk_buf, size_t chunksize,
 641                     time_t time_st, unsigned long dir_num)
 642 {
 643         char tempfile[PATH_MAX];
 644         char tempdir[PATH_MAX];
 645         unsigned long count = 0;
 646         struct stat64 file;
 647         int file_num = 0;
 648         ino_t inode_st = 0;
 649         struct timeval start_time;
 650         unsigned long long total_bytes;
 651         unsigned long long curr_bytes = 0;
 652
 653         /* calculate total bytes that need to be read */
 654         total_bytes = calc_total_bytes("read");
 655         if (total_bytes <= 0) {
 656                 fprintf(stderr, "\n%s: unable to calculate total bytes\n",
 657                         progname);
 658                 return 1;
 659         }
 660
 661         if (dir_num != 0)
 662                 total_bytes -= dir_num * files_in_dir * file_size;
 663
 664         gettimeofday(&start_time, NULL);
 665         for (count = 0; count < num_files && dir_num < num_dirs; count++) {
 666                 int fd, ret;
 667
 668                 if (file_num == 0) {
 669                         if (dir_num == num_dirs - 1)
 670                                 break;
 671
 672                         new_dir(tempdir, dir_num);
 673                         dir_num++;
 674                 }
 675
 676                 fd = open_file(new_file(tempfile, tempdir, file_num),
 677                                O_RDONLY | O_LARGEFILE);
 678                 if (fd >= 0) {
 679                         if (fstat64(fd, &file) == 0) {
 680                                 inode_st = file.st_ino;
 681                         } else {
 682                                 fprintf(stderr, "\n%s: read stat '%s': %s\n",
 683                                         progname, tempfile, strerror(errno));
 684                                 close(fd);
 685                                 return 1;
 686                         }
 687                 } else {
 688                         break;
 689                 }
 690
 691                 if (count == num_files)
 692                         file_size = file.st_size;
 693                 ret = read_chunks(fd, 0, file_size, chunk_buf, chunksize,
 694                                   time_st, inode_st, tempfile);
 695                 close(fd);
 696                 if (ret)
 697                         return 1;
 698
 699                 curr_bytes += file_size;
 700                 if (verbose > 1)
 701                         show_rate("read", tempfile, &start_time,
 702                                   total_bytes, curr_bytes);
 703
 704                 if (++file_num >= files_in_dir)
 705                         file_num = 0;
 706         }
 707         if (verbose > 1){
 708                 verbose++;
 709                 show_rate("read", tempfile, &start_time,
 710                           total_bytes, curr_bytes);
 711                 printf("\nread complete\n");
 712                 verbose--;
 713         }
 714         return 0;
 715 }
 716
 717 int main(int argc, char **argv)
 718 {
 719         time_t time_st = 0;             /* Default timestamp */
 720         size_t chunksize = ONE_MB;      /* IO chunk size(defailt=1MB) */
 721         char *chunk_buf;                /* chunk buffer */
 722         int error = 0;
 723         FILE *countfile = NULL;
 724         unsigned long dir_num = 0, dir_num_orig = 0;/* starting directory */
 725         int c;
 726
 727         progname = strrchr(argv[0], '/') ? strrchr(argv[0], '/') + 1 : argv[0];
 728         while ((c = getopt_long(argc, argv, "c:hlo:pqrs:t:vw",
 729                                       longopts, NULL)) != -1) {
 730                 switch (c) {
 731                 case 'c':
 732                         chunksize = strtoul(optarg, NULL, 0) * ONE_MB;
 733                         if (chunksize == 0) {
 734                                 fprintf(stderr, "%s: bad chunk size '%s'\n",
 735                                         optarg, progname);
 736                                 return -1;
 737                         }
 738                         break;
 739                 case 'l':
 740                         full = 1;
 741                         break;
 742                 case 'o': /* offset */
 743                         dir_num = strtoul(optarg, NULL, 0);
 744                         break;
 745                 case 'p':
 746                         file_size = ONE_MB;
 747                         chunksize = ONE_MB;
 748                         files_in_dir = 1;
 749                         full = 0;
 750                         break;
 751                 case 'q':
 752                         verbose = 0;
 753                         break;
 754                 case 'r':
 755                         readoption = 1;
 756                         break;
 757                 case 's':
 758                         file_size = strtoul(optarg, NULL, 0) * ONE_MB;
 759                         if (file_size == 0) {
 760                                 fprintf(stderr, "%s: bad file size '%s'\n",
 761                                         optarg, progname);
 762                                 return -1;
 763                         }
 764                         break;
 765                 case 't':
 766                         time_st = (time_t)strtoul(optarg, NULL, 0);
 767                         break;
 768                 case 'v':
 769                         verbose++;
 770                         break;
 771                 case 'w':
 772                         writeoption = 1;
 773                         break;
 774
 775                 case 'h':
 776                 default:
 777                         usage(1);
 778                         return 0;
 779                 }
 780         }
 781         testdir = argv[optind];
 782
 783         if (!testdir) {
 784                 fprintf(stderr, "%s: pathname not given\n", progname);
 785                 usage(1);
 786                 return -1;
 787         }
 788         if (!readoption && !writeoption) {
 789                 readoption = 1;
 790                 writeoption = 1;
 791         }
 792         if (!time_st)
 793                 (void) time(&time_st);
 794         printf("Timestamp: %lu\n", (unsigned long )time_st);
 795         isatty_flag = isatty(STDOUT_FILENO);
 796
 797         if (!full) {
 798 #ifdef HAVE_EXT2FS_EXT2FS_H
 799                 struct mntent *tempmnt;
 800                 FILE *fp = NULL;
 801                 ext2_filsys fs;
 802
 803                 if ((fp = setmntent("/etc/mtab", "r")) == NULL){
 804                         fprintf(stderr, "%s: fail to open /etc/mtab in read"
 805                                 "mode :%s\n", progname, strerror(errno));
 806                         goto guess;
 807                 }
 808
 809                 /* find device name using filesystem */
 810                 while ((tempmnt = getmntent(fp)) != NULL) {
 811                         if (strcmp(tempmnt->mnt_dir, testdir) == 0)
 812                                 break;
 813                 }
 814
 815                 if (tempmnt == NULL) {
 816                         fprintf(stderr, "%s: no device found for '%s'\n",
 817                                 progname, testdir);
 818                         endmntent(fp);
 819                         goto guess;
 820                 }
 821
 822                 if (ext2fs_open(tempmnt->mnt_fsname, 0, 0, 0,
 823                                 unix_io_manager, &fs)) {
 824                         fprintf(stderr, "%s: unable to open ext3 fs on '%s'\n",
 825                                 progname, testdir);
 826                         endmntent(fp);
 827                         goto guess;
 828                 }
 829                 endmntent(fp);
 830
 831                 num_dirs = (fs->super->s_blocks_count +
 832                             fs->super->s_blocks_per_group - 1) /
 833                         fs->super->s_blocks_per_group;
 834                 if (verbose)
 835                         printf("ext3 block groups: %u, fs blocks: %u "
 836                                "blocks per group: %u\n",
 837                                num_dirs, fs->super->s_blocks_count,
 838                                fs->super->s_blocks_per_group);
 839                 ext2fs_close(fs);
 840 #else
 841                 goto guess;
 842 #endif
 843                 if (0) { /* ugh */
 844                         struct statfs64 statbuf;
 845 guess:
 846                         if (statfs64(testdir, &statbuf) == 0) {
 847                                 num_dirs = (long long)statbuf.f_blocks *
 848                                         statbuf.f_bsize / (128ULL << 20);
 849                                 if (verbose)
 850                                         printf("dirs: %u, fs blocks: %llu\n",
 851                                                num_dirs,
 852                                                (long long)statbuf.f_blocks);
 853                         } else {
 854                                 fprintf(stderr, "%s: unable to stat '%s': %s\n",
 855                                         progname, testdir, strerror(errno));
 856                                 if (verbose)
 857                                         printf("dirs: %u\n", num_dirs);
 858                         }
 859                 }
 860         }
 861         chunk_buf = (char *)calloc(chunksize, 1);
 862         if (chunk_buf == NULL) {
 863                 fprintf(stderr, "Memory allocation failed for chunk_buf\n");
 864                 return 4;
 865         }
 866         sprintf(filecount, "%s/%s.filecount", testdir, progname);
 867         if (writeoption) {
 868                 (void)mkdir(testdir, dirmode);
 869
 870                 unlink(filecount);
 871                 if (dir_num != 0) {
 872                         num_files = dir_num * files_in_dir;
 873                         if (verbose)
 874                                 printf("\n%s: %lu files already written\n",
 875                                        progname, num_files);
 876                 }
 877                 if (dir_write(chunk_buf, chunksize, time_st, dir_num)) {
 878                         error = 3;
 879                         goto out;
 880                 }
 881                 dir_num = dir_num_orig;
 882         }
 883         if (readoption) {
 884                 if (!writeoption) {
 885                         countfile = fopen(filecount, "r");
 886                         if (countfile == NULL ||
 887                             fscanf(countfile, "%lu", &num_files) != 1 ||
 888                             num_files == 0) {
 889                                 fprintf(stderr, "\n%s: reading %s failed :%s\n",
 890                                         progname, filecount, strerror(errno));
 891                                 num_files = num_dirs * files_in_dir;
 892                         } else {
 893                                 num_files -= (dir_num * files_in_dir);
 894                         }
 895                         if (countfile)
 896                                 fclose(countfile);
 897                 }
 898                 if (dir_read(chunk_buf, chunksize, time_st, dir_num)) {
 899                         fprintf(stderr, "\n%s: Data verification failed\n",
 900                                 progname) ;
 901                         error = 2;
 902                         goto out;
 903                 }
 904         }
 905         error = error_count;
 906 out:
 907         free(chunk_buf);
 908         return error;
 909 }