lustre/utils/llverfs.c

   1 /*
   2  * ext3 Filesystem Verification Tool.
   3  * This program tests the correct operation of ext3 filesystem.
   4  * This tool have two working modes
   5  * 1. full mode
   6  * 2. fast mode
   7  *      The full mode is basic mode in which program creates a subdirectory
   8  * in the test fileysytem, writes n(files_in_dir, default=16) large(4GB) files
   9  * to the directory with the test pattern at the start of each 4kb block.
  10  * The test pattern contains timestamp, relative file offset and per file
  11  * unique idenfifier(inode number). this continues until whole filesystem is
  12  * full and then this tooll verifies that the data in all of the test files
  13  * is correct.
  14  *      In the fast mode the tool creates a test directories with
  15  * EXT3_TOPDIR_FL flag set. the number of directories equals to the number
  16  * of block groups in the filesystem(e.g. 65536 directories for 8TB filesystem)
  17  * and then writes a single 1MB file in each directory. The tool then verifies
  18  * that the data in each file is correct.
  19  */
  20
  21 #ifndef _GNU_SOURCE
  22 #define _GNU_SOURCE
  23 #endif
  24 #ifndef LUSTRE_UTILS
  25 #define LUSTRE_UTILS
  26 #endif
  27 #ifndef _LARGEFILE64_SOURCE
  28 #define _LARGEFILE64_SOURCE
  29 #endif
  30 #ifndef _FILE_OFFSET_BITS
  31 #define _FILE_OFFSET_BITS 64
  32 #endif
  33
  34 #include <features.h>
  35 #include <stdlib.h>
  36 #include <stdio.h>
  37 #include <string.h>
  38 #include <ctype.h>
  39 #include <fcntl.h>
  40 #include <unistd.h>
  41 #include <limits.h>
  42 #include <errno.h>
  43 #include <fcntl.h>
  44 #include <getopt.h>
  45 #include <time.h>
  46 #include <dirent.h>
  47 #include <mntent.h>
  48 #include <sys/types.h>
  49 #include <sys/stat.h>
  50 #include <sys/vfs.h>
  51 #include <gnu/stubs.h>
  52 #include <gnu/stubs.h>
  53
  54 #ifdef HAVE_EXT2FS_EXT2FS_H
  55 #  include <e2p/e2p.h>
  56 #  include <ext2fs/ext2fs.h>
  57 #endif
  58
  59 #define ONE_MB (1024 * 1024)
  60 #define ONE_GB ((unsigned long long)(1024 * 1024 * 1024))
  61 #define BLOCKSIZE 4096
  62
  63 /* Structure for writing test pattern */
  64 struct block_data {
  65         unsigned long long bd_offset;
  66         unsigned long long bd_time;
  67         unsigned long long bd_inode;
  68 };
  69 static char *progname;              /* name by which this program was run. */
  70 static unsigned verbose = 1;        /* prints offset in kB, operation rate */
  71 static int readoption;              /* run test in read-only (verify) mode */
  72 static int writeoption;             /* run test in write_only mode */
  73 char *testdir;                      /* name of device to be tested. */
  74 static unsigned full = 1;           /* flag to full check */
  75 static int errno_local;             /* local copy of errno */
  76 static unsigned long num_files;     /* Total number of files for read/write */
  77 static loff_t file_size = 4*ONE_GB; /* Size of each file */
  78 static unsigned files_in_dir = 32;  /* number of files in each directioy */
  79 static unsigned num_dirs = 30000;   /* total number of directories */
  80 const int dirmode = S_IRWXU | S_IRGRP | S_IXGRP | S_IROTH | S_IXOTH;
  81 static int fd = -1;
  82 static int isatty_flag;
  83 static int perms =  S_IRWXU | S_IRGRP | S_IROTH;
  84
  85 static struct option const longopts[] =
  86 {
  87         { "chunksize", required_argument, 0, 'c' },
  88         { "help", no_argument, 0, 'h' },
  89         { "offset", required_argument, 0, 'o' },
  90         { "long", no_argument, 0, 'l' },
  91         { "partial", required_argument, 0, 'p' },
  92         { "quiet", required_argument, 0, 'q' },
  93         { "read", no_argument, 0, 'r' },
  94         { "timestamp", required_argument, 0, 't' },
  95         { "verbose", no_argument, 0, 'v' },
  96         { "write", no_argument, 0, 'w' },
  97         { 0, 0, 0, 0}
  98 };
  99
 100 /*
 101  * Usages: displays help information, whenever user supply --help option in
 102  * command or enters incorrect command line.
 103  */
 104 void usage(int status)
 105 {
 106         if (status != 0)
 107         {
 108               printf("\nUsage: %s [OPTION]... <filesystem path> ...\n",
 109                         progname);
 110               printf("ext3 filesystem verification tool.\n"
 111                   "\t-t {seconds} for --timestamp,  set test time"
 112                   "(default=current time())\n"
 113                   "\t-o {offset}  for --offset, directory starting offset"
 114                   " from which tests should start\n"
 115                   "\t-r run test in read (verify) mode\n"
 116                   "\t-w run test in write (test-pattern) mode (default=r&w)\n"
 117                   "\t-v for verbose\n"
 118                   "\t-p for --partial, for partial check (1MB files)\n"
 119                   "\t-l for --long, full check (4GB file with 4k blocks)\n"
 120                   "\t-c for --chunksize, IO chunk size (default=1048576)\n"
 121                   "\t-h display this help and exit\n"
 122                   "\t--help display this help and exit\n");
 123         }
 124         exit(status);
 125 }
 126
 127 /*
 128  * open_file: Opens file in specified mode and returns fd.
 129  */
 130 static int open_file(const char *file, int flag)
 131 {
 132         fd = open(file, flag, perms);
 133         if (fd < 0) {
 134                 fprintf(stderr, "\n%s: Open '%s' failed:%s\n",
 135                         progname, file, strerror(errno));
 136                 exit(3);
 137         }
 138         return (fd);
 139 }
 140
 141 /*
 142  * Verify_chunk: Verifies test pattern in each 4kB (BLOCKSIZE) is correct.
 143  * Returns 0 if test offset and timestamp is correct otherwise 1.
 144  */
 145 int verify_chunk(char *chunk_buf, size_t chunksize,unsigned long long chunk_off,
 146                  unsigned long long time_st, unsigned long long inode_st,
 147                  char *file)
 148 {
 149         struct block_data *bd;
 150         char *chunk_end;
 151
 152         for (chunk_end = chunk_buf + chunksize - sizeof(*bd);
 153              (char *)chunk_buf < chunk_end;
 154              chunk_buf += BLOCKSIZE, chunk_off += BLOCKSIZE) {
 155                 bd = (struct block_data *)chunk_buf;
 156                 if ((bd->bd_offset == chunk_off) && (bd->bd_time == time_st) &&
 157                     (bd->bd_inode == inode_st))
 158                         continue;
 159                 fprintf(stderr,"\n%s: verify %s failed offset/timestamp/inode "
 160                         "%llu/%llu/%llu: found %llu/%llu/%llu instead\n",
 161                         progname, file, chunk_off, time_st, inode_st,
 162                         bd->bd_offset, bd->bd_time, bd->bd_inode);
 163                 return 1;
 164         }
 165         return 0;
 166 }
 167
 168 /*
 169  * fill_chunk: Fills the chunk with current or user specified timestamp
 170  * and  offset. The test patters is filled at the beginning of
 171  * each 4kB(BLOCKSIZE) blocks in chunk_buf.
 172  */
 173 void fill_chunk(char *chunk_buf, size_t chunksize, loff_t chunk_off,
 174                 time_t time_st, ino_t inode_st)
 175 {
 176         struct block_data *bd;
 177         char *chunk_end;
 178
 179         for (chunk_end = chunk_buf + chunksize - sizeof(*bd);
 180              (char *)chunk_buf < chunk_end;
 181              chunk_buf += BLOCKSIZE, chunk_off += BLOCKSIZE) {
 182                 bd = (struct block_data *)chunk_buf;
 183                 bd->bd_offset = chunk_off;
 184                 bd->bd_time = time_st;
 185                 bd->bd_inode = inode_st;
 186         }
 187 }
 188
 189 /*
 190  * write_chunk: write the chunk_buf on the device. The number of write
 191  * operations are based on the parameters write_end, offset, and chunksize.
 192  */
 193 int write_chunks(int fd, unsigned long long offset,unsigned long long write_end,
 194                  char *chunk_buf, size_t chunksize, time_t time_st,
 195                  ino_t inode_st, const char *file)
 196 {
 197         unsigned long long stride;
 198
 199         stride = full ? chunksize : (ONE_GB - chunksize);
 200         for (offset = offset & ~(chunksize - 1); offset < write_end;
 201              offset += stride) {
 202                 if (lseek64(fd, offset, SEEK_SET) == -1) {
 203                         fprintf(stderr, "\n%s: lseek64(%s+%llu) failed: %s\n",
 204                                 progname, file, offset, strerror(errno));
 205                         return 1;
 206                 }
 207                 if (offset + chunksize > write_end)
 208                         chunksize = write_end - offset;
 209                 if (!full && offset > chunksize) {
 210                         fill_chunk(chunk_buf, chunksize, offset, time_st,
 211                                     inode_st);
 212                         if (write(fd, chunk_buf, chunksize) < 0) {
 213                                 if (errno == ENOSPC) {
 214                                         errno_local = errno;
 215                                         return 0;
 216                                 }
 217                                 fprintf(stderr,
 218                                         "\n%s: write %s+%llu failed: %s\n",
 219                                         progname, file, offset,strerror(errno));
 220                                 return errno;
 221                         }
 222                         offset += chunksize;
 223                         if (offset + chunksize > write_end)
 224                                 chunksize = write_end - offset;
 225                 }
 226                 fill_chunk(chunk_buf, chunksize, offset, time_st, inode_st);
 227                 if (write(fd, (char *) chunk_buf, chunksize) < 0) {
 228                         if (errno == ENOSPC) {
 229                                 errno_local = errno;
 230                                 return 0;
 231                         }
 232                         fprintf(stderr, "\n%s: write %s+%llu failed: %s\n",
 233                                 progname, file, offset, strerror(errno));
 234                         return 1;
 235                 }
 236         }
 237         return 0;
 238 }
 239
 240 /*
 241  * read_chunk: reads the chunk_buf from the device. The number of read
 242  * operations are based on the parameters read_end, offset, and chunksize.
 243  */
 244 int read_chunks(int fd, unsigned long long offset, unsigned long long read_end,
 245                 char *chunk_buf, size_t chunksize, time_t time_st,
 246                 ino_t inode_st, char *file)
 247 {
 248         unsigned long long stride;
 249
 250         stride = full ? chunksize : (ONE_GB - chunksize);
 251         for (offset = offset & ~(chunksize - 1); offset < read_end;
 252              offset += stride) {
 253                 if (lseek64(fd, offset, SEEK_SET) == -1) {
 254                         fprintf(stderr, "\n%s: lseek64(%s+%llu) failed: %s\n",
 255                                 progname, file, offset, strerror(errno));
 256                         return 1;
 257                 }
 258                 if (offset + chunksize > read_end)
 259                         chunksize = read_end - offset;
 260                 if (!full && offset > chunksize) {
 261                         if (read(fd, chunk_buf, chunksize) < 0) {
 262                                 fprintf(stderr,
 263                                         "\n%s: read %s+%llu failed: %s\n",
 264                                         progname, file, offset,strerror(errno));
 265                                 return 1;
 266                         }
 267                         if (verify_chunk(chunk_buf, chunksize, offset,
 268                                          time_st, inode_st, file) != 0)
 269                                 return 1;
 270                         offset += chunksize;
 271                         if (offset + chunksize >= read_end)
 272                                 chunksize = read_end - offset;
 273                 }
 274                 if (read(fd, chunk_buf, chunksize) < 0) {
 275                         fprintf(stderr, "\n%s: read %s+%llu failed: %s\n",
 276                                 progname, file, offset, strerror(errno));
 277                         return 1;
 278                 }
 279                 if (verify_chunk(chunk_buf, chunksize, offset, time_st,
 280                                  inode_st, file) != 0)
 281                         return 1;
 282         }
 283         return 0;
 284 }
 285
 286 /*
 287  * new_file: prepares new filename using file counter and current dir.
 288  */
 289 char *new_file(char *tempfile, char *cur_dir, int file_num)
 290 {
 291         sprintf(tempfile, "%s/file%03d", cur_dir, file_num);
 292         return tempfile;
 293 }
 294
 295 /*
 296  * new_dir: prepares new dir name using dir counters.
 297  */
 298 char *new_dir(char *tempdir, int dir_num)
 299 {
 300         sprintf(tempdir, "%s/dir%05d", testdir, dir_num);
 301         return tempdir;
 302 }
 303
 304 /*
 305  * show_filename: Displays name of current file read/write
 306  */
 307 void show_filename(char *op, char *filename)
 308 {
 309         static time_t last;
 310         time_t now;
 311         double diff;
 312
 313         now = time(NULL);
 314         diff = now - last;
 315         if (diff > 4 || verbose > 2) {
 316                 if (isatty_flag)
 317                         printf("\r");
 318                 printf("%s File name: %s          ", op, filename);
 319                 if (isatty_flag)
 320                         fflush(stdout);
 321                 else
 322                         printf("\n");
 323                 last = now;
 324         }
 325 }
 326
 327 /*
 328  * dir_write: This function writes directories and files on device.
 329  * it works for both full and fast modes.
 330  */
 331 static int dir_write(char *chunk_buf, size_t chunksize,
 332                      time_t time_st, unsigned long dir_num)
 333 {
 334         char tempfile[PATH_MAX];
 335         char tempdir[PATH_MAX];
 336         struct stat64 file;
 337         int file_num = 999999999;
 338         ino_t inode_st = 0;
 339
 340 #ifdef HAVE_EXT2FS_EXT2FS_H
 341         if (!full && fsetflags(testdir, EXT2_TOPDIR_FL))
 342                 fprintf(stderr,
 343                         "\n%s: can't set TOPDIR_FL on %s: %s (ignoring)",
 344                         progname, testdir, strerror(errno));
 345 #endif
 346         for (; dir_num < num_dirs; num_files++, file_num++) {
 347                 if (file_num >= files_in_dir) {
 348                         if (dir_num == num_dirs - 1)
 349                                 break;
 350
 351                         file_num = 0;
 352                         if (mkdir(new_dir(tempdir, dir_num), dirmode) < 0) {
 353                                 if (errno == ENOSPC)
 354                                         break;
 355                                 if (errno != EEXIST) {
 356                                         fprintf(stderr, "\n%s: mkdir %s : %s\n",
 357                                                 progname, tempdir,
 358                                                 strerror(errno));
 359                                         return 1;
 360                                 }
 361                         }
 362                         dir_num++;
 363                 }
 364                 fd = open_file(new_file(tempfile, tempdir, file_num),
 365                                O_WRONLY | O_CREAT | O_TRUNC | O_LARGEFILE);
 366
 367                 if (fd >= 0 && fstat64(fd, &file) == 0) {
 368                         inode_st = file.st_ino;
 369                 } else {
 370                         fprintf(stderr, "\n%s: write stat64 to file %s: %s",
 371                                 progname, tempfile, strerror(errno));
 372                         exit(1);
 373                 }
 374
 375                 if (verbose > 1)
 376                         show_filename("write", tempfile);
 377
 378                 if (write_chunks(fd, 0, file_size, chunk_buf, chunksize,
 379                                  time_st, inode_st, tempfile)) {
 380                         close(fd);
 381                         return 1;
 382                 }
 383                 close(fd);
 384
 385                 if (errno_local == ENOSPC)
 386                         break;
 387         }
 388
 389         if (verbose) {
 390                 verbose++;
 391                 show_filename("write", tempfile);
 392                 printf("\nwrite complete\n");
 393                 verbose--;
 394         }
 395
 396         return 0;
 397 }
 398
 399 /*
 400  * dir_read: This function reads directories and files on device.
 401  * it works for both full and fast modes.
 402  */
 403 static int dir_read(char *chunk_buf, size_t chunksize,
 404                     time_t time_st, unsigned long dir_num)
 405 {
 406         char tempfile[PATH_MAX];
 407         char tempdir[PATH_MAX];
 408         unsigned long count = 0;
 409         struct stat64 file;
 410         int file_num = 0;
 411         ino_t inode_st = 0;
 412
 413         for (count = 0; count < num_files && dir_num < num_dirs; count++) {
 414                 if (file_num == 0) {
 415                         if (dir_num == num_dirs - 1)
 416                                 break;
 417
 418                         new_dir(tempdir, dir_num);
 419                         dir_num++;
 420                 }
 421
 422                 fd = open_file(new_file(tempfile, tempdir, file_num),
 423                                O_RDONLY | O_LARGEFILE);
 424                 if (fd >= 0 && fstat64(fd, &file) == 0) {
 425                         inode_st = file.st_ino;
 426                 } else {
 427                         fprintf(stderr, "\n%s: read stat64 file '%s': %s\n",
 428                                 progname, tempfile, strerror(errno));
 429                         return 1;
 430                 }
 431
 432                 if (verbose > 1)
 433                         show_filename("read", tempfile);
 434
 435                 if (count == num_files)
 436                         file_size = file.st_size;
 437                 if (read_chunks(fd, 0, file_size, chunk_buf, chunksize,
 438                                 time_st, inode_st, tempfile)) {
 439                         close(fd);
 440                         return 1;
 441                 }
 442                 close(fd);
 443
 444                 if (++file_num >= files_in_dir)
 445                         file_num = 0;
 446         }
 447         if (verbose > 1){
 448                 verbose++;
 449                 show_filename("read", tempfile);
 450                 printf("\nread complete\n");
 451                 verbose--;
 452         }
 453         return 0;
 454 }
 455
 456 int main(int argc, char **argv)
 457 {
 458         time_t time_st = 0;             /* Default timestamp */
 459         size_t chunksize = ONE_MB;      /* IO chunk size(defailt=1MB) */
 460         char *chunk_buf;                /* chunk buffer */
 461         int error = 0;
 462         FILE *countfile = NULL;
 463         char filecount[PATH_MAX];
 464         unsigned long dir_num = 0, dir_num_orig = 0;/* starting directory */
 465         char c;
 466
 467         progname = strrchr(argv[0], '/') ? strrchr(argv[0], '/') + 1 : argv[0];
 468         while ((c = (char)getopt_long(argc, argv, "t:rwvplo:h",
 469                                       longopts, NULL)) != -1) {
 470                 switch (c) {
 471                 case 'c':
 472                         chunksize = (strtoul(optarg, NULL, 0) * ONE_MB);
 473                         if (!chunksize) {
 474                                 fprintf(stderr, "%s: Chunk size value should be"
 475                                         "a multiple of 1MB\n", progname);
 476                                 return -1;
 477                         }
 478                         break;
 479                 case 'l':
 480                         full = 1;
 481                         break;
 482                 case 'o': /* offset */
 483                         dir_num = strtoul(optarg, NULL, 0);
 484                         break;
 485                 case 'p':
 486                         full = 0;
 487                         break;
 488                 case 'q':
 489                         verbose = 0;
 490                         break;
 491                 case 'r':
 492                         readoption = 1;
 493                         break;
 494                 case 't':
 495                         time_st = (time_t)strtoul(optarg, NULL, 0);
 496                         break;
 497                 case 'w':
 498                         writeoption = 1;
 499                         break;
 500                 case 'v':
 501                         verbose++;
 502                         break;
 503
 504                 case 'h':
 505                 default:
 506                         usage(1);
 507                         return 0;
 508                 }
 509         }
 510         testdir = argv[optind];
 511
 512         if (!testdir) {
 513                 fprintf(stderr, "%s: pathname not given\n", progname);
 514                 usage(1);
 515                 return -1;
 516         }
 517         if (!readoption && !writeoption) {
 518                 readoption = 1;
 519                 writeoption = 1;
 520         }
 521         if (!time_st)
 522                 (void) time(&time_st);
 523         printf("Timestamp: %lu\n", (unsigned long )time_st);
 524         isatty_flag = isatty(STDOUT_FILENO);
 525
 526         if (!full) {
 527 #ifdef HAVE_EXT2FS_EXT2FS_H
 528                 struct mntent *tempmnt;
 529                 FILE *fp = NULL;
 530                 ext2_filsys fs;
 531
 532                 if ((fp = setmntent("/etc/mtab", "r")) == NULL){
 533                         fprintf(stderr, "%s: fail to open /etc/mtab in read"
 534                                 "mode :%s\n", progname, strerror(errno));
 535                         goto guess;
 536                 }
 537
 538                 /* find device name using filesystem */
 539                 while ((tempmnt = getmntent(fp)) != NULL) {
 540                         if (strcmp(tempmnt->mnt_dir, testdir) == 0)
 541                                 break;
 542                 }
 543
 544                 if (tempmnt == NULL) {
 545                         fprintf(stderr, "%s: no device found for '%s'\n",
 546                                 progname, testdir);
 547                         endmntent(fp);
 548                         goto guess;
 549                 }
 550
 551                 if (ext2fs_open(tempmnt->mnt_fsname, 0, 0, 0,
 552                                 unix_io_manager, &fs)) {
 553                         fprintf(stderr, "%s: unable to open ext3 fs on '%s'\n",
 554                                 progname, testdir);
 555                         endmntent(fp);
 556                         goto guess;
 557                 }
 558                 endmntent(fp);
 559
 560                 num_dirs = (fs->super->s_blocks_count +
 561                             fs->super->s_blocks_per_group - 1) /
 562                         fs->super->s_blocks_per_group;
 563                 if (verbose)
 564                         printf("ext3 block groups: %u, fs blocks: %u "
 565                                "blocks per group: %u\n",
 566                                num_dirs, fs->super->s_blocks_count,
 567                                fs->super->s_blocks_per_group);
 568                 ext2fs_close(fs);
 569 #else
 570                 goto guess;
 571 #endif
 572                 if (0) { /* ugh */
 573                         struct statfs64 statbuf;
 574                 guess:
 575                         if (statfs64(testdir, &statbuf) == 0) {
 576                                 num_dirs = (long long)statbuf.f_blocks *
 577                                         statbuf.f_bsize / (128ULL << 20);
 578                                 if (verbose)
 579                                         printf("dirs: %u, fs blocks: %llu\n",
 580                                                num_dirs,
 581                                                (long long)statbuf.f_blocks);
 582                         } else {
 583                                 fprintf(stderr, "%s: unable to stat '%s': %s\n",
 584                                         progname, testdir, strerror(errno));
 585                                 if (verbose)
 586                                         printf("dirs: %u\n", num_dirs);
 587                         }
 588                 }
 589
 590                 file_size = ONE_MB;
 591                 chunksize = ONE_MB;
 592                 files_in_dir = 1;
 593         }
 594         chunk_buf = (char *)calloc(chunksize, 1);
 595         if (chunk_buf == NULL) {
 596                 fprintf(stderr, "Memory allocation failed for chunk_buf\n");
 597                 return 4;
 598         }
 599         sprintf(filecount, "%s/%s.filecount", testdir, progname);
 600         if (writeoption) {
 601                 (void)mkdir(testdir, dirmode);
 602
 603                 unlink(filecount);
 604                 if (dir_num != 0) {
 605                         num_files = dir_num * files_in_dir;
 606                         if (verbose)
 607                                 printf("\n%s: %lu files already written\n",
 608                                        progname, num_files);
 609                 }
 610                 if (dir_write(chunk_buf, chunksize, time_st, dir_num)) {
 611                         error = 3;
 612                         goto out;
 613                 }
 614                 countfile = fopen(filecount, "w");
 615                 if (countfile != NULL) {
 616                         if (fprintf(countfile, "%lu", num_files) < 1 ||
 617                             fflush(countfile) != 0) {
 618                                 fprintf(stderr, "\n%s: writing %s failed :%s\n",
 619                                         progname, filecount, strerror(errno));
 620                         }
 621                         fclose(countfile);
 622                 }
 623                 dir_num = dir_num_orig;
 624         }
 625         if (readoption) {
 626                 if (!writeoption) {
 627                         countfile = fopen(filecount, "r");
 628                         if (countfile == NULL ||
 629                             fscanf(countfile, "%lu", &num_files) != 1) {
 630                                 fprintf(stderr, "\n%s: reading %s failed :%s\n",
 631                                         progname, filecount, strerror(errno));
 632                                 num_files = num_dirs * files_in_dir;
 633                         } else {
 634                                 num_files -= (dir_num * files_in_dir);
 635                         }
 636                         if (countfile)
 637                                 fclose(countfile);
 638                 }
 639                 if (dir_read(chunk_buf, chunksize, time_st, dir_num)) {
 640                         fprintf(stderr, "\n%s: Data verification failed\n",
 641                                 progname) ;
 642                         error = 2;
 643                         goto out;
 644                 }
 645         }
 646         error = 0;
 647 out:
 648         free(chunk_buf);
 649         return error;
 650 }