lustre/tests/mpi/mdsrate.c

   1 /*
   2  * 2003, Copyright, Hewlett-Packard Development Compnay, LP.
   3  *
   4  * Developed under the sponsorship of the U.S. Government
   5  *     under Subcontract No. B514193
   6  */
   7
   8 /*
   9  * Copyright (c) 2008, 2010, Oracle and/or its affiliates. All rights reserved.
  10  * Use is subject to license terms.
  11  *
  12  * Copyright (c) 2012, Intel Corporation.
  13  */
  14
  15 #include <stdio.h>
  16 #include <getopt.h>
  17 #include <libgen.h>
  18 #include <sys/types.h>
  19 #include <sys/stat.h>
  20 #include <time.h>
  21 #include <limits.h>
  22 #include <errno.h>
  23 #include <string.h>
  24 #include <fcntl.h>
  25 #include <unistd.h>
  26 #include <stdlib.h>
  27 #include <stdarg.h>
  28 #include <signal.h>
  29 #include <sys/ioctl.h>
  30 #include <dirent.h>
  31
  32 #include "mpi.h"
  33
  34 /* lustre */
  35 #include <lustre/lustreapi.h>        /* for O_LOV_DELAY_CREATE */
  36
  37 #define CHECK_COUNT 10000
  38 #define DISPLAY_COUNT (CHECK_COUNT * 10)
  39 #define DISPLAY_TIME 100
  40
  41 enum {
  42         CREATE   = 'c',
  43         LOOKUP   = 'l',
  44         MKNOD    = 'm',
  45         OPEN     = 'o',
  46         STAT     = 's',
  47         UNLINK   = 'u',
  48         BEGIN    = 'b',
  49         ITERS    = 'i',
  50         TIME     = 't',
  51         DIRFMT   = 'd',
  52         NDIRS    = 'D',
  53         FILEFMT  = 'f',
  54         NFILES   = 'F',
  55         NOEXCL   = 'X',
  56         STRIPES  = 'S',
  57         SEED     = 'r',
  58         SEEDFILE = 'R',
  59         RANDOM   = 'A',
  60         READDIR  = 'B',
  61         RECREATE = 'C',
  62         IGNORE   = 'E',
  63         VERBOSE  = 'V',
  64         DEBUG    = 'v',
  65         HELP     = 'h',
  66         MNT      = 'M',
  67         MNTCOUNT = 'N',
  68         MDTCOUNT = 'T',
  69 };
  70
  71 struct option longOpts[] = {
  72         {"create",        0, NULL, CREATE     },
  73         {"lookup",        0, NULL, LOOKUP     },
  74         {"mknod",         0, NULL, MKNOD      },
  75         {"open",          0, NULL, OPEN       },
  76         {"stat",          0, NULL, STAT       },
  77         {"unlink",        0, NULL, UNLINK     },
  78         {"begin",         1, NULL, BEGIN      },
  79         {"iters",         1, NULL, ITERS      },
  80         {"time",          1, NULL, TIME       },   /* seconds */
  81         {"dirfmt",        1, NULL, DIRFMT     },
  82         {"ndirs",         1, NULL, NDIRS      },
  83         {"filefmt",       1, NULL, FILEFMT    },
  84         {"nfiles",        1, NULL, NFILES     },
  85         {"noexcl",        0, NULL, NOEXCL     },
  86         {"stripes",       1, NULL, STRIPES    },
  87         {"seed",          1, NULL, SEED       },
  88         {"seedfile",      1, NULL, SEEDFILE   },
  89         {"random_order",  0, NULL, RANDOM     },
  90         {"readdir_order", 0, NULL, READDIR    },
  91         {"recreate",      0, NULL, RECREATE   },
  92         {"ignore",        0, NULL, IGNORE     },
  93         {"verbose",       0, NULL, VERBOSE    },
  94         {"debug",         0, NULL, DEBUG      },
  95         {"help",          0, NULL, HELP       },
  96         {"mdtcount",      1, NULL, MDTCOUNT   },
  97         {"mntcount",      1, NULL, MNTCOUNT   },
  98         {"mntfmt",        1, NULL, MNT        },
  99         { 0,              0, NULL, 0          }
 100 };
 101
 102 int foo1, foo2;
 103
 104 char   shortOpts[128];
 105 int    myrank = -1;
 106 int    nthreads = -1;
 107 char * prog;
 108 char   hostname[512] = "unknown";
 109 char   mode;
 110 char * cmd;
 111 int    openflags = O_RDWR|O_CREAT|O_EXCL;
 112 int    ndirs = 1;
 113 char * dirfmt;
 114 char   dir[PATH_MAX];
 115 char   mkdir_cmd[PATH_MAX+14];
 116 int    dirthreads;
 117 int    dirnum;
 118 DIR *  directory;
 119 struct dirent *dir_entry;
 120 int    nfiles;
 121 char   filefmt[PATH_MAX];
 122 char   filename[PATH_MAX];
 123 char   path[PATH_MAX];
 124 int    stripes = -1;
 125 int    begin;
 126 int    beginsave;
 127 int    end;
 128 int    iters;
 129 int    seconds;
 130 int    alarm_caught;
 131 struct sigaction act;
 132 int    order = RANDOM;
 133 int    seed;
 134 int    recreate;
 135 int    ignore;
 136 int    verbose;
 137 int    debug;
 138 struct stat statbuf;
 139 int    mnt_count = -1;
 140 int    mdt_count = 1;
 141 char  *mntfmt;
 142
 143 #define dmesg if (debug) printf
 144
 145 #define DISPLAY_PROGRESS() {                                                \
 146         if (verbose && (nops % CHECK_COUNT == 0)) {                         \
 147                 curTime = MPI_Wtime();                                      \
 148                 interval = curTime - lastTime;                              \
 149                 if (interval > DISPLAY_TIME || nops % DISPLAY_COUNT == 0) { \
 150                         rate = (double)(nops - lastOps)/interval;           \
 151                         printf("Rank %d: %.2f %ss/sec %.2f secs "           \
 152                                "(total: %d %ss %.2f secs)\n",               \
 153                                myrank, rate, cmd, interval,                 \
 154                                nops, cmd, curTime - startTime);             \
 155                         lastOps = nops;                                     \
 156                         lastTime = curTime;                                 \
 157                 }                                                           \
 158         }                                                                   \
 159 }
 160
 161 char *usage_msg = "usage: %s\n"
 162                   "    { --create [ --noexcl ] | --lookup | --mknod |\n"
 163                   "      --open | --stat | --unlink  [ --recreate ] [ --ignore ] }\n"
 164                   "    [ --help ] [ --verbose ] [ --debug ]\n"
 165                   "    { [ --begin <num> ] --nfiles <num> }\n"
 166                   "    [ --iters <num> ] [ --time <secs> ]\n"
 167                   "    [ --dirfmt <str> ] [ --ndirs  <num> ]\n"
 168                   "    [ --filefmt <str> ] [ --stripes <num> ]\n"
 169                   "    [ --random_order [--seed <num> | --seedfile <file>] ]\n"
 170                   "    [ --readdir_order ] [ --mntfmt <str> ]\n"
 171                   "    [ --mntcount <num> ] [ --mdtcount <num> ]\n";
 172
 173 static void
 174 usage(FILE *stream, char *fmt, ...)
 175 {
 176         if (myrank == 0) {
 177                 if (fmt != NULL) {
 178                         va_list       ap;
 179
 180                         fprintf(stream, "%s: ", prog);
 181                         va_start(ap, fmt);
 182                         vfprintf(stderr, fmt, ap);
 183                         va_end(ap);
 184                 }
 185                 fprintf(stream, usage_msg, prog);
 186         }
 187
 188         MPI_Finalize();
 189         exit(stream == stderr);
 190 }
 191
 192 /* Print process myrank and message, and exit (i.e. a fatal error) */
 193 static int
 194 fatal(int rank, const char *fmt, ...)
 195 {
 196         if (rank == myrank) {
 197                 va_list       ap;
 198
 199                 fprintf(stderr, "rank %d: ", rank);
 200                 va_start(ap, fmt);
 201                 vfprintf(stderr, fmt, ap);
 202                 va_end(ap);
 203         }
 204
 205         MPI_Abort(MPI_COMM_WORLD, 1);
 206         exit(1);
 207 }
 208
 209 static void
 210 sigalrm_handler(int signum)
 211 {
 212         alarm_caught++;
 213 }
 214
 215 /* HAVE_LLAPI_FILE_LOOKUP is defined by liblustreapi.h if this function is
 216  * defined therein.  Otherwise we can do the equivalent operation via ioctl
 217  * if we have access to a complete lustre build tree to get the various
 218  * definitions - then compile with USE_MDC_LOOKUP defined. */
 219 #if defined(HAVE_LLAPI_FILE_LOOKUP)
 220 #define HAVE_MDC_LOOKUP
 221 #elif defined(USE_MDC_LOOKUP)
 222 #include <config.h>
 223 #include <liblustre.h>
 224 #include <linux/lustre_lib.h>
 225
 226 int llapi_file_lookup(int dirfd, const char *name)
 227 {
 228         struct obd_ioctl_data data = { 0 };
 229         char rawbuf[8192];
 230         char *buf = rawbuf;
 231         int rc;
 232
 233         if (dirfd < 0 || name == NULL)
 234                 return -EINVAL;
 235
 236         data.ioc_version = OBD_IOCTL_VERSION;
 237         data.ioc_len = sizeof(data);
 238         data.ioc_inlbuf1 = name;
 239         data.ioc_inllen1 = strlen(name) + 1;
 240
 241         rc = obd_ioctl_pack(&data, &buf, sizeof(rawbuf));
 242         if (rc) {
 243                 fatal(myrank, "ioctl_pack failed: rc = %d\n", rc);
 244                 return rc;
 245         }
 246
 247         return ioctl(fd, IOC_MDC_LOOKUP, buf);
 248 }
 249 #define HAVE_MDC_LOOKUP
 250 #endif
 251
 252 static void
 253 process_args(int argc, char *argv[])
 254 {
 255         char   *cp, *endptr;
 256         int    i, index, offset, tmpend, rc;
 257         char   tmp[16];
 258         FILE * seed_file;
 259         struct option *opt;
 260
 261         setbuf(stdout, 0);
 262         setbuf(stderr, 0);
 263         prog = basename(argv[0]);
 264         strcpy(filefmt, "f%d");
 265         gethostname(hostname, sizeof(hostname));
 266
 267         /* auto create shortOpts rather than maintaining a static string. */
 268         for (opt = longOpts, cp = shortOpts; opt->name != NULL; opt++, cp++) {
 269                 *cp = opt->val;
 270                 if (opt->has_arg)
 271                         *++cp = ':';
 272         }
 273
 274         while ((rc = getopt_long(argc,argv, shortOpts, longOpts,&index)) != -1) {
 275                 switch (rc) {
 276                 case OPEN:
 277                         openflags &= ~(O_CREAT|O_EXCL);
 278                 case CREATE:
 279 #ifdef HAVE_MDC_LOOKUP
 280                 case LOOKUP:
 281 #endif
 282                 case MKNOD:
 283                 case STAT:
 284                 case UNLINK:
 285                         if (cmd != NULL) {
 286                                 fatal(0, "Invalid - more than one operation "
 287                                            "specified: --%s\n",
 288                                         longOpts[index].name);
 289                         }
 290                         mode = rc;
 291                         cmd = (char *)longOpts[index].name;
 292                         break;
 293                 case NOEXCL:
 294                         if (mode != CREATE && mode != MKNOD) {
 295                                 usage(stderr, "--noexcl only applies to "
 296                                               "--create or --mknod.\n");
 297                         }
 298                         openflags &= ~O_EXCL;
 299                         break;
 300                 case RECREATE:
 301                         if (mode != UNLINK) {
 302                                 usage(stderr, "--recreate only makes sense"
 303                                               "with --unlink.\n");
 304                         }
 305                         recreate++;
 306                         break;
 307                 case BEGIN:
 308                         begin = strtol(optarg, &endptr, 0);
 309                         if ((*endptr != 0) || (begin < 0)) {
 310                                 fatal(0, "Invalid --start value.\n");
 311                         }
 312                         break;
 313                 case ITERS:
 314                         iters = strtol(optarg, &endptr, 0);
 315                         if ((*endptr != 0) || (iters <= 0)) {
 316                                 fatal(0, "Invalid --iters value.\n");
 317                         }
 318                         if (mode != LOOKUP && mode != OPEN) {
 319                                 usage(stderr, "--iters only makes sense with "
 320                                               "--lookup or --open.\n");
 321                         }
 322                         break;
 323                 case TIME:
 324                         seconds = strtol(optarg, &endptr, 0);
 325                         if ((*endptr != 0) || (seconds <= 0)) {
 326                                 fatal(0, "Invalid --time value.\n");
 327                         }
 328                         break;
 329                 case DIRFMT:
 330                         if (strlen(optarg) > (PATH_MAX - 16)) {
 331                                 fatal(0, "--dirfmt too long\n");
 332                         }
 333                         dirfmt = optarg;
 334                         break;
 335                 case NDIRS:
 336                         ndirs = strtol(optarg, &endptr, 0);
 337                         if ((*endptr != 0) || (ndirs <= 0)) {
 338                                 fatal(0, "Invalid --ndirs value.\n");
 339                         }
 340                         if ((ndirs > nthreads) &&
 341                             ((mode == CREATE) || (mode == MKNOD))) {
 342                                 fatal(0, "--ndirs=%d must be less than or "
 343                                       "equal to the number of threads (%d).\n",
 344                                       ndirs, nthreads);
 345                         }
 346                         break;
 347                 case FILEFMT:
 348                         if (strlen(optarg) > 4080) {
 349                                 fatal(0, "--filefmt too long\n");
 350                         }
 351
 352                         /* Use %%d where you want the file # in the name. */
 353                         sprintf(filefmt, optarg, myrank);
 354                         break;
 355                 case NFILES:
 356                         nfiles = strtol(optarg, &endptr, 0);
 357                         if ((*endptr != 0) || (nfiles <= 0)) {
 358                                 fatal(0, "Invalid --nfiles value.\n");
 359                         }
 360                         break;
 361                 case STRIPES:
 362                         stripes = strtol(optarg, &endptr, 0);
 363                         if ((*endptr != 0) || (stripes < 0)) {
 364                                 fatal(0, "Invalid --stripes value.\n");
 365                         }
 366
 367                         if (stripes == 0) {
 368                                 openflags |= O_LOV_DELAY_CREATE;
 369                         } else {
 370                                 fatal(0, "non-zero --stripes value "
 371                                          "not yet supported.\n");
 372                         }
 373
 374                         break;
 375                 case SEED:
 376                         seed = strtoul(optarg, &endptr, 0);
 377                         if (*endptr) {
 378                                 fatal(0, "bad --seed option %s\n", optarg);
 379                         }
 380                         break;
 381                 case SEEDFILE:
 382                         seed_file = fopen(optarg, "r");
 383                         if (!seed_file) {
 384                               fatal(myrank, "fopen(%s) error: %s\n",
 385                                       optarg, strerror(errno));
 386                         }
 387
 388                         for (i = -1; fgets(tmp, 16, seed_file) != NULL;) {
 389                                 if (++i == myrank)
 390                                         break;
 391                         }
 392
 393                         if (i == myrank) {
 394                                 rc = sscanf(tmp, "%d", &seed);
 395                                 if ((rc != 1) || (seed < 0)) {
 396                                         fatal(myrank, "Invalid seed value '%s' "
 397                                               "at line %d in %s.\n",
 398                                               tmp, i, optarg);
 399                                 }
 400                         } else {
 401                                 fatal(myrank, "File '%s' too short. Does not "
 402                                       "contain a seed for thread %d.\n",
 403                                       optarg, myrank);
 404                         }
 405
 406                         fclose(seed_file);
 407                         break;
 408                 case RANDOM:
 409                 case READDIR:
 410                         if (mode != LOOKUP && mode != OPEN)  {
 411                                 fatal(0, "--%s can only be specified with "
 412                                          "--lookup, or --open.\n",
 413                                       (char *)longOpts[index].name);
 414                         }
 415                         order = rc;
 416                         break;
 417                 case IGNORE:
 418                         ++ignore;
 419                         break;
 420                 case DEBUG:
 421                         ++debug;
 422                 case VERBOSE:
 423                         ++verbose;
 424                         break;
 425                 case HELP:
 426                         usage(stdout, NULL);
 427                         break;
 428                 case MNT:
 429                         if (strlen(optarg) > (PATH_MAX - 16))
 430                                 fatal(0, "--mnt too long\n");
 431                         mntfmt = optarg;
 432                         break;
 433                 case MNTCOUNT:
 434                         mnt_count = strtol(optarg, &endptr, 0);
 435                         if ((*endptr != 0) || (mnt_count <= 0)) {
 436                                 fatal(0, "Invalid --mnt_count value %s.\n",
 437                                       optarg);
 438                         }
 439                         break;
 440                 case MDTCOUNT:
 441                         mdt_count = strtol(optarg, &endptr, 0);
 442                         if ((*endptr != 0) || (mdt_count <= 0)) {
 443                                 fatal(0, "Invalid --mdt_count value %s.\n",
 444                                       optarg);
 445                         }
 446                         break;
 447                 default:
 448                         usage(stderr, "unrecognized option: '%c'.\n", optopt);
 449                 }
 450         }
 451
 452         if (optind < argc) {
 453                 usage(stderr, "too many arguments %d >= %d.\n", optind, argc);
 454         }
 455
 456         if ((mnt_count != -1 && mntfmt == NULL) ||
 457             (mnt_count == -1 && mntfmt != NULL)) {
 458                 usage(stderr, "mnt_count and mntfmt must be specified at the "
 459                              "same time\n");
 460         }
 461
 462         if (mode == CREATE || mode == MKNOD || mode == UNLINK || mode == STAT) {
 463                 if (seconds != 0) {
 464                         if (nfiles == 0)
 465                                 nfiles = INT_MAX;
 466                 } else if (nfiles == 0) {
 467                         usage(stderr, "--nfiles or --time must be specified "
 468                                       "with %s.\n", cmd);
 469                 }
 470         } else if (mode == LOOKUP || mode == OPEN) {
 471                 if (seconds != 0) {
 472                         if (iters == 0)
 473                                 iters = INT_MAX;
 474                 } else if (iters == 0) {
 475                         usage(stderr, "--iters or --time must be specifed "
 476                                       "with %s.\n", cmd);
 477                 }
 478
 479                 if (nfiles == 0) {
 480                         usage(stderr, "--nfiles must be specifed with --%s.\n",
 481                               cmd);
 482                 }
 483
 484                 if (seed == 0) {
 485                         int fd = open("/dev/urandom", O_RDONLY);
 486
 487                         if (fd >= 0) {
 488                                 if (read(fd, &seed, sizeof(seed)) <
 489                                     sizeof(seed))
 490                                         seed = time(0);
 491                                 close(fd);
 492                         } else {
 493                                 seed = time(0);
 494                         }
 495                 }
 496
 497                 srand(seed);
 498
 499                 dmesg("%s: rank %d seed %d (%s).\n", prog, myrank, seed,
 500                       (order == RANDOM) ? "random_order" : "readdir_order");
 501         } else {
 502                 usage(stderr, "one --create, --mknod, --open, --stat,"
 503 #ifdef HAVE_MDC_LOOKUP
 504                       " --lookup,"
 505 #endif
 506                       " or --unlink must be specifed.");
 507         }
 508
 509         /* support for multiple threads in a dir, set begin/end appropriately.*/
 510         dirnum = myrank % ndirs;
 511         dirthreads = nthreads / ndirs;
 512         if (nthreads > (ndirs * dirthreads + dirnum))
 513                 ++dirthreads;
 514
 515         offset = myrank / ndirs;
 516
 517         tmpend = begin + nfiles - 1;
 518         if (tmpend <= 0)
 519                 tmpend = INT_MAX;
 520
 521         end = begin + (nfiles / dirthreads) * dirthreads + offset;
 522         if ((end > tmpend) || (end <= 0))
 523                 end -= dirthreads;
 524
 525         /* make sure mnt_count <= nthreads, otherwise it might div 0 in
 526          * the following test */
 527         if (mnt_count > nthreads)
 528                 mnt_count = nthreads;
 529
 530         begin += offset;
 531         if (begin < 0)
 532                 begin = INT_MAX;
 533
 534         beginsave = begin;
 535
 536         dmesg("%d: iters %d nfiles %d time %d begin %d end %d dirthreads %d."
 537               "\n", myrank, iters, nfiles, seconds, begin, end, dirthreads);
 538
 539         if (dirfmt == NULL) {
 540                 strcpy(dir, ".");
 541         } else {
 542                 int dir_len = 0;
 543
 544                 if (mntfmt != NULL) {
 545                         sprintf(dir, mntfmt, (myrank / (nthreads/mnt_count)));
 546                         strcat(dir, "/");
 547                         dir_len = strlen(dir);
 548                 }
 549                 sprintf(dir + dir_len, dirfmt, dirnum);
 550
 551                 if (mdt_count > 1) {
 552                         struct stat sb;
 553                         if (stat(dir, &sb) == 0) {
 554                                 if (!S_ISDIR(sb.st_mode))
 555                                         fatal(myrank, "'%s' is not dir\n", dir);
 556                         } else if (errno == ENOENT) {
 557                                 sprintf(mkdir_cmd, "lfs mkdir -i %d %s",
 558                                         myrank % mdt_count, dir);
 559                         } else {
 560                                 fatal(myrank, "'%s' stat failed\n", dir);
 561                         }
 562                 } else {
 563                         sprintf(mkdir_cmd, "mkdir -p %s", dir);
 564                 }
 565
 566                 dmesg("%d: %s\n", myrank, mkdir_cmd);
 567 #ifdef _LIGHTWEIGHT_KERNEL
 568                 printf("NOTICE: not running system(%s)\n", mkdir_cmd);
 569 #else
 570                 rc = system(mkdir_cmd);
 571                 if (rc)
 572                         fatal(myrank, "'%s' failed.\n", mkdir_cmd);
 573 #endif
 574
 575                 rc = chdir(dir);
 576                 if (rc) {
 577                         fatal(myrank, "unable to chdir to '%s'.\n", dir);
 578                 }
 579         }
 580 }
 581
 582 static inline char *next_file()
 583 {
 584         if (order == RANDOM) {
 585                 sprintf(filename, filefmt, random() % nfiles);
 586                 return(filename);
 587         }
 588
 589         /* readdir order */
 590
 591         dir_entry = readdir(directory);
 592         if (dir_entry == NULL) {
 593                 rewinddir(directory);
 594                 while ((dir_entry = readdir(directory)) != NULL) {
 595                         if (dir_entry->d_name[0] != '.')
 596                                 return(dir_entry->d_name);
 597                 }
 598
 599                 fatal(myrank, "unable to read directory %s (%s).\n",
 600                       dir, strerror(errno));
 601         }
 602
 603         return(dir_entry->d_name);
 604 }
 605
 606 int
 607 main(int argc, char *argv[])
 608 {
 609         int    i, j, fd, rc, nops, lastOps;
 610         int ag_ops = 0;
 611         double ag_interval = 0;
 612         double ag_rate = 0;
 613         double rate, avg_rate, effective_rate;
 614         double startTime, curTime, lastTime, interval;
 615         time_t timestamp;
 616         char * file;
 617
 618         rc = MPI_Init(&argc, &argv);
 619         if (rc != MPI_SUCCESS)
 620                 fatal(myrank, "MPI_Init failed: %d\n", rc);
 621
 622         rc = MPI_Comm_size(MPI_COMM_WORLD, &nthreads);
 623         if (rc != MPI_SUCCESS)
 624                 fatal(myrank, "MPI_Comm_size failed: %d\n", rc);
 625
 626         rc = MPI_Comm_rank(MPI_COMM_WORLD, &myrank);
 627         if (rc != MPI_SUCCESS)
 628                 fatal(myrank, "MPI_Comm_rank failed: %d\n", rc);
 629
 630         process_args(argc, argv);
 631
 632         timestamp = time(0);
 633         if ((myrank == 0) || debug) {
 634                 printf("%d: %s starting at %s",
 635                        myrank, hostname, ctime(&timestamp));
 636         }
 637
 638         /* if we're not measuring creation rates then precreate
 639          * the files we're operating on. */
 640         if ((mode != CREATE) && (mode != MKNOD) && !ignore &&
 641             (mode != UNLINK || recreate)) {
 642                 /* create the files in reverse order. When we encounter
 643                  * a file that already exists, assume the remainder of
 644                  * the files exist to save time. The timed performance
 645                  * test scripts make use of this behavior. */
 646                 for (i = end, j = 0; i >= begin; i -= dirthreads) {
 647                         sprintf(filename, filefmt, i);
 648                         fd = open(filename, openflags, 0644);
 649                         if (fd < 0) {
 650                                 if (errno == EEXIST)
 651                                         break;
 652                                 rc = errno;
 653                                 fatal(myrank, "precreate open(%s) error: %s\n",
 654                                       filename, strerror(rc));
 655                         }
 656                         j++;
 657                         close(fd);
 658                 }
 659                 dmesg("%d: %s pre-created %d files.\n",myrank,hostname,j);
 660
 661                 rc = MPI_Barrier(MPI_COMM_WORLD);
 662                 if (rc != MPI_SUCCESS)
 663                         fatal(myrank, "prep MPI_Barrier failed: %d\n", rc);
 664         }
 665
 666         if (order == READDIR) {
 667                 directory = opendir(dir);
 668                 if (directory == NULL) {
 669                         rc = errno;
 670                         fatal(myrank, "opendir(%s) error: %s\n",
 671                               dir, strerror(rc));
 672                 }
 673
 674                 timestamp = time(0);
 675                 j = random() % nfiles;
 676                 dmesg("%d: %s initializing dir offset %u: %s",
 677                       myrank, hostname, j, ctime(&timestamp));
 678
 679                 for (i = 0; i <= j; i++) {
 680                         if ((dir_entry = readdir(directory)) == NULL) {
 681                                 fatal(myrank, "could not read entry number %d "
 682                                       "in directory %s.\n", i, dir);
 683                         }
 684                 }
 685
 686                 timestamp = time(0);
 687                 dmesg("%d: index %d, filename %s, offset %ld: "
 688                       "%s initialization complete: %s",
 689                       myrank, i, dir_entry->d_name, telldir(directory),
 690                       hostname, ctime(&timestamp));
 691         }
 692
 693         if (seconds) {
 694                 act.sa_handler = sigalrm_handler;
 695                 (void)sigemptyset(&act.sa_mask);
 696                 act.sa_flags = 0;
 697                 sigaction(SIGALRM, &act, NULL);
 698                 alarm(seconds);
 699         }
 700
 701         rc = MPI_Barrier(MPI_COMM_WORLD);
 702         if (rc != MPI_SUCCESS)
 703                 fatal(myrank, "prep MPI_Barrier failed: %d\n", rc);
 704
 705         startTime = lastTime = MPI_Wtime();
 706         nops = lastOps = 0;
 707
 708         switch (mode) {
 709         case CREATE:
 710                 for (; begin <= end && !alarm_caught; begin += dirthreads) {
 711                         sprintf(filename, filefmt, begin);
 712                         if ((fd = open(filename, openflags, 0644)) < 0) {
 713                                 if (((rc = errno) == EINTR) && alarm_caught)
 714                                         break;
 715                                 fatal(myrank, "open(%s) error: %s\n",
 716                                       filename, strerror(rc));
 717                         }
 718
 719                         close(fd);
 720                         nops++;
 721                         DISPLAY_PROGRESS();
 722                 }
 723
 724                 dmesg("%d: created %d files, last file '%s'.\n",
 725                       myrank, nops, filename);
 726                 break;
 727 #ifdef HAVE_MDC_LOOKUP
 728         case LOOKUP:
 729                 fd = open(dir, O_RDONLY);
 730                 if (fd < 0) {
 731                         fatal(myrank, "open(dir == '%s') error: %s\n",
 732                               dir, strerror(errno));
 733                 }
 734
 735                 for (; nops < iters && !alarm_caught;) {
 736                         char *filename = next_file();
 737                         rc = llapi_file_lookup(fd, filename);
 738                         if (rc < 0) {
 739                                 if (((rc = errno) == EINTR) && alarm_caught)
 740                                         break;
 741                                 fatal(myrank, "llapi_file_lookup(%s) "
 742                                       "error: %s\n", filename, strerror(rc));
 743                         }
 744
 745                         nops++;
 746                         DISPLAY_PROGRESS();
 747                 }
 748                 break;
 749 #endif
 750         case MKNOD:
 751                 for (; begin <= end && !alarm_caught; begin += dirthreads) {
 752                         sprintf(filename, filefmt, begin);
 753                         rc = mknod(filename, S_IFREG| 0644, 0);
 754                         if (rc) {
 755                                 if (((rc = errno) == EINTR) && alarm_caught)
 756                                         break;
 757                                 fatal(myrank, "mknod(%s) error: %s\n",
 758                                       filename, strerror(rc));
 759                         }
 760
 761                         nops++;
 762                         DISPLAY_PROGRESS();
 763                 }
 764                 break;
 765         case OPEN:
 766                 for (; nops < iters && !alarm_caught;) {
 767                         file = next_file();
 768                         if ((fd = open(file, openflags, 0644)) < 0) {
 769                                 if (((rc = errno) == EINTR) && alarm_caught)
 770                                         break;
 771                                 fatal(myrank, "open(%s) error: %s\n",
 772                                       file, strerror(rc));
 773                         }
 774
 775                         close(fd);
 776
 777                         nops++;
 778                         DISPLAY_PROGRESS();
 779                 }
 780                 break;
 781         case STAT:
 782                 for (; begin <= end && !alarm_caught; begin += dirthreads) {
 783                         sprintf(filename, filefmt, begin);
 784                         rc = stat(filename, &statbuf);
 785                         if (rc) {
 786                                 if (((rc = errno) == EINTR) && alarm_caught)
 787                                         break;
 788                                 if (((rc = errno) == ENOENT) && ignore)
 789                                         continue;
 790                                 fatal(myrank, "stat(%s) error: %s\n",
 791                                       filename, strerror(rc));
 792                         }
 793
 794                         nops++;
 795                         DISPLAY_PROGRESS();
 796                 }
 797                 break;
 798         case UNLINK:
 799                 for (; begin <= end && !alarm_caught; begin += dirthreads) {
 800                         sprintf(filename, filefmt, begin);
 801                         rc = unlink(filename);
 802                         if (rc) {
 803                                 if (((rc = errno) == EINTR) && alarm_caught)
 804                                         break;
 805                                 if ((rc = errno) == ENOENT) {
 806                                         if (ignore)
 807                                                 continue;
 808                                         /* no more files to unlink */
 809                                         break;
 810                                 }
 811                                 fatal(myrank, "unlink(%s) error: %s\n",
 812                                       filename, strerror(rc));
 813                         }
 814
 815                         nops++;
 816                         DISPLAY_PROGRESS();
 817                 }
 818                 break;
 819         }
 820
 821         rc = MPI_Barrier(MPI_COMM_WORLD);
 822         if (rc != MPI_SUCCESS)
 823                fatal(myrank, "prep MPI_Barrier failed: %d\n", rc);
 824         curTime = MPI_Wtime();
 825         interval = curTime - startTime;
 826         rate = (double) (nops) / interval;
 827
 828         rc = MPI_Reduce(&nops, &ag_ops, 1, MPI_INT, MPI_SUM, 0,
 829                         MPI_COMM_WORLD);
 830         if (rc != MPI_SUCCESS) {
 831                 fatal(myrank, "Failure in MPI_Reduce of total ops.\n");
 832         }
 833
 834         rc = MPI_Reduce(&interval, &ag_interval, 1, MPI_DOUBLE, MPI_SUM, 0,
 835                         MPI_COMM_WORLD);
 836         if (rc != MPI_SUCCESS) {
 837                 fatal(myrank, "Failure in MPI_Reduce of total interval.\n");
 838         }
 839
 840         rc = MPI_Reduce(&rate, &ag_rate, 1, MPI_DOUBLE, MPI_SUM, 0,
 841                         MPI_COMM_WORLD);
 842         if (rc != MPI_SUCCESS) {
 843                 fatal(myrank, "Failure in MPI_Reduce of aggregated rate.\n");
 844         }
 845
 846         if (myrank == 0) {
 847
 848                 curTime = MPI_Wtime();
 849                 interval = curTime - startTime;
 850                 effective_rate = (double) ag_ops / interval;
 851                 avg_rate = (double) ag_ops / ag_interval;
 852
 853                 printf("Rate: %.2f eff %.2f aggr %.2f avg client %ss/sec "
 854                        "(total: %d threads %d %ss %d dirs %d threads/dir %.2f secs)\n",
 855                        effective_rate, ag_rate, avg_rate, cmd, nthreads, ag_ops,
 856                        cmd, ndirs, dirthreads, interval);
 857                 if (mode == UNLINK && !recreate && !ignore && ag_ops != nfiles)
 858                         printf("Warning: only unlinked %d files instead of %d"
 859                                "\n", ag_ops, nfiles);
 860         }
 861
 862         if (recreate) {
 863                 for (begin = beginsave; begin <= end; begin += dirthreads) {
 864                         sprintf(filename, filefmt, begin);
 865                         if ((fd = open(filename, openflags, 0644)) < 0) {
 866                                 rc = errno;
 867                                 if (rc == EEXIST)
 868                                         break;
 869                                 fatal(myrank, "recreate open(%s) error: %s\n",
 870                                       filename, strerror(rc));
 871                         }
 872
 873                         close(fd);
 874                 }
 875         }
 876
 877         timestamp = time(0);
 878         if ((myrank == 0) || debug) {
 879                 printf("%d: %s finished at %s",
 880                        myrank, hostname, ctime(&timestamp));
 881         }
 882
 883         MPI_Finalize();
 884         return(0);
 885 }