lustre/tests/mpi/mdsrate.c

   1 /* -*- mode: c; c-basic-offset: 8; indent-tabs-mode: nil; -*-
   2  * vim:expandtab:shiftwidth=8:tabstop=8:
   3  *
   4  * 2003, Copyright, Hewlett-Packard Development Compnay, LP.
   5  *
   6  * Developed under the sponsorship of the U.S. Government
   7  *     under Subcontract No. B514193
   8  */
   9
  10 /*
  11  * Copyright 2008 Sun Microsystems, Inc. All rights reserved.
  12  * Use is subject to license terms.
  13  */
  14
  15 #include <stdio.h>
  16 #include <getopt.h>
  17 #include <libgen.h>
  18 #include <sys/types.h>
  19 #include <sys/stat.h>
  20 #include <time.h>
  21 #include <limits.h>
  22 #include <errno.h>
  23 #include <string.h>
  24 #include <fcntl.h>
  25 #include <unistd.h>
  26 #include <stdlib.h>
  27 #include <stdarg.h>
  28 #include <signal.h>
  29 #include <sys/ioctl.h>
  30 #include <dirent.h>
  31
  32 #include "mpi.h"
  33
  34 /* lustre */
  35 #include <lustre/liblustreapi.h>        /* for O_LOV_DELAY_CREATE */
  36
  37 #define CHECK_COUNT 10000
  38 #define DISPLAY_COUNT (CHECK_COUNT * 10)
  39 #define DISPLAY_TIME 100
  40
  41 enum {
  42         CREATE   = 'c',
  43         LOOKUP   = 'l',
  44         MKNOD    = 'm',
  45         OPEN     = 'o',
  46         STAT     = 's',
  47         UNLINK   = 'u',
  48         BEGIN    = 'b',
  49         ITERS    = 'i',
  50         TIME     = 't',
  51         DIRFMT   = 'd',
  52         NDIRS    = 'D',
  53         FILEFMT  = 'f',
  54         NFILES   = 'F',
  55         NOEXCL   = 'X',
  56         STRIPES  = 'S',
  57         SEED     = 'r',
  58         SEEDFILE = 'R',
  59         RANDOM   = 'A',
  60         READDIR  = 'B',
  61         RECREATE = 'C',
  62         IGNORE   = 'E',
  63         VERBOSE  = 'V',
  64         DEBUG    = 'v',
  65         HELP     = 'h',
  66 };
  67
  68 struct option longOpts[] = {
  69         {"create",        0, NULL, CREATE     },
  70         {"lookup",        0, NULL, LOOKUP     },
  71         {"mknod",         0, NULL, MKNOD      },
  72         {"open",          0, NULL, OPEN       },
  73         {"stat",          0, NULL, STAT       },
  74         {"unlink",        0, NULL, UNLINK     },
  75         {"begin",         1, NULL, BEGIN      },
  76         {"iters",         1, NULL, ITERS      },
  77         {"time",          1, NULL, TIME       },   /* seconds */
  78         {"dirfmt",        1, NULL, DIRFMT     },
  79         {"ndirs",         1, NULL, NDIRS      },
  80         {"filefmt",       1, NULL, FILEFMT    },
  81         {"nfiles",        1, NULL, NFILES     },
  82         {"noexcl",        0, NULL, NOEXCL     },
  83         {"stripes",       1, NULL, STRIPES    },
  84         {"seed",          1, NULL, SEED       },
  85         {"seedfile",      1, NULL, SEEDFILE   },
  86         {"random_order",  0, NULL, RANDOM     },
  87         {"readdir_order", 0, NULL, READDIR    },
  88         {"recreate",      0, NULL, RECREATE   },
  89         {"ignore",        0, NULL, IGNORE     },
  90         {"verbose",       0, NULL, VERBOSE    },
  91         {"debug",         0, NULL, DEBUG      },
  92         {"help",          0, NULL, HELP       },
  93         { 0,              0, NULL, 0          }
  94 };
  95
  96 int foo1, foo2;
  97
  98 char   shortOpts[128];
  99 int    myrank = -1;
 100 int    nthreads = -1;
 101 char * prog;
 102 char   hostname[512] = "unknown";
 103 char   mode;
 104 char * cmd;
 105 int    openflags = O_RDWR|O_CREAT|O_EXCL;
 106 int    ndirs = 1;
 107 char * dirfmt;
 108 char   dir[PATH_MAX];
 109 char   mkdir_cmd[PATH_MAX+14];
 110 int    dirthreads;
 111 int    dirnum;
 112 DIR *  directory;
 113 struct dirent *dir_entry;
 114 int    nfiles;
 115 char   filefmt[PATH_MAX];
 116 char   filename[PATH_MAX];
 117 int    stripes = -1;
 118 int    begin;
 119 int    beginsave;
 120 int    end;
 121 int    iters;
 122 int    seconds;
 123 int    alarm_caught;
 124 struct sigaction act;
 125 int    order = RANDOM;
 126 int    seed;
 127 int    recreate;
 128 int    ignore;
 129 int    verbose;
 130 int    debug;
 131 struct stat statbuf;
 132
 133 #define dmesg if (debug) printf
 134
 135 #define DISPLAY_PROGRESS() {                                                \
 136         if ((++nops % CHECK_COUNT) == 0 && verbose) {                       \
 137                 curTime = time(0);                                          \
 138                 interval = curTime - lastTime;                              \
 139                 if (interval > DISPLAY_TIME || nops % DISPLAY_COUNT == 0) { \
 140                         rate = (float)(nops - lastOps);                     \
 141                         if (interval > 1)                                   \
 142                                 rate /= (float)interval;                    \
 143                         printf("Rank %d: %.2f %ss/sec %lu secs "            \
 144                                "(total: %d %ss %lu secs)\n",                \
 145                                myrank, rate, cmd, interval,                 \
 146                                nops, cmd, curTime - startTime);             \
 147                         lastOps = nops;                                     \
 148                         lastTime = curTime;                                 \
 149                 }                                                           \
 150         }                                                                   \
 151 }
 152
 153 char *usage_msg = "usage: %s\n"
 154                   "    { --create [ --noexcl ] | --lookup | --mknod |\n"
 155                   "      --open | --stat | --unlink  [ --recreate ] [ --ignore ] }\n"
 156                   "    [ --help ] [ --verbose ] [ --debug ]\n"
 157                   "    { [ --begin <num> ] --nfiles <num> }\n"
 158                   "    [ --iters <num> ] [ --time <secs> ]\n"
 159                   "    [ --dirfmt <str> ] [ --ndirs  <num> ]\n"
 160                   "    [ --filefmt <str> ] [ --stripes <num> ]\n"
 161                   "    [ --random_order [--seed <num> | --seedfile <file>] ]\n"
 162                   "    [ --readdir_order ]\n";
 163
 164 static void
 165 usage(FILE *stream, char *fmt, ...)
 166 {
 167         if (myrank == 0) {
 168                 if (fmt != NULL) {
 169                         va_list       ap;
 170
 171                         fprintf(stream, "%s: ", prog);
 172                         va_start(ap, fmt);
 173                         vfprintf(stderr, fmt, ap);
 174                         va_end(ap);
 175                 }
 176                 fprintf(stream, usage_msg, prog);
 177         }
 178
 179         MPI_Finalize();
 180         exit(stream == stderr);
 181 }
 182
 183 /* Print process myrank and message, and exit (i.e. a fatal error) */
 184 static int
 185 fatal(int rank, const char *fmt, ...)
 186 {
 187         if (rank == myrank) {
 188                 va_list       ap;
 189
 190                 fprintf(stderr, "rank %d: ", rank);
 191                 va_start(ap, fmt);
 192                 vfprintf(stderr, fmt, ap);
 193                 va_end(ap);
 194         }
 195
 196         MPI_Abort(MPI_COMM_WORLD, 1);
 197         exit(1);
 198 }
 199
 200 static void
 201 sigalrm_handler(int signum)
 202 {
 203         alarm_caught++;
 204 }
 205
 206 /* HAVE_LLAPI_FILE_LOOKUP is defined by liblustreapi.h if this function is
 207  * defined therein.  Otherwise we can do the equivalent operation via ioctl
 208  * if we have access to a complete lustre build tree to get the various
 209  * definitions - then compile with USE_MDC_LOOKUP defined. */
 210 #if defined(HAVE_LLAPI_FILE_LOOKUP)
 211 #define HAVE_MDC_LOOKUP
 212 #elif defined(USE_MDC_LOOKUP)
 213 #include <config.h>
 214 #include <liblustre.h>
 215 #include <linux/lustre_lib.h>
 216
 217 int llapi_file_lookup(int dirfd, const char *name)
 218 {
 219         struct obd_ioctl_data data = { 0 };
 220         char rawbuf[8192];
 221         char *buf = rawbuf;
 222         int rc;
 223
 224         if (dirfd < 0 || name == NULL)
 225                 return -EINVAL;
 226
 227         data.ioc_version = OBD_IOCTL_VERSION;
 228         data.ioc_len = sizeof(data);
 229         data.ioc_inlbuf1 = name;
 230         data.ioc_inllen1 = strlen(name) + 1;
 231
 232         rc = obd_ioctl_pack(&data, &buf, sizeof(rawbuf));
 233         if (rc) {
 234                 fatal(myrank, "ioctl_pack failed: rc = %d\n", rc);
 235                 return rc;
 236         }
 237
 238         return ioctl(fd, IOC_MDC_LOOKUP, buf);
 239 }
 240 #define HAVE_MDC_LOOKUP
 241 #endif
 242
 243 static void
 244 process_args(int argc, char *argv[])
 245 {
 246         char   *cp, *endptr;
 247         int    i, index, offset, tmpend, rc, c;
 248         char   tmp[16];
 249         FILE * seed_file;
 250         struct option *opt;
 251
 252         setbuf(stdout, 0);
 253         setbuf(stderr, 0);
 254         prog = basename(argv[0]);
 255         strcpy(filefmt, "f%d");
 256         gethostname(hostname, sizeof(hostname));
 257
 258         /* auto create shortOpts rather than maintaining a static string. */
 259         for (opt = longOpts, cp = shortOpts; opt->name != NULL; opt++, cp++) {
 260                 *cp = opt->val;
 261                 if (opt->has_arg)
 262                         *++cp = ':';
 263         }
 264
 265         while ((c = getopt_long(argc,argv, shortOpts, longOpts,&index)) != -1) {
 266                 switch (c) {
 267                 case OPEN:
 268                         openflags &= ~(O_CREAT|O_EXCL);
 269                 case CREATE:
 270 #ifdef HAVE_MDC_LOOKUP
 271                 case LOOKUP:
 272 #endif
 273                 case MKNOD:
 274                 case STAT:
 275                 case UNLINK:
 276                         if (cmd != NULL) {
 277                                 fatal(0, "Invalid - more than one operation "
 278                                            "specified: --%s\n",
 279                                         longOpts[index].name);
 280                         }
 281                         mode = c;
 282                         cmd = (char *)longOpts[index].name;
 283                         break;
 284                 case NOEXCL:
 285                         if (mode != CREATE && mode != MKNOD) {
 286                                 usage(stderr, "--noexcl only applies to "
 287                                               "--create or --mknod.\n");
 288                         }
 289                         openflags &= ~O_EXCL;
 290                         break;
 291                 case RECREATE:
 292                         if (mode != UNLINK) {
 293                                 usage(stderr, "--recreate only makes sense"
 294                                               "with --unlink.\n");
 295                         }
 296                         recreate++;
 297                         break;
 298                 case BEGIN:
 299                         begin = strtol(optarg, &endptr, 0);
 300                         if ((*endptr != 0) || (begin < 0)) {
 301                                 fatal(0, "Invalid --start value.\n");
 302                         }
 303                         break;
 304                 case ITERS:
 305                         iters = strtol(optarg, &endptr, 0);
 306                         if ((*endptr != 0) || (iters <= 0)) {
 307                                 fatal(0, "Invalid --iters value.\n");
 308                         }
 309                         if (mode != LOOKUP && mode != OPEN && mode != STAT) {
 310                                 usage(stderr, "--iters only makes sense with "
 311                                               "--lookup, --open, or --stat.\n");
 312                         }
 313                         break;
 314                 case TIME:
 315                         seconds = strtol(optarg, &endptr, 0);
 316                         if ((*endptr != 0) || (seconds <= 0)) {
 317                                 fatal(0, "Invalid --time value.\n");
 318                         }
 319                         break;
 320                 case DIRFMT:
 321                         if (strlen(optarg) > (PATH_MAX - 16)) {
 322                                 fatal(0, "--dirfmt too long\n");
 323                         }
 324                         dirfmt = optarg;
 325                         break;
 326                 case NDIRS:
 327                         ndirs = strtol(optarg, &endptr, 0);
 328                         if ((*endptr != 0) || (ndirs <= 0)) {
 329                                 fatal(0, "Invalid --ndirs value.\n");
 330                         }
 331                         if ((ndirs > nthreads) &&
 332                             ((mode == CREATE) || (mode == MKNOD))) {
 333                                 fatal(0, "--ndirs=%d must be less than or "
 334                                       "equal to the number of threads (%d).\n",
 335                                       ndirs, nthreads);
 336                         }
 337                         break;
 338                 case FILEFMT:
 339                         if (strlen(optarg) > 4080) {
 340                                 fatal(0, "--filefmt too long\n");
 341                         }
 342
 343                         /* Use %%d where you want the file # in the name. */
 344                         sprintf(filefmt, optarg, myrank);
 345                         break;
 346                 case NFILES:
 347                         nfiles = strtol(optarg, &endptr, 0);
 348                         if ((*endptr != 0) || (nfiles <= 0)) {
 349                                 fatal(0, "Invalid --nfiles value.\n");
 350                         }
 351                         break;
 352                 case STRIPES:
 353                         stripes = strtol(optarg, &endptr, 0);
 354                         if ((*endptr != 0) || (stripes < 0)) {
 355                                 fatal(0, "Invalid --stripes value.\n");
 356                         }
 357
 358                         if (stripes == 0) {
 359                                 openflags |= O_LOV_DELAY_CREATE;
 360                         } else {
 361                                 fatal(0, "non-zero --stripes value "
 362                                          "not yet supported.\n");
 363                         }
 364
 365                         break;
 366                 case SEED:
 367                         seed = strtoul(optarg, &endptr, 0);
 368                         if (*endptr) {
 369                                 fatal(0, "bad --seed option %s\n", optarg);
 370                         }
 371                         break;
 372                 case SEEDFILE:
 373                         seed_file = fopen(optarg, "r");
 374                         if (!seed_file) {
 375                               fatal(myrank, "fopen(%s) error: %s\n",
 376                                       optarg, strerror(errno));
 377                         }
 378
 379                         for (i = -1; fgets(tmp, 16, seed_file) != NULL;) {
 380                                 if (++i == myrank)
 381                                         break;
 382                         }
 383
 384                         if (i == myrank) {
 385                                 rc = sscanf(tmp, "%d", &seed);
 386                                 if ((rc != 1) || (seed < 0)) {
 387                                         fatal(myrank, "Invalid seed value '%s' "
 388                                               "at line %d in %s.\n",
 389                                               tmp, i, optarg);
 390                                 }
 391                         } else {
 392                                 fatal(myrank, "File '%s' too short. Does not "
 393                                       "contain a seed for thread %d.\n",
 394                                       optarg, myrank);
 395                         }
 396
 397                         fclose(seed_file);
 398                         break;
 399                 case RANDOM:
 400                 case READDIR:
 401                         if (mode != LOOKUP && mode != OPEN && mode != STAT)  {
 402                                 fatal(0, "--%s can only be specified with "
 403                                          "--lookup, --open, or --stat.\n",
 404                                       (char *)longOpts[index].name);
 405                         }
 406                         order = c;
 407                         break;
 408                 case IGNORE:
 409                         ++ignore;
 410                         break;
 411                 case DEBUG:
 412                         ++debug;
 413                 case VERBOSE:
 414                         ++verbose;
 415                         break;
 416                 case HELP:
 417                         usage(stdout, NULL);
 418                 default:
 419                         usage(stderr, "unrecognized option: '%c'.\n", optopt);
 420                 }
 421         }
 422
 423         if (optind < argc) {
 424                 usage(stderr, "too many arguments %d >= %d.\n", optind, argc);
 425         }
 426
 427         if (mode == CREATE || mode == MKNOD || mode == UNLINK) {
 428                 if (seconds != 0) {
 429                         if (nfiles == 0)
 430                                 nfiles = INT_MAX;
 431                 } else if (nfiles == 0) {
 432                         usage(stderr, "--nfiles or --time must be specified "
 433                                       "with %s.\n", cmd);
 434                 }
 435         } else if (mode == LOOKUP || mode == OPEN || mode == STAT) {
 436                 if (seconds != 0) {
 437                         if (iters == 0)
 438                                 iters = INT_MAX;
 439                 } else if (iters == 0) {
 440                         usage(stderr, "--iters or --time must be specifed "
 441                                       "with %s.\n", cmd);
 442                 }
 443
 444                 if (nfiles == 0) {
 445                         usage(stderr, "--nfiles must be specifed with --%s.\n",
 446                               cmd);
 447                 }
 448
 449                 if (seed == 0) {
 450                         int fd = open("/dev/urandom", O_RDONLY);
 451
 452                         if (fd >= 0) {
 453                                 if (read(fd, &seed, sizeof(seed)) <
 454                                     sizeof(seed))
 455                                         seed = time(0);
 456                                 close(fd);
 457                         } else {
 458                                 seed = time(0);
 459                         }
 460                 }
 461
 462                 srand(seed);
 463
 464                 dmesg("%s: rank %d seed %d (%s).\n", prog, myrank, seed,
 465                       (order == RANDOM) ? "random_order" : "readdir_order");
 466         } else {
 467                 usage(stderr, "one --create, --mknod, --open, --stat,"
 468 #ifdef HAVE_MDC_LOOKUP
 469                       " --lookup,"
 470 #endif
 471                       " or --unlink must be specifed.");
 472         }
 473
 474         /* support for multiple threads in a dir, set begin/end appropriately.*/
 475         dirnum = myrank % ndirs;
 476         dirthreads = nthreads / ndirs;
 477         if (nthreads > (ndirs * dirthreads + dirnum))
 478                 ++dirthreads;
 479
 480         offset = myrank / ndirs;
 481
 482         tmpend = begin + nfiles - 1;
 483         if (tmpend <= 0)
 484                 tmpend = INT_MAX;
 485
 486         end = begin + (nfiles / dirthreads) * dirthreads + offset;
 487         if ((end > tmpend) || (end <= 0))
 488                 end -= dirthreads;
 489
 490         begin += offset;
 491         if (begin < 0)
 492                 begin = INT_MAX;
 493
 494         beginsave = begin;
 495
 496         dmesg("%d: iters %d nfiles %d time %d begin %d end %d dirthreads %d."
 497               "\n", myrank, iters, nfiles, seconds, begin, end, dirthreads);
 498
 499         if (dirfmt == NULL) {
 500                 strcpy(dir, ".");
 501         } else {
 502                 sprintf(dir, dirfmt, dirnum);
 503
 504                 sprintf(mkdir_cmd, "/bin/mkdir -p %s", dir);
 505                 #ifdef _LIGHTWEIGHT_KERNEL
 506                         printf("NOTICE: not running system(%s)\n", mkdir_cmd);
 507                 #else
 508                         rc = system(mkdir_cmd);
 509                         if (rc) {
 510                                 fatal(myrank, "'%s' failed.\n", mkdir_cmd);
 511                         }
 512                 #endif
 513
 514                 rc = chdir(dir);
 515                 if (rc) {
 516                         fatal(myrank, "unable to chdir to '%s'.\n", dir);
 517                 }
 518         }
 519 }
 520
 521 static inline char *next_file()
 522 {
 523         if (order == RANDOM) {
 524                 sprintf(filename, filefmt, random() % nfiles);
 525                 return(filename);
 526         }
 527
 528         /* readdir order */
 529
 530         dir_entry = readdir(directory);
 531         if (dir_entry == NULL) {
 532                 rewinddir(directory);
 533                 while ((dir_entry = readdir(directory)) != NULL) {
 534                         if (dir_entry->d_name[0] != '.')
 535                                 return(dir_entry->d_name);
 536                 }
 537
 538                 fatal(myrank, "unable to read directory %s (%s).\n",
 539                       dir, strerror(errno));
 540         }
 541
 542         return(dir_entry->d_name);
 543 }
 544
 545 int
 546 main(int argc, char *argv[])
 547 {
 548         int    i, j, fd, rc, nops, lastOps, ag_ops;
 549         float  rate, ag_rate;
 550         time_t startTime, lastTime, curTime, interval;
 551         char * file;
 552
 553         rc = MPI_Init(&argc, &argv);
 554         if (rc != MPI_SUCCESS)
 555                 fatal(myrank, "MPI_Init failed: %d\n", rc);
 556
 557         rc = MPI_Comm_size(MPI_COMM_WORLD, &nthreads);
 558         if (rc != MPI_SUCCESS)
 559                 fatal(myrank, "MPI_Comm_size failed: %d\n", rc);
 560
 561         rc = MPI_Comm_rank(MPI_COMM_WORLD, &myrank);
 562         if (rc != MPI_SUCCESS)
 563                 fatal(myrank, "MPI_Comm_rank failed: %d\n", rc);
 564
 565         process_args(argc, argv);
 566
 567         startTime = time(0);
 568         if ((myrank == 0) || debug) {
 569                 printf("%d: %s starting at %s",
 570                        myrank, hostname, ctime(&startTime));
 571         }
 572
 573         /* if we're not measuring creation rates then precreate
 574          * the files we're operating on. */
 575         if ((mode != CREATE) && (mode != MKNOD) && !ignore) {
 576                 /* create the files in reverse order. When we encounter
 577                  * a file that already exists, assume the remainder of
 578                  * the files exist to save time. The timed performance
 579                  * test scripts make use of this behavior. */
 580                 for (i = end, j = 0; i >= begin; i -= dirthreads) {
 581                         sprintf(filename, filefmt, i);
 582                         fd = open(filename, openflags, 0644);
 583                         if (fd < 0) {
 584                                 if (errno == EEXIST)
 585                                         break;
 586                                 rc = errno;
 587                                 fatal(myrank, "precreate open(%s) error: %s\n",
 588                                       filename, strerror(rc));
 589                         }
 590                         j++;
 591                         close(fd);
 592                 }
 593                 dmesg("%d: %s pre-created %d files.\n",myrank,hostname,j);
 594
 595                 rc = MPI_Barrier(MPI_COMM_WORLD);
 596                 if (rc != MPI_SUCCESS)
 597                         fatal(myrank, "prep MPI_Barrier failed: %d\n", rc);
 598         }
 599
 600         if (order == READDIR) {
 601                 directory = opendir(dir);
 602                 if (directory == NULL) {
 603                         rc = errno;
 604                         fatal(myrank, "opendir(%s) error: %s\n",
 605                               dir, strerror(rc));
 606                 }
 607
 608                 startTime = time(0);
 609                 j = random() % nfiles;
 610                 dmesg("%d: %s initializing dir offset %u: %s",
 611                       myrank, hostname, j, ctime(&startTime));
 612
 613                 for (i = 0; i <= j; i++) {
 614                         if ((dir_entry = readdir(directory)) == NULL) {
 615                                 fatal(myrank, "could not read entry number %d "
 616                                       "in directory %s.\n", i, dir);
 617                         }
 618                 }
 619
 620                 lastTime = time(0);
 621                 dmesg("%d: index %d, filename %s, offset %ld: "
 622                       "%s initialization complete: %s",
 623                       myrank, i, dir_entry->d_name, telldir(directory),
 624                       hostname, ctime(&lastTime));
 625         }
 626
 627         rc = MPI_Barrier(MPI_COMM_WORLD);
 628         if (rc != MPI_SUCCESS)
 629                 fatal(myrank, "prep MPI_Barrier failed: %d\n", rc);
 630
 631         if (seconds) {
 632                 act.sa_handler = sigalrm_handler;
 633                 (void)sigemptyset(&act.sa_mask);
 634                 act.sa_flags = 0;
 635                 sigaction(SIGALRM, &act, NULL);
 636                 alarm(seconds);
 637         }
 638
 639         startTime = lastTime = time(0);
 640         nops = lastOps = 0;
 641
 642         switch (mode) {
 643         case CREATE:
 644                 for (; begin <= end && !alarm_caught; begin += dirthreads) {
 645                         sprintf(filename, filefmt, begin);
 646                         if ((fd = open(filename, openflags, 0644)) < 0) {
 647                                 if (((rc = errno) == EINTR) && alarm_caught)
 648                                         break;
 649                                 fatal(myrank, "open(%s) error: %s\n",
 650                                       filename, strerror(rc));
 651                         }
 652
 653                         close(fd);
 654                         DISPLAY_PROGRESS();
 655                 }
 656
 657                 dmesg("%d: created %d files, last file '%s'.\n",
 658                       myrank, nops, filename);
 659                 break;
 660 #ifdef HAVE_MDC_LOOKUP
 661         case LOOKUP:
 662                 fd = open(dir, O_RDONLY);
 663                 if (fd < 0) {
 664                         fatal(myrank, "open(dir == '%s') error: %s\n",
 665                               dir, strerror(errno));
 666                 }
 667
 668                 for (; nops < iters && !alarm_caught;) {
 669                         char *filename = next_file();
 670                         rc = llapi_file_lookup(fd, filename);
 671                         if (rc < 0) {
 672                                 if (((rc = errno) == EINTR) && alarm_caught)
 673                                         break;
 674                                 fatal(myrank, "llapi_file_lookup(%s) "
 675                                       "error: %s\n", filename, strerror(rc));
 676                         }
 677
 678                         DISPLAY_PROGRESS();
 679                 }
 680                 break;
 681 #endif
 682         case MKNOD:
 683                 for (; begin <= end && !alarm_caught; begin += dirthreads) {
 684                         sprintf(filename, filefmt, begin);
 685                         rc = mknod(filename, S_IFREG| 0644, 0);
 686                         if (rc) {
 687                                 if (((rc = errno) == EINTR) && alarm_caught)
 688                                         break;
 689                                 fatal(myrank, "mknod(%s) error: %s\n",
 690                                       filename, strerror(rc));
 691                         }
 692
 693                         DISPLAY_PROGRESS();
 694                 }
 695                 break;
 696         case OPEN:
 697                 for (; nops < iters && !alarm_caught;) {
 698                         file = next_file();
 699                         if ((fd = open(file, openflags, 0644)) < 0) {
 700                                 if (((rc = errno) == EINTR) && alarm_caught)
 701                                         break;
 702                                 fatal(myrank, "open(%s) error: %s\n",
 703                                       file, strerror(rc));
 704                         }
 705
 706                         close(fd);
 707
 708                         DISPLAY_PROGRESS();
 709                 }
 710                 break;
 711         case STAT:
 712                 for (; nops < iters && !alarm_caught;) {
 713                         rc = stat(file = next_file(), &statbuf);
 714                         if (rc) {
 715                                 if (((rc = errno) == EINTR) && alarm_caught)
 716                                         break;
 717                                 fatal(myrank, "stat(%s) error: %s\n",
 718                                       file, strerror(rc));
 719                         }
 720
 721                         DISPLAY_PROGRESS();
 722                 }
 723                 break;
 724         case UNLINK:
 725                 for (; begin <= end && !alarm_caught; begin += dirthreads) {
 726                         sprintf(filename, filefmt, begin);
 727                         rc = unlink(filename);
 728                         if (rc) {
 729                                 if (((rc = errno) == EINTR) && alarm_caught)
 730                                         break;
 731                                 if (((rc = errno) == ENOENT) && ignore)
 732                                         continue;
 733                                 fatal(myrank, "unlink(%s) error: %s\n",
 734                                       filename, strerror(rc));
 735                         }
 736
 737                         DISPLAY_PROGRESS();
 738                 }
 739                 break;
 740         }
 741
 742         curTime = time(0);
 743         interval = curTime - startTime;
 744         rate = (float)(nops);
 745         if (interval != 0)
 746                 rate /= (float)interval;
 747
 748         rc = MPI_Reduce(&nops, &ag_ops, 1, MPI_INT, MPI_SUM, 0,
 749                         MPI_COMM_WORLD);
 750         if (rc != MPI_SUCCESS) {
 751                 fatal(myrank, "Failure in MPI_Reduce of total ops.\n");
 752         }
 753
 754         rc = MPI_Reduce(&rate, &ag_rate, 1, MPI_FLOAT, MPI_SUM, 0,
 755                         MPI_COMM_WORLD);
 756         if (rc != MPI_SUCCESS) {
 757                 fatal(myrank, "Failure in MPI_Reduce of aggregated rate.\n");
 758         }
 759
 760         if (myrank == 0) {
 761                 printf("Rate: %.2f %ss/sec (total: %d threads %d %ss %lu secs)"
 762                        "\n", ag_rate, cmd, nthreads, ag_ops, cmd, interval);
 763         }
 764
 765         if (recreate) {
 766                 for (begin = beginsave; begin <= end; begin += dirthreads) {
 767                         sprintf(filename, filefmt, begin);
 768                         if ((fd = open(filename, openflags, 0644)) < 0) {
 769                                 rc = errno;
 770                                 if (rc == EEXIST)
 771                                         break;
 772                                 fatal(myrank, "recreate open(%s) error: %s\n",
 773                                       filename, strerror(rc));
 774                         }
 775
 776                         close(fd);
 777                 }
 778         }
 779
 780         curTime = time(0);
 781         if ((myrank == 0) || debug) {
 782                 printf("%d: %s finished at %s",
 783                        myrank, hostname, ctime(&curTime));
 784         }
 785
 786         MPI_Finalize();
 787         return(0);
 788 }