lustre/tests/mpi/mdsrate.c

   1 /* -*- mode: c; c-basic-offset: 8; indent-tabs-mode: nil; -*-
   2  * vim:expandtab:shiftwidth=8:tabstop=8:
   3  *
   4  * 2003, Copyright, Hewlett-Packard Development Compnay, LP.
   5  *
   6  * Developed under the sponsorship of the U.S. Government
   7  *     under Subcontract No. B514193
   8  */
   9
  10 /*
  11  * Copyright 2008 Sun Microsystems, Inc. All rights reserved.
  12  * Use is subject to license terms.
  13  */
  14
  15 #include <stdio.h>
  16 #include <getopt.h>
  17 #include <libgen.h>
  18 #include <sys/types.h>
  19 #include <sys/stat.h>
  20 #include <time.h>
  21 #include <limits.h>
  22 #include <errno.h>
  23 #include <string.h>
  24 #include <fcntl.h>
  25 #include <unistd.h>
  26 #include <stdlib.h>
  27 #include <stdarg.h>
  28 #include <signal.h>
  29 #include <sys/ioctl.h>
  30 #include <dirent.h>
  31
  32 #include "mpi.h"
  33
  34 /* lustre */
  35 #include <liblustre.h>
  36 #include <lustre/liblustreapi.h>        /* for O_LOV_DELAY_CREATE */
  37
  38 #define CHECK_COUNT 10000
  39 #define DISPLAY_COUNT (CHECK_COUNT * 10)
  40 #define DISPLAY_TIME 100
  41
  42 enum {
  43         CREATE   = 'c',
  44         LOOKUP   = 'l',
  45         MKNOD    = 'm',
  46         OPEN     = 'o',
  47         STAT     = 's',
  48         UNLINK   = 'u',
  49         BEGIN    = 'b',
  50         ITERS    = 'i',
  51         TIME     = 't',
  52         DIRFMT   = 'd',
  53         NDIRS    = 'D',
  54         FILEFMT  = 'f',
  55         NFILES   = 'F',
  56         NOEXCL   = 'X',
  57         STRIPES  = 'S',
  58         SEED     = 'r',
  59         SEEDFILE = 'R',
  60         RANDOM   = 'A',
  61         READDIR  = 'B',
  62         RECREATE = 'C',
  63         IGNORE   = 'E',
  64         VERBOSE  = 'V',
  65         DEBUG    = 'v',
  66         HELP     = 'h',
  67 };
  68
  69 struct option longOpts[] = {
  70         {"create",        0, NULL, CREATE     },
  71         {"lookup",        0, NULL, LOOKUP     },
  72         {"mknod",         0, NULL, MKNOD      },
  73         {"open",          0, NULL, OPEN       },
  74         {"stat",          0, NULL, STAT       },
  75         {"unlink",        0, NULL, UNLINK     },
  76         {"begin",         1, NULL, BEGIN      },
  77         {"iters",         1, NULL, ITERS      },
  78         {"time",          1, NULL, TIME       },   /* seconds */
  79         {"dirfmt",        1, NULL, DIRFMT     },
  80         {"ndirs",         1, NULL, NDIRS      },
  81         {"filefmt",       1, NULL, FILEFMT    },
  82         {"nfiles",        1, NULL, NFILES     },
  83         {"noexcl",        0, NULL, NOEXCL     },
  84         {"stripes",       1, NULL, STRIPES    },
  85         {"seed",          1, NULL, SEED       },
  86         {"seedfile",      1, NULL, SEEDFILE   },
  87         {"random_order",  0, NULL, RANDOM     },
  88         {"readdir_order", 0, NULL, READDIR    },
  89         {"recreate",      0, NULL, RECREATE   },
  90         {"ignore",        0, NULL, IGNORE     },
  91         {"verbose",       0, NULL, VERBOSE    },
  92         {"debug",         0, NULL, DEBUG      },
  93         {"help",          0, NULL, HELP       },
  94         { 0,              0, NULL, 0          }
  95 };
  96
  97 int foo1, foo2;
  98
  99 char   shortOpts[128];
 100 int    myrank = -1;
 101 int    nthreads = -1;
 102 char * prog;
 103 char   hostname[512] = "unknown";
 104 char   mode;
 105 char * cmd;
 106 int    openflags = O_RDWR|O_CREAT|O_EXCL;
 107 int    ndirs = 1;
 108 char * dirfmt;
 109 char   dir[PATH_MAX];
 110 char   mkdir_cmd[PATH_MAX+14];
 111 int    dirthreads;
 112 int    dirnum;
 113 DIR *  directory;
 114 struct dirent *dir_entry;
 115 int    nfiles;
 116 char   filefmt[PATH_MAX];
 117 char   filename[PATH_MAX];
 118 int    stripes = -1;
 119 int    begin;
 120 int    beginsave;
 121 int    end;
 122 int    iters;
 123 int    seconds;
 124 int    alarm_caught;
 125 struct sigaction act;
 126 int    order = RANDOM;
 127 int    seed;
 128 int    recreate;
 129 int    ignore;
 130 int    verbose;
 131 int    debug;
 132 struct stat statbuf;
 133
 134 #define dmesg if (debug) printf
 135
 136 #define DISPLAY_PROGRESS() {                                                \
 137         if ((++nops % CHECK_COUNT) == 0 && verbose) {                       \
 138                 curTime = time(0);                                          \
 139                 interval = curTime - lastTime;                              \
 140                 if (interval > DISPLAY_TIME || nops % DISPLAY_COUNT == 0) { \
 141                         rate = (float)(nops - lastOps);                     \
 142                         if (interval > 1)                                   \
 143                                 rate /= (float)interval;                    \
 144                         printf("Rank %d: %.2f %ss/sec %lu secs "            \
 145                                "(total: %d %ss %lu secs)\n",                \
 146                                myrank, rate, cmd, interval,                 \
 147                                nops, cmd, curTime - startTime);             \
 148                         lastOps = nops;                                     \
 149                         lastTime = curTime;                                 \
 150                 }                                                           \
 151         }                                                                   \
 152 }
 153
 154 char *usage_msg = "usage: %s\n"
 155                   "    { --create [ --noexcl ] | --lookup | --mknod |\n"
 156                   "      --open | --stat | --unlink  [ --recreate ] [ --ignore ] }\n"
 157                   "    [ --help ] [ --verbose ] [ --debug ]\n"
 158                   "    { [ --begin <num> ] --nfiles <num> }\n"
 159                   "    [ --iters <num> ] [ --time <secs> ]\n"
 160                   "    [ --dirfmt <str> ] [ --ndirs  <num> ]\n"
 161                   "    [ --filefmt <str> ] [ --stripes <num> ]\n"
 162                   "    [ --random_order [--seed <num> | --seedfile <file>] ]\n"
 163                   "    [ --readdir_order ]\n";
 164
 165 static void
 166 usage(FILE *stream, char *fmt, ...)
 167 {
 168         if (myrank == 0) {
 169                 if (fmt != NULL) {
 170                         va_list       ap;
 171
 172                         fprintf(stream, "%s: ", prog);
 173                         va_start(ap, fmt);
 174                         vfprintf(stderr, fmt, ap);
 175                         va_end(ap);
 176                 }
 177                 fprintf(stream, usage_msg, prog);
 178         }
 179
 180         MPI_Finalize();
 181         exit(stream == stderr);
 182 }
 183
 184 /* Print process myrank and message, and exit (i.e. a fatal error) */
 185 static int
 186 fatal(int rank, const char *fmt, ...)
 187 {
 188         if (rank == myrank) {
 189                 va_list       ap;
 190
 191                 fprintf(stderr, "rank %d: ", rank);
 192                 va_start(ap, fmt);
 193                 vfprintf(stderr, fmt, ap);
 194                 va_end(ap);
 195         }
 196
 197         MPI_Abort(MPI_COMM_WORLD, 1);
 198         exit(1);
 199 }
 200
 201 static void
 202 sigalrm_handler(int signum)
 203 {
 204         alarm_caught++;
 205 }
 206
 207 /* HAVE_LLAPI_FILE_LOOKUP is defined by liblustreapi.h if this function is
 208  * defined therein.  Otherwise we can do the equivalent operation via ioctl
 209  * if we have access to a complete lustre build tree to get the various
 210  * definitions - then compile with USE_MDC_LOOKUP defined. */
 211 #if defined(HAVE_LLAPI_FILE_LOOKUP)
 212 #define HAVE_MDC_LOOKUP
 213 #elif defined(USE_MDC_LOOKUP)
 214 #include <config.h>
 215 #include <liblustre.h>
 216 #include <linux/lustre_lib.h>
 217
 218 int llapi_file_lookup(int dirfd, const char *name)
 219 {
 220         struct obd_ioctl_data data = { 0 };
 221         char rawbuf[8192];
 222         char *buf = rawbuf;
 223         int rc;
 224
 225         if (dirfd < 0 || name == NULL)
 226                 return -EINVAL;
 227
 228         data.ioc_version = OBD_IOCTL_VERSION;
 229         data.ioc_len = sizeof(data);
 230         data.ioc_inlbuf1 = name;
 231         data.ioc_inllen1 = strlen(name) + 1;
 232
 233         rc = obd_ioctl_pack(&data, &buf, sizeof(rawbuf));
 234         if (rc) {
 235                 fatal(myrank, "ioctl_pack failed: rc = %d\n", rc);
 236                 return rc;
 237         }
 238
 239         return ioctl(fd, IOC_MDC_LOOKUP, buf);
 240 }
 241 #define HAVE_MDC_LOOKUP
 242 #endif
 243
 244 static void
 245 process_args(int argc, char *argv[])
 246 {
 247         char   *cp, *endptr;
 248         int    i, index, offset, tmpend, rc;
 249         char   tmp[16];
 250         FILE * seed_file;
 251         struct option *opt;
 252
 253         setbuf(stdout, 0);
 254         setbuf(stderr, 0);
 255         prog = basename(argv[0]);
 256         strcpy(filefmt, "f%d");
 257         gethostname(hostname, sizeof(hostname));
 258
 259         /* auto create shortOpts rather than maintaining a static string. */
 260         for (opt = longOpts, cp = shortOpts; opt->name != NULL; opt++, cp++) {
 261                 *cp = opt->val;
 262                 if (opt->has_arg)
 263                         *++cp = ':';
 264         }
 265
 266         while ((rc = getopt_long(argc,argv, shortOpts, longOpts,&index)) != -1) {
 267                 switch (rc) {
 268                 case OPEN:
 269                         openflags &= ~(O_CREAT|O_EXCL);
 270                 case CREATE:
 271 #ifdef HAVE_MDC_LOOKUP
 272                 case LOOKUP:
 273 #endif
 274                 case MKNOD:
 275                 case STAT:
 276                 case UNLINK:
 277                         if (cmd != NULL) {
 278                                 fatal(0, "Invalid - more than one operation "
 279                                            "specified: --%s\n",
 280                                         longOpts[index].name);
 281                         }
 282                         mode = rc;
 283                         cmd = (char *)longOpts[index].name;
 284                         break;
 285                 case NOEXCL:
 286                         if (mode != CREATE && mode != MKNOD) {
 287                                 usage(stderr, "--noexcl only applies to "
 288                                               "--create or --mknod.\n");
 289                         }
 290                         openflags &= ~O_EXCL;
 291                         break;
 292                 case RECREATE:
 293                         if (mode != UNLINK) {
 294                                 usage(stderr, "--recreate only makes sense"
 295                                               "with --unlink.\n");
 296                         }
 297                         recreate++;
 298                         break;
 299                 case BEGIN:
 300                         begin = strtol(optarg, &endptr, 0);
 301                         if ((*endptr != 0) || (begin < 0)) {
 302                                 fatal(0, "Invalid --start value.\n");
 303                         }
 304                         break;
 305                 case ITERS:
 306                         iters = strtol(optarg, &endptr, 0);
 307                         if ((*endptr != 0) || (iters <= 0)) {
 308                                 fatal(0, "Invalid --iters value.\n");
 309                         }
 310                         if (mode != LOOKUP && mode != OPEN && mode != STAT) {
 311                                 usage(stderr, "--iters only makes sense with "
 312                                               "--lookup, --open, or --stat.\n");
 313                         }
 314                         break;
 315                 case TIME:
 316                         seconds = strtol(optarg, &endptr, 0);
 317                         if ((*endptr != 0) || (seconds <= 0)) {
 318                                 fatal(0, "Invalid --time value.\n");
 319                         }
 320                         break;
 321                 case DIRFMT:
 322                         if (strlen(optarg) > (PATH_MAX - 16)) {
 323                                 fatal(0, "--dirfmt too long\n");
 324                         }
 325                         dirfmt = optarg;
 326                         break;
 327                 case NDIRS:
 328                         ndirs = strtol(optarg, &endptr, 0);
 329                         if ((*endptr != 0) || (ndirs <= 0)) {
 330                                 fatal(0, "Invalid --ndirs value.\n");
 331                         }
 332                         if ((ndirs > nthreads) &&
 333                             ((mode == CREATE) || (mode == MKNOD))) {
 334                                 fatal(0, "--ndirs=%d must be less than or "
 335                                       "equal to the number of threads (%d).\n",
 336                                       ndirs, nthreads);
 337                         }
 338                         break;
 339                 case FILEFMT:
 340                         if (strlen(optarg) > 4080) {
 341                                 fatal(0, "--filefmt too long\n");
 342                         }
 343
 344                         /* Use %%d where you want the file # in the name. */
 345                         sprintf(filefmt, optarg, myrank);
 346                         break;
 347                 case NFILES:
 348                         nfiles = strtol(optarg, &endptr, 0);
 349                         if ((*endptr != 0) || (nfiles <= 0)) {
 350                                 fatal(0, "Invalid --nfiles value.\n");
 351                         }
 352                         break;
 353                 case STRIPES:
 354                         stripes = strtol(optarg, &endptr, 0);
 355                         if ((*endptr != 0) || (stripes < 0)) {
 356                                 fatal(0, "Invalid --stripes value.\n");
 357                         }
 358
 359                         if (stripes == 0) {
 360                                 openflags |= O_LOV_DELAY_CREATE;
 361                         } else {
 362                                 fatal(0, "non-zero --stripes value "
 363                                          "not yet supported.\n");
 364                         }
 365
 366                         break;
 367                 case SEED:
 368                         seed = strtoul(optarg, &endptr, 0);
 369                         if (*endptr) {
 370                                 fatal(0, "bad --seed option %s\n", optarg);
 371                         }
 372                         break;
 373                 case SEEDFILE:
 374                         seed_file = fopen(optarg, "r");
 375                         if (!seed_file) {
 376                               fatal(myrank, "fopen(%s) error: %s\n",
 377                                       optarg, strerror(errno));
 378                         }
 379
 380                         for (i = -1; fgets(tmp, 16, seed_file) != NULL;) {
 381                                 if (++i == myrank)
 382                                         break;
 383                         }
 384
 385                         if (i == myrank) {
 386                                 rc = sscanf(tmp, "%d", &seed);
 387                                 if ((rc != 1) || (seed < 0)) {
 388                                         fatal(myrank, "Invalid seed value '%s' "
 389                                               "at line %d in %s.\n",
 390                                               tmp, i, optarg);
 391                                 }
 392                         } else {
 393                                 fatal(myrank, "File '%s' too short. Does not "
 394                                       "contain a seed for thread %d.\n",
 395                                       optarg, myrank);
 396                         }
 397
 398                         fclose(seed_file);
 399                         break;
 400                 case RANDOM:
 401                 case READDIR:
 402                         if (mode != LOOKUP && mode != OPEN && mode != STAT)  {
 403                                 fatal(0, "--%s can only be specified with "
 404                                          "--lookup, --open, or --stat.\n",
 405                                       (char *)longOpts[index].name);
 406                         }
 407                         order = rc;
 408                         break;
 409                 case IGNORE:
 410                         ++ignore;
 411                         break;
 412                 case DEBUG:
 413                         ++debug;
 414                 case VERBOSE:
 415                         ++verbose;
 416                         break;
 417                 case HELP:
 418                         usage(stdout, NULL);
 419                 default:
 420                         usage(stderr, "unrecognized option: '%c'.\n", optopt);
 421                 }
 422         }
 423
 424         if (optind < argc) {
 425                 usage(stderr, "too many arguments %d >= %d.\n", optind, argc);
 426         }
 427
 428         if (mode == CREATE || mode == MKNOD || mode == UNLINK) {
 429                 if (seconds != 0) {
 430                         if (nfiles == 0)
 431                                 nfiles = INT_MAX;
 432                 } else if (nfiles == 0) {
 433                         usage(stderr, "--nfiles or --time must be specified "
 434                                       "with %s.\n", cmd);
 435                 }
 436         } else if (mode == LOOKUP || mode == OPEN || mode == STAT) {
 437                 if (seconds != 0) {
 438                         if (iters == 0)
 439                                 iters = INT_MAX;
 440                 } else if (iters == 0) {
 441                         usage(stderr, "--iters or --time must be specifed "
 442                                       "with %s.\n", cmd);
 443                 }
 444
 445                 if (nfiles == 0) {
 446                         usage(stderr, "--nfiles must be specifed with --%s.\n",
 447                               cmd);
 448                 }
 449
 450                 if (seed == 0) {
 451                         int fd = open("/dev/urandom", O_RDONLY);
 452
 453                         if (fd >= 0) {
 454                                 if (read(fd, &seed, sizeof(seed)) <
 455                                     sizeof(seed))
 456                                         seed = time(0);
 457                                 close(fd);
 458                         } else {
 459                                 seed = time(0);
 460                         }
 461                 }
 462
 463                 srand(seed);
 464
 465                 dmesg("%s: rank %d seed %d (%s).\n", prog, myrank, seed,
 466                       (order == RANDOM) ? "random_order" : "readdir_order");
 467         } else {
 468                 usage(stderr, "one --create, --mknod, --open, --stat,"
 469 #ifdef HAVE_MDC_LOOKUP
 470                       " --lookup,"
 471 #endif
 472                       " or --unlink must be specifed.");
 473         }
 474
 475         /* support for multiple threads in a dir, set begin/end appropriately.*/
 476         dirnum = myrank % ndirs;
 477         dirthreads = nthreads / ndirs;
 478         if (nthreads > (ndirs * dirthreads + dirnum))
 479                 ++dirthreads;
 480
 481         offset = myrank / ndirs;
 482
 483         tmpend = begin + nfiles - 1;
 484         if (tmpend <= 0)
 485                 tmpend = INT_MAX;
 486
 487         end = begin + (nfiles / dirthreads) * dirthreads + offset;
 488         if ((end > tmpend) || (end <= 0))
 489                 end -= dirthreads;
 490
 491         begin += offset;
 492         if (begin < 0)
 493                 begin = INT_MAX;
 494
 495         beginsave = begin;
 496
 497         dmesg("%d: iters %d nfiles %d time %d begin %d end %d dirthreads %d."
 498               "\n", myrank, iters, nfiles, seconds, begin, end, dirthreads);
 499
 500         if (dirfmt == NULL) {
 501                 strcpy(dir, ".");
 502         } else {
 503                 sprintf(dir, dirfmt, dirnum);
 504
 505                 sprintf(mkdir_cmd, "/bin/mkdir -p %s", dir);
 506                 #ifdef _LIGHTWEIGHT_KERNEL
 507                         printf("NOTICE: not running system(%s)\n", mkdir_cmd);
 508                 #else
 509                         rc = system(mkdir_cmd);
 510                         if (rc) {
 511                                 fatal(myrank, "'%s' failed.\n", mkdir_cmd);
 512                         }
 513                 #endif
 514
 515                 rc = chdir(dir);
 516                 if (rc) {
 517                         fatal(myrank, "unable to chdir to '%s'.\n", dir);
 518                 }
 519         }
 520 }
 521
 522 static inline char *next_file()
 523 {
 524         if (order == RANDOM) {
 525                 sprintf(filename, filefmt, random() % nfiles);
 526                 return(filename);
 527         }
 528
 529         /* readdir order */
 530
 531         dir_entry = readdir(directory);
 532         if (dir_entry == NULL) {
 533                 rewinddir(directory);
 534                 while ((dir_entry = readdir(directory)) != NULL) {
 535                         if (dir_entry->d_name[0] != '.')
 536                                 return(dir_entry->d_name);
 537                 }
 538
 539                 fatal(myrank, "unable to read directory %s (%s).\n",
 540                       dir, strerror(errno));
 541         }
 542
 543         return(dir_entry->d_name);
 544 }
 545
 546 int
 547 main(int argc, char *argv[])
 548 {
 549         int    i, j, fd, rc, nops, lastOps, ag_ops;
 550         int    ag_interval = 0;
 551         float  rate, ag_rate;
 552         float  avg_rate = 0;
 553         float  effective_rate = 0;
 554         time_t startTime, lastTime, curTime, interval;
 555         char * file;
 556
 557         rc = MPI_Init(&argc, &argv);
 558         if (rc != MPI_SUCCESS)
 559                 fatal(myrank, "MPI_Init failed: %d\n", rc);
 560
 561         rc = MPI_Comm_size(MPI_COMM_WORLD, &nthreads);
 562         if (rc != MPI_SUCCESS)
 563                 fatal(myrank, "MPI_Comm_size failed: %d\n", rc);
 564
 565         rc = MPI_Comm_rank(MPI_COMM_WORLD, &myrank);
 566         if (rc != MPI_SUCCESS)
 567                 fatal(myrank, "MPI_Comm_rank failed: %d\n", rc);
 568
 569         process_args(argc, argv);
 570
 571         startTime = time(0);
 572         if ((myrank == 0) || debug) {
 573                 printf("%d: %s starting at %s",
 574                        myrank, hostname, ctime(&startTime));
 575         }
 576
 577         /* if we're not measuring creation rates then precreate
 578          * the files we're operating on. */
 579         if ((mode != CREATE) && (mode != MKNOD) && !ignore) {
 580                 /* create the files in reverse order. When we encounter
 581                  * a file that already exists, assume the remainder of
 582                  * the files exist to save time. The timed performance
 583                  * test scripts make use of this behavior. */
 584                 for (i = end, j = 0; i >= begin; i -= dirthreads) {
 585                         sprintf(filename, filefmt, i);
 586                         fd = open(filename, openflags, 0644);
 587                         if (fd < 0) {
 588                                 if (errno == EEXIST)
 589                                         break;
 590                                 rc = errno;
 591                                 fatal(myrank, "precreate open(%s) error: %s\n",
 592                                       filename, strerror(rc));
 593                         }
 594                         j++;
 595                         close(fd);
 596                 }
 597                 dmesg("%d: %s pre-created %d files.\n",myrank,hostname,j);
 598
 599                 rc = MPI_Barrier(MPI_COMM_WORLD);
 600                 if (rc != MPI_SUCCESS)
 601                         fatal(myrank, "prep MPI_Barrier failed: %d\n", rc);
 602         }
 603
 604         if (order == READDIR) {
 605                 directory = opendir(dir);
 606                 if (directory == NULL) {
 607                         rc = errno;
 608                         fatal(myrank, "opendir(%s) error: %s\n",
 609                               dir, strerror(rc));
 610                 }
 611
 612                 startTime = time(0);
 613                 j = random() % nfiles;
 614                 dmesg("%d: %s initializing dir offset %u: %s",
 615                       myrank, hostname, j, ctime(&startTime));
 616
 617                 for (i = 0; i <= j; i++) {
 618                         if ((dir_entry = readdir(directory)) == NULL) {
 619                                 fatal(myrank, "could not read entry number %d "
 620                                       "in directory %s.\n", i, dir);
 621                         }
 622                 }
 623
 624                 lastTime = time(0);
 625                 dmesg("%d: index %d, filename %s, offset %ld: "
 626                       "%s initialization complete: %s",
 627                       myrank, i, dir_entry->d_name, telldir(directory),
 628                       hostname, ctime(&lastTime));
 629         }
 630
 631         rc = MPI_Barrier(MPI_COMM_WORLD);
 632         if (rc != MPI_SUCCESS)
 633                 fatal(myrank, "prep MPI_Barrier failed: %d\n", rc);
 634
 635         if (seconds) {
 636                 act.sa_handler = sigalrm_handler;
 637                 (void)sigemptyset(&act.sa_mask);
 638                 act.sa_flags = 0;
 639                 sigaction(SIGALRM, &act, NULL);
 640                 alarm(seconds);
 641         }
 642
 643         startTime = lastTime = time(0);
 644         nops = lastOps = 0;
 645
 646         switch (mode) {
 647         case CREATE:
 648                 for (; begin <= end && !alarm_caught; begin += dirthreads) {
 649                         sprintf(filename, filefmt, begin);
 650                         if ((fd = open(filename, openflags, 0644)) < 0) {
 651                                 if (((rc = errno) == EINTR) && alarm_caught)
 652                                         break;
 653                                 fatal(myrank, "open(%s) error: %s\n",
 654                                       filename, strerror(rc));
 655                         }
 656
 657                         close(fd);
 658                         DISPLAY_PROGRESS();
 659                 }
 660
 661                 dmesg("%d: created %d files, last file '%s'.\n",
 662                       myrank, nops, filename);
 663                 break;
 664 #ifdef HAVE_MDC_LOOKUP
 665         case LOOKUP:
 666                 fd = open(dir, O_RDONLY);
 667                 if (fd < 0) {
 668                         fatal(myrank, "open(dir == '%s') error: %s\n",
 669                               dir, strerror(errno));
 670                 }
 671
 672                 for (; nops < iters && !alarm_caught;) {
 673                         char *filename = next_file();
 674                         rc = llapi_file_lookup(fd, filename);
 675                         if (rc < 0) {
 676                                 if (((rc = errno) == EINTR) && alarm_caught)
 677                                         break;
 678                                 fatal(myrank, "llapi_file_lookup(%s) "
 679                                       "error: %s\n", filename, strerror(rc));
 680                         }
 681
 682                         DISPLAY_PROGRESS();
 683                 }
 684                 break;
 685 #endif
 686         case MKNOD:
 687                 for (; begin <= end && !alarm_caught; begin += dirthreads) {
 688                         sprintf(filename, filefmt, begin);
 689                         rc = mknod(filename, S_IFREG| 0644, 0);
 690                         if (rc) {
 691                                 if (((rc = errno) == EINTR) && alarm_caught)
 692                                         break;
 693                                 fatal(myrank, "mknod(%s) error: %s\n",
 694                                       filename, strerror(rc));
 695                         }
 696
 697                         DISPLAY_PROGRESS();
 698                 }
 699                 break;
 700         case OPEN:
 701                 for (; nops < iters && !alarm_caught;) {
 702                         file = next_file();
 703                         if ((fd = open(file, openflags, 0644)) < 0) {
 704                                 if (((rc = errno) == EINTR) && alarm_caught)
 705                                         break;
 706                                 fatal(myrank, "open(%s) error: %s\n",
 707                                       file, strerror(rc));
 708                         }
 709
 710                         close(fd);
 711
 712                         DISPLAY_PROGRESS();
 713                 }
 714                 break;
 715         case STAT:
 716                 for (; nops < iters && !alarm_caught;) {
 717                         rc = stat(file = next_file(), &statbuf);
 718                         if (rc) {
 719                                 if (((rc = errno) == EINTR) && alarm_caught)
 720                                         break;
 721                                 fatal(myrank, "stat(%s) error: %s\n",
 722                                       file, strerror(rc));
 723                         }
 724
 725                         DISPLAY_PROGRESS();
 726                 }
 727                 break;
 728         case UNLINK:
 729                 for (; begin <= end && !alarm_caught; begin += dirthreads) {
 730                         sprintf(filename, filefmt, begin);
 731                         rc = unlink(filename);
 732                         if (rc) {
 733                                 if (((rc = errno) == EINTR) && alarm_caught)
 734                                         break;
 735                                 if (((rc = errno) == ENOENT) && ignore)
 736                                         continue;
 737                                 fatal(myrank, "unlink(%s) error: %s\n",
 738                                       filename, strerror(rc));
 739                         }
 740
 741                         DISPLAY_PROGRESS();
 742                 }
 743                 break;
 744         }
 745
 746         curTime = time(0);
 747         interval = curTime - startTime;
 748         rate = (float)(nops);
 749         if (interval != 0)
 750                 rate /= (float)interval;
 751
 752         rc = MPI_Reduce(&nops, &ag_ops, 1, MPI_INT, MPI_SUM, 0,
 753                         MPI_COMM_WORLD);
 754         if (rc != MPI_SUCCESS) {
 755                 fatal(myrank, "Failure in MPI_Reduce of total ops.\n");
 756         }
 757
 758         rc = MPI_Reduce(&interval, &ag_interval, 1, MPI_INT, MPI_SUM, 0,
 759                         MPI_COMM_WORLD);
 760         if (rc != MPI_SUCCESS) {
 761                 fatal(myrank, "Failure in MPI_Reduce of total interval.\n");
 762         }
 763
 764         rc = MPI_Reduce(&rate, &ag_rate, 1, MPI_FLOAT, MPI_SUM, 0,
 765                         MPI_COMM_WORLD);
 766         if (rc != MPI_SUCCESS) {
 767                 fatal(myrank, "Failure in MPI_Reduce of aggregated rate.\n");
 768         }
 769
 770         if (myrank == 0) {
 771                 curTime = time(0);
 772                 interval = curTime - startTime;
 773                 if (interval != 0)
 774                         effective_rate = (float) ag_ops/ (float)interval;
 775
 776                 if (ag_interval > 0)
 777                         avg_rate = (float)ag_ops/(float)ag_interval;
 778
 779
 780                 printf("Rate: %.2f eff %.2f aggr %.2f client %ss/sec "
 781                        "(total: %d threads %d %ss %d dirs %d threads/dir %lu secs)\n",
 782                        effective_rate, ag_rate, avg_rate, cmd, nthreads, ag_ops, cmd,
 783                        ndirs, dirthreads, interval);
 784         }
 785
 786         if (recreate) {
 787                 for (begin = beginsave; begin <= end; begin += dirthreads) {
 788                         sprintf(filename, filefmt, begin);
 789                         if ((fd = open(filename, openflags, 0644)) < 0) {
 790                                 rc = errno;
 791                                 if (rc == EEXIST)
 792                                         break;
 793                                 fatal(myrank, "recreate open(%s) error: %s\n",
 794                                       filename, strerror(rc));
 795                         }
 796
 797                         close(fd);
 798                 }
 799         }
 800
 801         curTime = time(0);
 802         if ((myrank == 0) || debug) {
 803                 printf("%d: %s finished at %s",
 804                        myrank, hostname, ctime(&curTime));
 805         }
 806
 807         MPI_Finalize();
 808         return(0);
 809 }