lustre/tests/mpi/mdsrate.c

   1 /*
   2  * 2003, Copyright, Hewlett-Packard Development Compnay, LP.
   3  *
   4  * Developed under the sponsorship of the U.S. Government
   5  *     under Subcontract No. B514193
   6  */
   7
   8 /*
   9  * Copyright (c) 2008, 2010, Oracle and/or its affiliates. All rights reserved.
  10  * Use is subject to license terms.
  11  */
  12
  13 #include <stdio.h>
  14 #include <getopt.h>
  15 #include <libgen.h>
  16 #include <sys/types.h>
  17 #include <sys/stat.h>
  18 #include <time.h>
  19 #include <limits.h>
  20 #include <errno.h>
  21 #include <string.h>
  22 #include <fcntl.h>
  23 #include <unistd.h>
  24 #include <stdlib.h>
  25 #include <stdarg.h>
  26 #include <signal.h>
  27 #include <sys/ioctl.h>
  28 #include <dirent.h>
  29
  30 #include "mpi.h"
  31
  32 /* lustre */
  33 #include <lustre/lustreapi.h>        /* for O_LOV_DELAY_CREATE */
  34
  35 #define CHECK_COUNT 10000
  36 #define DISPLAY_COUNT (CHECK_COUNT * 10)
  37 #define DISPLAY_TIME 100
  38
  39 enum {
  40         CREATE   = 'c',
  41         LOOKUP   = 'l',
  42         MKNOD    = 'm',
  43         OPEN     = 'o',
  44         STAT     = 's',
  45         UNLINK   = 'u',
  46         BEGIN    = 'b',
  47         ITERS    = 'i',
  48         TIME     = 't',
  49         DIRFMT   = 'd',
  50         NDIRS    = 'D',
  51         FILEFMT  = 'f',
  52         NFILES   = 'F',
  53         NOEXCL   = 'X',
  54         STRIPES  = 'S',
  55         SEED     = 'r',
  56         SEEDFILE = 'R',
  57         RANDOM   = 'A',
  58         READDIR  = 'B',
  59         RECREATE = 'C',
  60         IGNORE   = 'E',
  61         VERBOSE  = 'V',
  62         DEBUG    = 'v',
  63         HELP     = 'h',
  64 };
  65
  66 struct option longOpts[] = {
  67         {"create",        0, NULL, CREATE     },
  68         {"lookup",        0, NULL, LOOKUP     },
  69         {"mknod",         0, NULL, MKNOD      },
  70         {"open",          0, NULL, OPEN       },
  71         {"stat",          0, NULL, STAT       },
  72         {"unlink",        0, NULL, UNLINK     },
  73         {"begin",         1, NULL, BEGIN      },
  74         {"iters",         1, NULL, ITERS      },
  75         {"time",          1, NULL, TIME       },   /* seconds */
  76         {"dirfmt",        1, NULL, DIRFMT     },
  77         {"ndirs",         1, NULL, NDIRS      },
  78         {"filefmt",       1, NULL, FILEFMT    },
  79         {"nfiles",        1, NULL, NFILES     },
  80         {"noexcl",        0, NULL, NOEXCL     },
  81         {"stripes",       1, NULL, STRIPES    },
  82         {"seed",          1, NULL, SEED       },
  83         {"seedfile",      1, NULL, SEEDFILE   },
  84         {"random_order",  0, NULL, RANDOM     },
  85         {"readdir_order", 0, NULL, READDIR    },
  86         {"recreate",      0, NULL, RECREATE   },
  87         {"ignore",        0, NULL, IGNORE     },
  88         {"verbose",       0, NULL, VERBOSE    },
  89         {"debug",         0, NULL, DEBUG      },
  90         {"help",          0, NULL, HELP       },
  91         { 0,              0, NULL, 0          }
  92 };
  93
  94 int foo1, foo2;
  95
  96 char   shortOpts[128];
  97 int    myrank = -1;
  98 int    nthreads = -1;
  99 char * prog;
 100 char   hostname[512] = "unknown";
 101 char   mode;
 102 char * cmd;
 103 int    openflags = O_RDWR|O_CREAT|O_EXCL;
 104 int    ndirs = 1;
 105 char * dirfmt;
 106 char   dir[PATH_MAX];
 107 char   mkdir_cmd[PATH_MAX+14];
 108 int    dirthreads;
 109 int    dirnum;
 110 DIR *  directory;
 111 struct dirent *dir_entry;
 112 int    nfiles;
 113 char   filefmt[PATH_MAX];
 114 char   filename[PATH_MAX];
 115 int    stripes = -1;
 116 int    begin;
 117 int    beginsave;
 118 int    end;
 119 int    iters;
 120 int    seconds;
 121 int    alarm_caught;
 122 struct sigaction act;
 123 int    order = RANDOM;
 124 int    seed;
 125 int    recreate;
 126 int    ignore;
 127 int    verbose;
 128 int    debug;
 129 struct stat statbuf;
 130
 131 #define dmesg if (debug) printf
 132
 133 #define DISPLAY_PROGRESS() {                                                \
 134         if (verbose && (nops % CHECK_COUNT == 0)) {                         \
 135                 curTime = MPI_Wtime();                                      \
 136                 interval = curTime - lastTime;                              \
 137                 if (interval > DISPLAY_TIME || nops % DISPLAY_COUNT == 0) { \
 138                         rate = (double)(nops - lastOps)/interval;           \
 139                         printf("Rank %d: %.2f %ss/sec %.2f secs "           \
 140                                "(total: %d %ss %.2f secs)\n",               \
 141                                myrank, rate, cmd, interval,                 \
 142                                nops, cmd, curTime - startTime);             \
 143                         lastOps = nops;                                     \
 144                         lastTime = curTime;                                 \
 145                 }                                                           \
 146         }                                                                   \
 147 }
 148
 149 char *usage_msg = "usage: %s\n"
 150                   "    { --create [ --noexcl ] | --lookup | --mknod |\n"
 151                   "      --open | --stat | --unlink  [ --recreate ] [ --ignore ] }\n"
 152                   "    [ --help ] [ --verbose ] [ --debug ]\n"
 153                   "    { [ --begin <num> ] --nfiles <num> }\n"
 154                   "    [ --iters <num> ] [ --time <secs> ]\n"
 155                   "    [ --dirfmt <str> ] [ --ndirs  <num> ]\n"
 156                   "    [ --filefmt <str> ] [ --stripes <num> ]\n"
 157                   "    [ --random_order [--seed <num> | --seedfile <file>] ]\n"
 158                   "    [ --readdir_order ]\n";
 159
 160 static void
 161 usage(FILE *stream, char *fmt, ...)
 162 {
 163         if (myrank == 0) {
 164                 if (fmt != NULL) {
 165                         va_list       ap;
 166
 167                         fprintf(stream, "%s: ", prog);
 168                         va_start(ap, fmt);
 169                         vfprintf(stderr, fmt, ap);
 170                         va_end(ap);
 171                 }
 172                 fprintf(stream, usage_msg, prog);
 173         }
 174
 175         MPI_Finalize();
 176         exit(stream == stderr);
 177 }
 178
 179 /* Print process myrank and message, and exit (i.e. a fatal error) */
 180 static int
 181 fatal(int rank, const char *fmt, ...)
 182 {
 183         if (rank == myrank) {
 184                 va_list       ap;
 185
 186                 fprintf(stderr, "rank %d: ", rank);
 187                 va_start(ap, fmt);
 188                 vfprintf(stderr, fmt, ap);
 189                 va_end(ap);
 190         }
 191
 192         MPI_Abort(MPI_COMM_WORLD, 1);
 193         exit(1);
 194 }
 195
 196 static void
 197 sigalrm_handler(int signum)
 198 {
 199         alarm_caught++;
 200 }
 201
 202 /* HAVE_LLAPI_FILE_LOOKUP is defined by liblustreapi.h if this function is
 203  * defined therein.  Otherwise we can do the equivalent operation via ioctl
 204  * if we have access to a complete lustre build tree to get the various
 205  * definitions - then compile with USE_MDC_LOOKUP defined. */
 206 #if defined(HAVE_LLAPI_FILE_LOOKUP)
 207 #define HAVE_MDC_LOOKUP
 208 #elif defined(USE_MDC_LOOKUP)
 209 #include <config.h>
 210 #include <liblustre.h>
 211 #include <linux/lustre_lib.h>
 212
 213 int llapi_file_lookup(int dirfd, const char *name)
 214 {
 215         struct obd_ioctl_data data = { 0 };
 216         char rawbuf[8192];
 217         char *buf = rawbuf;
 218         int rc;
 219
 220         if (dirfd < 0 || name == NULL)
 221                 return -EINVAL;
 222
 223         data.ioc_version = OBD_IOCTL_VERSION;
 224         data.ioc_len = sizeof(data);
 225         data.ioc_inlbuf1 = name;
 226         data.ioc_inllen1 = strlen(name) + 1;
 227
 228         rc = obd_ioctl_pack(&data, &buf, sizeof(rawbuf));
 229         if (rc) {
 230                 fatal(myrank, "ioctl_pack failed: rc = %d\n", rc);
 231                 return rc;
 232         }
 233
 234         return ioctl(fd, IOC_MDC_LOOKUP, buf);
 235 }
 236 #define HAVE_MDC_LOOKUP
 237 #endif
 238
 239 static void
 240 process_args(int argc, char *argv[])
 241 {
 242         char   *cp, *endptr;
 243         int    i, index, offset, tmpend, rc;
 244         char   tmp[16];
 245         FILE * seed_file;
 246         struct option *opt;
 247
 248         setbuf(stdout, 0);
 249         setbuf(stderr, 0);
 250         prog = basename(argv[0]);
 251         strcpy(filefmt, "f%d");
 252         gethostname(hostname, sizeof(hostname));
 253
 254         /* auto create shortOpts rather than maintaining a static string. */
 255         for (opt = longOpts, cp = shortOpts; opt->name != NULL; opt++, cp++) {
 256                 *cp = opt->val;
 257                 if (opt->has_arg)
 258                         *++cp = ':';
 259         }
 260
 261         while ((rc = getopt_long(argc,argv, shortOpts, longOpts,&index)) != -1) {
 262                 switch (rc) {
 263                 case OPEN:
 264                         openflags &= ~(O_CREAT|O_EXCL);
 265                 case CREATE:
 266 #ifdef HAVE_MDC_LOOKUP
 267                 case LOOKUP:
 268 #endif
 269                 case MKNOD:
 270                 case STAT:
 271                 case UNLINK:
 272                         if (cmd != NULL) {
 273                                 fatal(0, "Invalid - more than one operation "
 274                                            "specified: --%s\n",
 275                                         longOpts[index].name);
 276                         }
 277                         mode = rc;
 278                         cmd = (char *)longOpts[index].name;
 279                         break;
 280                 case NOEXCL:
 281                         if (mode != CREATE && mode != MKNOD) {
 282                                 usage(stderr, "--noexcl only applies to "
 283                                               "--create or --mknod.\n");
 284                         }
 285                         openflags &= ~O_EXCL;
 286                         break;
 287                 case RECREATE:
 288                         if (mode != UNLINK) {
 289                                 usage(stderr, "--recreate only makes sense"
 290                                               "with --unlink.\n");
 291                         }
 292                         recreate++;
 293                         break;
 294                 case BEGIN:
 295                         begin = strtol(optarg, &endptr, 0);
 296                         if ((*endptr != 0) || (begin < 0)) {
 297                                 fatal(0, "Invalid --start value.\n");
 298                         }
 299                         break;
 300                 case ITERS:
 301                         iters = strtol(optarg, &endptr, 0);
 302                         if ((*endptr != 0) || (iters <= 0)) {
 303                                 fatal(0, "Invalid --iters value.\n");
 304                         }
 305                         if (mode != LOOKUP && mode != OPEN) {
 306                                 usage(stderr, "--iters only makes sense with "
 307                                               "--lookup or --open.\n");
 308                         }
 309                         break;
 310                 case TIME:
 311                         seconds = strtol(optarg, &endptr, 0);
 312                         if ((*endptr != 0) || (seconds <= 0)) {
 313                                 fatal(0, "Invalid --time value.\n");
 314                         }
 315                         break;
 316                 case DIRFMT:
 317                         if (strlen(optarg) > (PATH_MAX - 16)) {
 318                                 fatal(0, "--dirfmt too long\n");
 319                         }
 320                         dirfmt = optarg;
 321                         break;
 322                 case NDIRS:
 323                         ndirs = strtol(optarg, &endptr, 0);
 324                         if ((*endptr != 0) || (ndirs <= 0)) {
 325                                 fatal(0, "Invalid --ndirs value.\n");
 326                         }
 327                         if ((ndirs > nthreads) &&
 328                             ((mode == CREATE) || (mode == MKNOD))) {
 329                                 fatal(0, "--ndirs=%d must be less than or "
 330                                       "equal to the number of threads (%d).\n",
 331                                       ndirs, nthreads);
 332                         }
 333                         break;
 334                 case FILEFMT:
 335                         if (strlen(optarg) > 4080) {
 336                                 fatal(0, "--filefmt too long\n");
 337                         }
 338
 339                         /* Use %%d where you want the file # in the name. */
 340                         sprintf(filefmt, optarg, myrank);
 341                         break;
 342                 case NFILES:
 343                         nfiles = strtol(optarg, &endptr, 0);
 344                         if ((*endptr != 0) || (nfiles <= 0)) {
 345                                 fatal(0, "Invalid --nfiles value.\n");
 346                         }
 347                         break;
 348                 case STRIPES:
 349                         stripes = strtol(optarg, &endptr, 0);
 350                         if ((*endptr != 0) || (stripes < 0)) {
 351                                 fatal(0, "Invalid --stripes value.\n");
 352                         }
 353
 354                         if (stripes == 0) {
 355                                 openflags |= O_LOV_DELAY_CREATE;
 356                         } else {
 357                                 fatal(0, "non-zero --stripes value "
 358                                          "not yet supported.\n");
 359                         }
 360
 361                         break;
 362                 case SEED:
 363                         seed = strtoul(optarg, &endptr, 0);
 364                         if (*endptr) {
 365                                 fatal(0, "bad --seed option %s\n", optarg);
 366                         }
 367                         break;
 368                 case SEEDFILE:
 369                         seed_file = fopen(optarg, "r");
 370                         if (!seed_file) {
 371                               fatal(myrank, "fopen(%s) error: %s\n",
 372                                       optarg, strerror(errno));
 373                         }
 374
 375                         for (i = -1; fgets(tmp, 16, seed_file) != NULL;) {
 376                                 if (++i == myrank)
 377                                         break;
 378                         }
 379
 380                         if (i == myrank) {
 381                                 rc = sscanf(tmp, "%d", &seed);
 382                                 if ((rc != 1) || (seed < 0)) {
 383                                         fatal(myrank, "Invalid seed value '%s' "
 384                                               "at line %d in %s.\n",
 385                                               tmp, i, optarg);
 386                                 }
 387                         } else {
 388                                 fatal(myrank, "File '%s' too short. Does not "
 389                                       "contain a seed for thread %d.\n",
 390                                       optarg, myrank);
 391                         }
 392
 393                         fclose(seed_file);
 394                         break;
 395                 case RANDOM:
 396                 case READDIR:
 397                         if (mode != LOOKUP && mode != OPEN)  {
 398                                 fatal(0, "--%s can only be specified with "
 399                                          "--lookup, or --open.\n",
 400                                       (char *)longOpts[index].name);
 401                         }
 402                         order = rc;
 403                         break;
 404                 case IGNORE:
 405                         ++ignore;
 406                         break;
 407                 case DEBUG:
 408                         ++debug;
 409                 case VERBOSE:
 410                         ++verbose;
 411                         break;
 412                 case HELP:
 413                         usage(stdout, NULL);
 414                 default:
 415                         usage(stderr, "unrecognized option: '%c'.\n", optopt);
 416                 }
 417         }
 418
 419         if (optind < argc) {
 420                 usage(stderr, "too many arguments %d >= %d.\n", optind, argc);
 421         }
 422
 423         if (mode == CREATE || mode == MKNOD || mode == UNLINK || mode == STAT) {
 424                 if (seconds != 0) {
 425                         if (nfiles == 0)
 426                                 nfiles = INT_MAX;
 427                 } else if (nfiles == 0) {
 428                         usage(stderr, "--nfiles or --time must be specified "
 429                                       "with %s.\n", cmd);
 430                 }
 431         } else if (mode == LOOKUP || mode == OPEN) {
 432                 if (seconds != 0) {
 433                         if (iters == 0)
 434                                 iters = INT_MAX;
 435                 } else if (iters == 0) {
 436                         usage(stderr, "--iters or --time must be specifed "
 437                                       "with %s.\n", cmd);
 438                 }
 439
 440                 if (nfiles == 0) {
 441                         usage(stderr, "--nfiles must be specifed with --%s.\n",
 442                               cmd);
 443                 }
 444
 445                 if (seed == 0) {
 446                         int fd = open("/dev/urandom", O_RDONLY);
 447
 448                         if (fd >= 0) {
 449                                 if (read(fd, &seed, sizeof(seed)) <
 450                                     sizeof(seed))
 451                                         seed = time(0);
 452                                 close(fd);
 453                         } else {
 454                                 seed = time(0);
 455                         }
 456                 }
 457
 458                 srand(seed);
 459
 460                 dmesg("%s: rank %d seed %d (%s).\n", prog, myrank, seed,
 461                       (order == RANDOM) ? "random_order" : "readdir_order");
 462         } else {
 463                 usage(stderr, "one --create, --mknod, --open, --stat,"
 464 #ifdef HAVE_MDC_LOOKUP
 465                       " --lookup,"
 466 #endif
 467                       " or --unlink must be specifed.");
 468         }
 469
 470         /* support for multiple threads in a dir, set begin/end appropriately.*/
 471         dirnum = myrank % ndirs;
 472         dirthreads = nthreads / ndirs;
 473         if (nthreads > (ndirs * dirthreads + dirnum))
 474                 ++dirthreads;
 475
 476         offset = myrank / ndirs;
 477
 478         tmpend = begin + nfiles - 1;
 479         if (tmpend <= 0)
 480                 tmpend = INT_MAX;
 481
 482         end = begin + (nfiles / dirthreads) * dirthreads + offset;
 483         if ((end > tmpend) || (end <= 0))
 484                 end -= dirthreads;
 485
 486         begin += offset;
 487         if (begin < 0)
 488                 begin = INT_MAX;
 489
 490         beginsave = begin;
 491
 492         dmesg("%d: iters %d nfiles %d time %d begin %d end %d dirthreads %d."
 493               "\n", myrank, iters, nfiles, seconds, begin, end, dirthreads);
 494
 495         if (dirfmt == NULL) {
 496                 strcpy(dir, ".");
 497         } else {
 498                 sprintf(dir, dirfmt, dirnum);
 499
 500                 sprintf(mkdir_cmd, "/bin/mkdir -p %s", dir);
 501                 #ifdef _LIGHTWEIGHT_KERNEL
 502                         printf("NOTICE: not running system(%s)\n", mkdir_cmd);
 503                 #else
 504                         rc = system(mkdir_cmd);
 505                         if (rc) {
 506                                 fatal(myrank, "'%s' failed.\n", mkdir_cmd);
 507                         }
 508                 #endif
 509
 510                 rc = chdir(dir);
 511                 if (rc) {
 512                         fatal(myrank, "unable to chdir to '%s'.\n", dir);
 513                 }
 514         }
 515 }
 516
 517 static inline char *next_file()
 518 {
 519         if (order == RANDOM) {
 520                 sprintf(filename, filefmt, random() % nfiles);
 521                 return(filename);
 522         }
 523
 524         /* readdir order */
 525
 526         dir_entry = readdir(directory);
 527         if (dir_entry == NULL) {
 528                 rewinddir(directory);
 529                 while ((dir_entry = readdir(directory)) != NULL) {
 530                         if (dir_entry->d_name[0] != '.')
 531                                 return(dir_entry->d_name);
 532                 }
 533
 534                 fatal(myrank, "unable to read directory %s (%s).\n",
 535                       dir, strerror(errno));
 536         }
 537
 538         return(dir_entry->d_name);
 539 }
 540
 541 int
 542 main(int argc, char *argv[])
 543 {
 544         int    i, j, fd, rc, nops, lastOps;
 545         int ag_ops = 0;
 546         double ag_interval = 0;
 547         double ag_rate = 0;
 548         double rate, avg_rate, effective_rate;
 549         double startTime, curTime, lastTime, interval;
 550         time_t timestamp;
 551         char * file;
 552
 553         rc = MPI_Init(&argc, &argv);
 554         if (rc != MPI_SUCCESS)
 555                 fatal(myrank, "MPI_Init failed: %d\n", rc);
 556
 557         rc = MPI_Comm_size(MPI_COMM_WORLD, &nthreads);
 558         if (rc != MPI_SUCCESS)
 559                 fatal(myrank, "MPI_Comm_size failed: %d\n", rc);
 560
 561         rc = MPI_Comm_rank(MPI_COMM_WORLD, &myrank);
 562         if (rc != MPI_SUCCESS)
 563                 fatal(myrank, "MPI_Comm_rank failed: %d\n", rc);
 564
 565         process_args(argc, argv);
 566
 567         timestamp = time(0);
 568         if ((myrank == 0) || debug) {
 569                 printf("%d: %s starting at %s",
 570                        myrank, hostname, ctime(&timestamp));
 571         }
 572
 573         /* if we're not measuring creation rates then precreate
 574          * the files we're operating on. */
 575         if ((mode != CREATE) && (mode != MKNOD) && !ignore &&
 576             (mode != UNLINK || recreate)) {
 577                 /* create the files in reverse order. When we encounter
 578                  * a file that already exists, assume the remainder of
 579                  * the files exist to save time. The timed performance
 580                  * test scripts make use of this behavior. */
 581                 for (i = end, j = 0; i >= begin; i -= dirthreads) {
 582                         sprintf(filename, filefmt, i);
 583                         fd = open(filename, openflags, 0644);
 584                         if (fd < 0) {
 585                                 if (errno == EEXIST)
 586                                         break;
 587                                 rc = errno;
 588                                 fatal(myrank, "precreate open(%s) error: %s\n",
 589                                       filename, strerror(rc));
 590                         }
 591                         j++;
 592                         close(fd);
 593                 }
 594                 dmesg("%d: %s pre-created %d files.\n",myrank,hostname,j);
 595
 596                 rc = MPI_Barrier(MPI_COMM_WORLD);
 597                 if (rc != MPI_SUCCESS)
 598                         fatal(myrank, "prep MPI_Barrier failed: %d\n", rc);
 599         }
 600
 601         if (order == READDIR) {
 602                 directory = opendir(dir);
 603                 if (directory == NULL) {
 604                         rc = errno;
 605                         fatal(myrank, "opendir(%s) error: %s\n",
 606                               dir, strerror(rc));
 607                 }
 608
 609                 timestamp = time(0);
 610                 j = random() % nfiles;
 611                 dmesg("%d: %s initializing dir offset %u: %s",
 612                       myrank, hostname, j, ctime(&timestamp));
 613
 614                 for (i = 0; i <= j; i++) {
 615                         if ((dir_entry = readdir(directory)) == NULL) {
 616                                 fatal(myrank, "could not read entry number %d "
 617                                       "in directory %s.\n", i, dir);
 618                         }
 619                 }
 620
 621                 timestamp = time(0);
 622                 dmesg("%d: index %d, filename %s, offset %ld: "
 623                       "%s initialization complete: %s",
 624                       myrank, i, dir_entry->d_name, telldir(directory),
 625                       hostname, ctime(&timestamp));
 626         }
 627
 628         if (seconds) {
 629                 act.sa_handler = sigalrm_handler;
 630                 (void)sigemptyset(&act.sa_mask);
 631                 act.sa_flags = 0;
 632                 sigaction(SIGALRM, &act, NULL);
 633                 alarm(seconds);
 634         }
 635
 636         rc = MPI_Barrier(MPI_COMM_WORLD);
 637         if (rc != MPI_SUCCESS)
 638                 fatal(myrank, "prep MPI_Barrier failed: %d\n", rc);
 639
 640         startTime = lastTime = MPI_Wtime();
 641         nops = lastOps = 0;
 642
 643         switch (mode) {
 644         case CREATE:
 645                 for (; begin <= end && !alarm_caught; begin += dirthreads) {
 646                         sprintf(filename, filefmt, begin);
 647                         if ((fd = open(filename, openflags, 0644)) < 0) {
 648                                 if (((rc = errno) == EINTR) && alarm_caught)
 649                                         break;
 650                                 fatal(myrank, "open(%s) error: %s\n",
 651                                       filename, strerror(rc));
 652                         }
 653
 654                         close(fd);
 655                         nops++;
 656                         DISPLAY_PROGRESS();
 657                 }
 658
 659                 dmesg("%d: created %d files, last file '%s'.\n",
 660                       myrank, nops, filename);
 661                 break;
 662 #ifdef HAVE_MDC_LOOKUP
 663         case LOOKUP:
 664                 fd = open(dir, O_RDONLY);
 665                 if (fd < 0) {
 666                         fatal(myrank, "open(dir == '%s') error: %s\n",
 667                               dir, strerror(errno));
 668                 }
 669
 670                 for (; nops < iters && !alarm_caught;) {
 671                         char *filename = next_file();
 672                         rc = llapi_file_lookup(fd, filename);
 673                         if (rc < 0) {
 674                                 if (((rc = errno) == EINTR) && alarm_caught)
 675                                         break;
 676                                 fatal(myrank, "llapi_file_lookup(%s) "
 677                                       "error: %s\n", filename, strerror(rc));
 678                         }
 679
 680                         nops++;
 681                         DISPLAY_PROGRESS();
 682                 }
 683                 break;
 684 #endif
 685         case MKNOD:
 686                 for (; begin <= end && !alarm_caught; begin += dirthreads) {
 687                         sprintf(filename, filefmt, begin);
 688                         rc = mknod(filename, S_IFREG| 0644, 0);
 689                         if (rc) {
 690                                 if (((rc = errno) == EINTR) && alarm_caught)
 691                                         break;
 692                                 fatal(myrank, "mknod(%s) error: %s\n",
 693                                       filename, strerror(rc));
 694                         }
 695
 696                         nops++;
 697                         DISPLAY_PROGRESS();
 698                 }
 699                 break;
 700         case OPEN:
 701                 for (; nops < iters && !alarm_caught;) {
 702                         file = next_file();
 703                         if ((fd = open(file, openflags, 0644)) < 0) {
 704                                 if (((rc = errno) == EINTR) && alarm_caught)
 705                                         break;
 706                                 fatal(myrank, "open(%s) error: %s\n",
 707                                       file, strerror(rc));
 708                         }
 709
 710                         close(fd);
 711
 712                         nops++;
 713                         DISPLAY_PROGRESS();
 714                 }
 715                 break;
 716         case STAT:
 717                 for (; begin <= end && !alarm_caught; begin += dirthreads) {
 718                         sprintf(filename, filefmt, begin);
 719                         rc = stat(filename, &statbuf);
 720                         if (rc) {
 721                                 if (((rc = errno) == EINTR) && alarm_caught)
 722                                         break;
 723                                 if (((rc = errno) == ENOENT) && ignore)
 724                                         continue;
 725                                 fatal(myrank, "stat(%s) error: %s\n",
 726                                       filename, strerror(rc));
 727                         }
 728
 729                         nops++;
 730                         DISPLAY_PROGRESS();
 731                 }
 732                 break;
 733         case UNLINK:
 734                 for (; begin <= end && !alarm_caught; begin += dirthreads) {
 735                         sprintf(filename, filefmt, begin);
 736                         rc = unlink(filename);
 737                         if (rc) {
 738                                 if (((rc = errno) == EINTR) && alarm_caught)
 739                                         break;
 740                                 if ((rc = errno) == ENOENT) {
 741                                         if (ignore)
 742                                                 continue;
 743                                         /* no more files to unlink */
 744                                         break;
 745                                 }
 746                                 fatal(myrank, "unlink(%s) error: %s\n",
 747                                       filename, strerror(rc));
 748                         }
 749
 750                         nops++;
 751                         DISPLAY_PROGRESS();
 752                 }
 753                 break;
 754         }
 755
 756         rc = MPI_Barrier(MPI_COMM_WORLD);
 757         if (rc != MPI_SUCCESS)
 758                fatal(myrank, "prep MPI_Barrier failed: %d\n", rc);
 759         curTime = MPI_Wtime();
 760         interval = curTime - startTime;
 761         rate = (double) (nops) / interval;
 762
 763         rc = MPI_Reduce(&nops, &ag_ops, 1, MPI_INT, MPI_SUM, 0,
 764                         MPI_COMM_WORLD);
 765         if (rc != MPI_SUCCESS) {
 766                 fatal(myrank, "Failure in MPI_Reduce of total ops.\n");
 767         }
 768
 769         rc = MPI_Reduce(&interval, &ag_interval, 1, MPI_DOUBLE, MPI_SUM, 0,
 770                         MPI_COMM_WORLD);
 771         if (rc != MPI_SUCCESS) {
 772                 fatal(myrank, "Failure in MPI_Reduce of total interval.\n");
 773         }
 774
 775         rc = MPI_Reduce(&rate, &ag_rate, 1, MPI_DOUBLE, MPI_SUM, 0,
 776                         MPI_COMM_WORLD);
 777         if (rc != MPI_SUCCESS) {
 778                 fatal(myrank, "Failure in MPI_Reduce of aggregated rate.\n");
 779         }
 780
 781         if (myrank == 0) {
 782
 783                 curTime = MPI_Wtime();
 784                 interval = curTime - startTime;
 785                 effective_rate = (double) ag_ops / interval;
 786                 avg_rate = (double) ag_ops / ag_interval;
 787
 788                 printf("Rate: %.2f eff %.2f aggr %.2f avg client %ss/sec "
 789                        "(total: %d threads %d %ss %d dirs %d threads/dir %.2f secs)\n",
 790                        effective_rate, ag_rate, avg_rate, cmd, nthreads, ag_ops,
 791                        cmd, ndirs, dirthreads, interval);
 792                 if (mode == UNLINK && !recreate && !ignore && ag_ops != nfiles)
 793                         printf("Warning: only unlinked %d files instead of %d"
 794                                "\n", ag_ops, nfiles);
 795         }
 796
 797         if (recreate) {
 798                 for (begin = beginsave; begin <= end; begin += dirthreads) {
 799                         sprintf(filename, filefmt, begin);
 800                         if ((fd = open(filename, openflags, 0644)) < 0) {
 801                                 rc = errno;
 802                                 if (rc == EEXIST)
 803                                         break;
 804                                 fatal(myrank, "recreate open(%s) error: %s\n",
 805                                       filename, strerror(rc));
 806                         }
 807
 808                         close(fd);
 809                 }
 810         }
 811
 812         timestamp = time(0);
 813         if ((myrank == 0) || debug) {
 814                 printf("%d: %s finished at %s",
 815                        myrank, hostname, ctime(&timestamp));
 816         }
 817
 818         MPI_Finalize();
 819         return(0);
 820 }