lustre/tests/mpi/mdsrate.c

   1 /*
   2  * 2003, Copyright, Hewlett-Packard Development Compnay, LP.
   3  *
   4  * Developed under the sponsorship of the U.S. Government
   5  *     under Subcontract No. B514193
   6  */
   7
   8 /*
   9  * Copyright (c) 2008, 2010, Oracle and/or its affiliates. All rights reserved.
  10  * Use is subject to license terms.
  11  *
  12  * Copyright (c) 2012, 2015, Intel Corporation.
  13  */
  14
  15 #include <stdio.h>
  16 #include <getopt.h>
  17 #include <libgen.h>
  18 #include <sys/types.h>
  19 #include <sys/stat.h>
  20 #include <time.h>
  21 #include <limits.h>
  22 #include <errno.h>
  23 #include <string.h>
  24 #include <fcntl.h>
  25 #include <unistd.h>
  26 #include <stdlib.h>
  27 #include <stdarg.h>
  28 #include <signal.h>
  29 #include <sys/ioctl.h>
  30 #include <dirent.h>
  31 #include <sys/xattr.h>
  32
  33 #include "mpi.h"
  34
  35 /* lustre */
  36 #include <lustre/lustreapi.h>        /* for O_LOV_DELAY_CREATE */
  37
  38 #define CHECK_COUNT 10000
  39 #define DISPLAY_COUNT (CHECK_COUNT * 10)
  40 #define DISPLAY_TIME 100
  41
  42 enum {
  43         CREATE          = 'c',
  44         LOOKUP          = 'l',
  45         MKNOD           = 'm',
  46         OPEN            = 'o',
  47         STAT            = 's',
  48         UNLINK          = 'u',
  49         BEGIN           = 'b',
  50         ITERS           = 'i',
  51         TIME            = 't',
  52         DIRFMT          = 'd',
  53         NDIRS           = 'D',
  54         FILEFMT         = 'f',
  55         NFILES          = 'F',
  56         NOEXCL          = 'X',
  57         STRIPES         = 'S',
  58         SEED            = 'r',
  59         SEEDFILE        = 'R',
  60         RANDOM          = 'A',
  61         READDIR         = 'B',
  62         RECREATE        = 'C',
  63         SETXATTR        = 'x',
  64         SMALLWRITE      = 'w',
  65         IGNORE          = 'E',
  66         VERBOSE         = 'V',
  67         DEBUG           = 'v',
  68         HELP            = 'h',
  69         MNT             = 'M',
  70         MNTCOUNT        = 'N',
  71         MDTCOUNT        = 'T',
  72 };
  73
  74 struct option longOpts[] = {
  75         { .name = "create", .has_arg = no_argument, .val = CREATE },
  76         { .name = "lookup", .has_arg = no_argument, .val = LOOKUP },
  77         { .name = "mknod", .has_arg = no_argument, .val = MKNOD },
  78         { .name = "open", .has_arg = no_argument, .val = OPEN },
  79         { .name = "stat", .has_arg = no_argument, .val = STAT },
  80         { .name = "unlink", .has_arg = no_argument, .val = UNLINK },
  81         { .name = "begin", .has_arg = required_argument, .val = BEGIN },
  82         { .name = "iters", .has_arg = required_argument, .val = ITERS },
  83         /* time is in seconds */
  84         { .name = "time", .has_arg = required_argument, .val = TIME },
  85         { .name = "dirfmt", .has_arg = required_argument, .val = DIRFMT },
  86         { .name = "ndirs", .has_arg = required_argument, .val = NDIRS },
  87         { .name = "filefmt", .has_arg = required_argument, .val = FILEFMT },
  88         { .name = "nfiles", .has_arg = required_argument, .val = NFILES },
  89         { .name = "noexcl", .has_arg = no_argument, .val = NOEXCL },
  90         { .name = "stripes", .has_arg = required_argument, .val = STRIPES },
  91         { .name = "seed", .has_arg = required_argument, .val = SEED },
  92         { .name = "seedfile", .has_arg = required_argument, .val = SEEDFILE },
  93         { .name = "random_order", .has_arg = no_argument, .val = RANDOM },
  94         { .name = "readdir_order", .has_arg = no_argument, .val = READDIR },
  95         { .name = "recreate", .has_arg = no_argument, .val = RECREATE },
  96         { .name = "setxattr", .has_arg = no_argument, .val = SETXATTR },
  97         { .name = "smallwrite", .has_arg = no_argument, .val = SMALLWRITE },
  98         { .name = "ignore", .has_arg = no_argument, .val = IGNORE },
  99         { .name = "verbose", .has_arg = no_argument, .val = VERBOSE },
 100         { .name = "debug", .has_arg = no_argument, .val = DEBUG },
 101         { .name = "help", .has_arg = no_argument, .val = HELP },
 102         { .name = "mdtcount", .has_arg = required_argument, .val = MDTCOUNT },
 103         { .name = "mntcount", .has_arg = required_argument, .val = MNTCOUNT },
 104         { .name = "mntfmt", .has_arg = required_argument, .val = MNT },
 105         { .name = NULL }
 106 };
 107
 108 int foo1, foo2;
 109
 110 char   shortOpts[128];
 111 int    myrank = -1;
 112 int    nthreads = -1;
 113 char *prog;
 114 char   hostname[512] = "unknown";
 115 char   mode;
 116 char *cmd;
 117 int    openflags = O_RDWR | O_CREAT | O_EXCL;
 118 int    ndirs = 1;
 119 char *dirfmt;
 120 char   dir[PATH_MAX];
 121 char   mkdir_cmd[PATH_MAX + 32];
 122 int    dirthreads;
 123 int    dirnum;
 124 DIR *directory;
 125 struct dirent *dir_entry;
 126 int    nfiles;
 127 char   filefmt[PATH_MAX];
 128 char   filename[PATH_MAX];
 129 char   path[PATH_MAX];
 130 int    stripes = -1;
 131 int    begin;
 132 int    beginsave;
 133 int    end;
 134 int    iters;
 135 int    seconds;
 136 int    alarm_caught;
 137 struct sigaction act;
 138 int    order = RANDOM;
 139 int    seed;
 140 int    recreate;
 141 int    ignore;
 142 int    verbose;
 143 int    debug;
 144 struct stat statbuf;
 145 bool   with_xattr;
 146 char   xattrname[] = "user.mdsrate";
 147 char   xattrbuf[4096];
 148 /* max xattr name + value length is block size, use 4000 here to avoid ENOSPC */
 149 int    xattrlen = 4000;
 150 bool   smallwrite;
 151 int    mnt_count = -1;
 152 int    mdt_count = 1;
 153 char  *mntfmt;
 154
 155 #define dmesg if (debug) printf
 156
 157 #define DISPLAY_PROGRESS() {                                                \
 158         if (verbose && (nops % CHECK_COUNT == 0)) {                         \
 159                 curTime = MPI_Wtime();                                      \
 160                 interval = curTime - lastTime;                              \
 161                 if (interval > DISPLAY_TIME || nops % DISPLAY_COUNT == 0) { \
 162                         rate = (double)(nops - lastOps) / interval;         \
 163                         printf("Rank %d: %.2f %ss/sec %.2f secs "           \
 164                         "(total: %d %ss %.2f secs)\n",                      \
 165                         myrank, rate, cmd, interval,                        \
 166                         nops, cmd, curTime - startTime);                    \
 167                         lastOps = nops;                                     \
 168                         lastTime = curTime;                                 \
 169                 }                                                           \
 170         }                                                                   \
 171 }
 172
 173 char *usage_msg = "usage: %s\n"
 174                   "    { --create [ --noexcl | --setxattr | --smallwrite ] |\n"
 175                   "      --lookup | --mknod [ --setxattr ] | --open |\n"
 176                   "      --stat | --unlink [ --recreate ] [ --ignore ] |\n"
 177                   "      --setxattr }\n"
 178                   "    [ --help ] [ --verbose ] [ --debug ]\n"
 179                   "    { [ --begin <num> ] --nfiles <num> }\n"
 180                   "    [ --iters <num> ] [ --time <secs> ]\n"
 181                   "    [ --dirfmt <str> ] [ --ndirs  <num> ]\n"
 182                   "    [ --filefmt <str> ] [ --stripes <num> ]\n"
 183                   "    [ --random_order [--seed <num> | --seedfile <file>] ]\n"
 184                   "    [ --readdir_order ] [ --mntfmt <str> ]\n"
 185                   "    [ --mntcount <num> ] [ --mdtcount <num> ]\n"
 186                   "    [ --setxattr ] }\n";
 187
 188 static void
 189 usage(FILE *stream, char *fmt, ...)
 190 {
 191         if (myrank == 0) {
 192                 if (fmt) {
 193                         va_list       ap;
 194
 195                         fprintf(stream, "%s: ", prog);
 196                         va_start(ap, fmt);
 197                         vfprintf(stderr, fmt, ap);
 198                         va_end(ap);
 199                 }
 200                 fprintf(stream, usage_msg, prog);
 201         }
 202
 203         MPI_Finalize();
 204         exit(stream == stderr);
 205 }
 206
 207 /* Print process myrank and message, and exit (i.e. a fatal error) */
 208 static int
 209 fatal(int rank, const char *fmt, ...)
 210 {
 211         if (rank == myrank) {
 212                 va_list       ap;
 213
 214                 fprintf(stderr, "rank %d: ", rank);
 215                 va_start(ap, fmt);
 216                 vfprintf(stderr, fmt, ap);
 217                 va_end(ap);
 218         }
 219
 220         MPI_Abort(MPI_COMM_WORLD, 1);
 221         exit(1);
 222 }
 223
 224 static void
 225 sigalrm_handler(int signum)
 226 {
 227         alarm_caught++;
 228 }
 229
 230 static void
 231 process_args(int argc, char *argv[])
 232 {
 233         char   *cp, *endptr;
 234         int    i, index, offset, tmpend, rc;
 235         char   tmp[16];
 236         FILE *seed_file;
 237         struct option *opt;
 238
 239         setbuf(stdout, 0);
 240         setbuf(stderr, 0);
 241         prog = basename(argv[0]);
 242         strcpy(filefmt, "f%d");
 243         gethostname(hostname, sizeof(hostname));
 244
 245         /* auto create shortOpts rather than maintaining a static string. */
 246         for (opt = longOpts, cp = shortOpts; opt->name != NULL; opt++, cp++) {
 247                 *cp = opt->val;
 248                 if (opt->has_arg)
 249                         *++cp = ':';
 250         }
 251
 252         while ((rc = getopt_long(argc, argv, shortOpts,
 253                                  longOpts, &index)) != -1) {
 254                 switch (rc) {
 255                 case OPEN:
 256                         openflags &= ~(O_CREAT | O_EXCL);
 257                 case CREATE:
 258                 case LOOKUP:
 259                 case MKNOD:
 260                 case STAT:
 261                 case UNLINK:
 262                         if (cmd) {
 263                                 fatal(0,
 264                                       "Invalid - more than one operation specified: --%s\n",
 265                                       longOpts[index].name);
 266                         }
 267                         mode = rc;
 268                         cmd = (char *)longOpts[index].name;
 269                         break;
 270                 case NOEXCL:
 271                         if (mode != CREATE && mode != MKNOD) {
 272                                 usage(stderr,
 273                                       "--noexcl only applies to --create or --mknod.\n");
 274                         }
 275                         openflags &= ~O_EXCL;
 276                         break;
 277                 case RECREATE:
 278                         if (mode != UNLINK) {
 279                                 usage(stderr,
 280                                       "--recreate only makes sense with --unlink.\n");
 281                         }
 282                         recreate++;
 283                         break;
 284                 case SETXATTR:
 285                         if (!cmd) {
 286                                 mode = SETXATTR;
 287                                 cmd = (char *)longOpts[index].name;
 288                         } else if (mode == CREATE || mode == MKNOD) {
 289                                 with_xattr = true;
 290                         } else {
 291                                 usage(stderr,
 292                                       "--setxattr only makes sense with --create, --mknod or alone.\n");
 293                         }
 294                         break;
 295                 case SMALLWRITE:
 296                         if (mode != CREATE)
 297                                 usage(stderr,
 298                                       "--smallwrite only applies to --create.\n");
 299                         smallwrite = true;
 300                         break;
 301                 case BEGIN:
 302                         begin = strtol(optarg, &endptr, 0);
 303                         if ((*endptr != 0) || (begin < 0))
 304                                 fatal(0, "Invalid --start value.\n");
 305                         break;
 306                 case ITERS:
 307                         iters = strtol(optarg, &endptr, 0);
 308                         if ((*endptr != 0) || (iters <= 0))
 309                                 fatal(0, "Invalid --iters value.\n");
 310                         if (mode != LOOKUP && mode != OPEN)
 311                                 usage(stderr,
 312                                       "--iters only makes sense with --lookup or --open.\n");
 313                         break;
 314                 case TIME:
 315                         seconds = strtol(optarg, &endptr, 0);
 316                         if ((*endptr != 0) || (seconds <= 0))
 317                                 fatal(0, "Invalid --time value.\n");
 318                         break;
 319                 case DIRFMT:
 320                         if (strlen(optarg) > (PATH_MAX - 16))
 321                                 fatal(0, "--dirfmt too long\n");
 322                         dirfmt = optarg;
 323                         break;
 324                 case NDIRS:
 325                         ndirs = strtol(optarg, &endptr, 0);
 326                         if ((*endptr != 0) || (ndirs <= 0))
 327                                 fatal(0, "Invalid --ndirs value.\n");
 328                         if ((ndirs > nthreads) &&
 329                             ((mode == CREATE) || (mode == MKNOD))) {
 330                                 fatal(0,
 331                                       "--ndirs=%d must be less than or equal to the number of threads (%d).\n",
 332                                       ndirs, nthreads);
 333                         }
 334                         break;
 335                 case FILEFMT:
 336                         if (strlen(optarg) > 4080)
 337                                 fatal(0, "--filefmt too long\n");
 338
 339                         /* Use %%d where you want the file # in the name. */
 340                         sprintf(filefmt, optarg, myrank);
 341                         break;
 342                 case NFILES:
 343                         nfiles = strtol(optarg, &endptr, 0);
 344                         if ((*endptr != 0) || (nfiles <= 0))
 345                                 fatal(0, "Invalid --nfiles value.\n");
 346                         break;
 347                 case STRIPES:
 348                         stripes = strtol(optarg, &endptr, 0);
 349                         if ((*endptr != 0) || (stripes < 0))
 350                                 fatal(0, "Invalid --stripes value.\n");
 351
 352                         if (stripes == 0) {
 353                                 openflags |= O_LOV_DELAY_CREATE;
 354                         } else {
 355                                 fatal(0,
 356                                       "non-zero --stripes value not yet supported.\n");
 357                         }
 358
 359                         break;
 360                 case SEED:
 361                         seed = strtoul(optarg, &endptr, 0);
 362                         if (*endptr)
 363                                 fatal(0, "bad --seed option %s\n", optarg);
 364                         break;
 365                 case SEEDFILE:
 366                         seed_file = fopen(optarg, "r");
 367                         if (!seed_file) {
 368                                 fatal(myrank, "fopen(%s) error: %s\n",
 369                                       optarg, strerror(errno));
 370                         }
 371
 372                         for (i = -1; fgets(tmp, 16, seed_file) != NULL;) {
 373                                 if (++i == myrank)
 374                                         break;
 375                         }
 376
 377                         if (i == myrank) {
 378                                 rc = sscanf(tmp, "%d", &seed);
 379                                 if ((rc != 1) || (seed < 0)) {
 380                                         fatal(myrank,
 381                                               "Invalid seed value '%s' at line %d in %s.\n",
 382                                               tmp, i, optarg);
 383                                 }
 384                         } else {
 385                                 fatal(myrank,
 386                                       "File '%s' too short. Does not contain a seed for thread %d.\n",
 387                                       optarg, myrank);
 388                         }
 389
 390                         fclose(seed_file);
 391                         break;
 392                 case RANDOM:
 393                 case READDIR:
 394                         if (mode != LOOKUP && mode != OPEN)  {
 395                                 fatal(0,
 396                                       "--%s can only be specified with --lookup, or --open.\n",
 397                                       (char *)longOpts[index].name);
 398                         }
 399                         order = rc;
 400                         break;
 401                 case IGNORE:
 402                         ++ignore;
 403                         break;
 404                 case DEBUG:
 405                         ++debug;
 406                 case VERBOSE:
 407                         ++verbose;
 408                         break;
 409                 case HELP:
 410                         usage(stdout, NULL);
 411                         break;
 412                 case MNT:
 413                         if (strlen(optarg) > (PATH_MAX - 16))
 414                                 fatal(0, "--mnt too long\n");
 415                         mntfmt = optarg;
 416                         break;
 417                 case MNTCOUNT:
 418                         mnt_count = strtol(optarg, &endptr, 0);
 419                         if ((*endptr != 0) || (mnt_count <= 0)) {
 420                                 fatal(0, "Invalid --mnt_count value %s.\n",
 421                                       optarg);
 422                         }
 423                         break;
 424                 case MDTCOUNT:
 425                         mdt_count = strtol(optarg, &endptr, 0);
 426                         if ((*endptr != 0) || (mdt_count <= 0)) {
 427                                 fatal(0, "Invalid --mdt_count value %s.\n",
 428                                       optarg);
 429                         }
 430                         break;
 431                 default:
 432                         usage(stderr, "unrecognized option: '%c'.\n", optopt);
 433                 }
 434         }
 435
 436         if (optind < argc)
 437                 usage(stderr, "too many arguments %d >= %d.\n", optind, argc);
 438
 439         if ((mnt_count != -1 && !mntfmt) ||
 440             (mnt_count == -1 && mntfmt))
 441                 usage(stderr,
 442                       "mnt_count and mntfmt must be specified at the same time\n");
 443
 444         if (mode == CREATE || mode == MKNOD || mode == UNLINK ||
 445             mode == STAT || mode == SETXATTR) {
 446                 if (seconds != 0) {
 447                         if (nfiles == 0)
 448                                 nfiles = INT_MAX;
 449                 } else if (nfiles == 0) {
 450                         usage(stderr,
 451                               "--nfiles or --time must be specified with %s.\n",
 452                               cmd);
 453                 }
 454         } else if (mode == LOOKUP || mode == OPEN) {
 455                 if (seconds != 0) {
 456                         if (iters == 0)
 457                                 iters = INT_MAX;
 458                 } else if (iters == 0) {
 459                         usage(stderr,
 460                               "--iters or --time must be specifed with %s.\n",
 461                               cmd);
 462                 }
 463
 464                 if (nfiles == 0) {
 465                         usage(stderr, "--nfiles must be specifed with --%s.\n",
 466                               cmd);
 467                 }
 468
 469                 if (seed == 0) {
 470                         int fd = open("/dev/urandom", O_RDONLY);
 471
 472                         if (fd >= 0) {
 473                                 if (read(fd, &seed, sizeof(seed)) <
 474                                     sizeof(seed))
 475                                         seed = time(0);
 476                                 close(fd);
 477                         } else {
 478                                 seed = time(0);
 479                         }
 480                 }
 481
 482                 srand(seed);
 483
 484                 dmesg("%s: rank %d seed %d (%s).\n", prog, myrank, seed,
 485                       (order == RANDOM) ? "random_order" : "readdir_order");
 486         } else {
 487                 usage(stderr,
 488                       "one --create, --mknod, --open, --stat, --lookup, --unlink or --setxattr must be specifed.");
 489         }
 490
 491         /* support for multiple threads in a dir, set begin/end appropriately.*/
 492         dirnum = myrank % ndirs;
 493         dirthreads = nthreads / ndirs;
 494         if (nthreads > (ndirs * dirthreads + dirnum))
 495                 ++dirthreads;
 496
 497         offset = myrank / ndirs;
 498
 499         tmpend = begin + nfiles - 1;
 500         if (tmpend <= 0)
 501                 tmpend = INT_MAX;
 502
 503         end = begin + (nfiles / dirthreads) * dirthreads + offset;
 504         if ((end > tmpend) || (end <= 0))
 505                 end -= dirthreads;
 506
 507         /*
 508          * make sure mnt_count <= nthreads, otherwise it might div 0 in
 509          * the following test
 510          */
 511         if (mnt_count > nthreads)
 512                 mnt_count = nthreads;
 513
 514         begin += offset;
 515         if (begin < 0)
 516                 begin = INT_MAX;
 517
 518         beginsave = begin;
 519
 520         dmesg("%d: iters %d nfiles %d time %d begin %d end %d dirthreads %d.\n",
 521               myrank, iters, nfiles, seconds, begin, end, dirthreads);
 522
 523         if (!dirfmt) {
 524                 strcpy(dir, ".");
 525         } else {
 526                 int dir_len = 0;
 527
 528                 if (mntfmt) {
 529                         sprintf(dir, mntfmt, (myrank / (nthreads / mnt_count)));
 530                         strcat(dir, "/");
 531                         dir_len = strlen(dir);
 532                 }
 533                 sprintf(dir + dir_len, dirfmt, dirnum);
 534
 535                 if (mdt_count > 1) {
 536                         struct stat sb;
 537
 538                         if (stat(dir, &sb) == 0) {
 539                                 if (!S_ISDIR(sb.st_mode))
 540                                         fatal(myrank, "'%s' is not dir\n", dir);
 541                         } else if (errno == ENOENT) {
 542                                 sprintf(mkdir_cmd, "lfs mkdir -i %d %s",
 543                                         myrank % mdt_count, dir);
 544                         } else {
 545                                 fatal(myrank, "'%s' stat failed\n", dir);
 546                         }
 547                 } else {
 548                         sprintf(mkdir_cmd, "mkdir -p %s", dir);
 549                 }
 550
 551                 dmesg("%d: %s\n", myrank, mkdir_cmd);
 552 #ifdef _LIGHTWEIGHT_KERNEL
 553                 printf("NOTICE: not running system(%s)\n", mkdir_cmd);
 554 #else
 555                 rc = system(mkdir_cmd);
 556                 if (rc)
 557                         fatal(myrank, "'%s' failed.\n", mkdir_cmd);
 558 #endif
 559
 560                 rc = chdir(dir);
 561                 if (rc)
 562                         fatal(myrank, "unable to chdir to '%s'.\n", dir);
 563         }
 564 }
 565
 566 static inline char *next_file()
 567 {
 568         if (order == RANDOM) {
 569                 sprintf(filename, filefmt, random() % nfiles);
 570                 return filename;
 571         }
 572
 573         /* readdir order */
 574
 575         dir_entry = readdir(directory);
 576         if (!dir_entry) {
 577                 rewinddir(directory);
 578                 while ((dir_entry = readdir(directory)) != NULL) {
 579                         if (dir_entry->d_name[0] != '.')
 580                                 return dir_entry->d_name;
 581                 }
 582
 583                 fatal(myrank, "unable to read directory %s (%s).\n",
 584                       dir, strerror(errno));
 585         }
 586
 587         return dir_entry->d_name;
 588 }
 589
 590 int
 591 main(int argc, char *argv[])
 592 {
 593         int    i, j, fd, rc, nops, lastOps;
 594         int ag_ops = 0;
 595         double ag_interval = 0;
 596         double ag_rate = 0;
 597         double rate, avg_rate, effective_rate;
 598         double startTime, curTime, lastTime, interval;
 599         time_t timestamp;
 600         char *file;
 601
 602         rc = MPI_Init(&argc, &argv);
 603         if (rc != MPI_SUCCESS)
 604                 fatal(myrank, "MPI_Init failed: %d\n", rc);
 605
 606         rc = MPI_Comm_size(MPI_COMM_WORLD, &nthreads);
 607         if (rc != MPI_SUCCESS)
 608                 fatal(myrank, "MPI_Comm_size failed: %d\n", rc);
 609
 610         rc = MPI_Comm_rank(MPI_COMM_WORLD, &myrank);
 611         if (rc != MPI_SUCCESS)
 612                 fatal(myrank, "MPI_Comm_rank failed: %d\n", rc);
 613
 614         process_args(argc, argv);
 615
 616         timestamp = time(0);
 617         if ((myrank == 0) || debug) {
 618                 printf("%d: %s starting at %s",
 619                        myrank, hostname, ctime(&timestamp));
 620         }
 621
 622         /*
 623          * if we're not measuring creation rates then precreate
 624          * the files we're operating on.
 625          */
 626         if ((mode != CREATE) && (mode != MKNOD) && !ignore &&
 627             (mode != UNLINK || recreate)) {
 628                 /*
 629                  * create the files in reverse order. When we encounter
 630                  * a file that already exists, assume the remainder of
 631                  * the files exist to save time. The timed performance
 632                  * test scripts make use of this behavior.
 633                  */
 634                 for (i = end, j = 0; i >= begin; i -= dirthreads) {
 635                         sprintf(filename, filefmt, i);
 636                         fd = open(filename, openflags, 0644);
 637                         if (fd < 0) {
 638                                 if (errno == EEXIST)
 639                                         break;
 640                                 rc = errno;
 641                                 fatal(myrank, "precreate open(%s) error: %s\n",
 642                                       filename, strerror(rc));
 643                         }
 644                         j++;
 645                         close(fd);
 646                 }
 647                 dmesg("%d: %s pre-created %d files.\n", myrank, hostname, j);
 648
 649                 rc = MPI_Barrier(MPI_COMM_WORLD);
 650                 if (rc != MPI_SUCCESS)
 651                         fatal(myrank, "prep MPI_Barrier failed: %d\n", rc);
 652         }
 653
 654         if (order == READDIR) {
 655                 directory = opendir(dir);
 656                 if (!directory) {
 657                         rc = errno;
 658                         fatal(myrank, "opendir(%s) error: %s\n",
 659                               dir, strerror(rc));
 660                 }
 661
 662                 timestamp = time(0);
 663                 j = random() % nfiles;
 664                 dmesg("%d: %s initializing dir offset %u: %s",
 665                       myrank, hostname, j, ctime(&timestamp));
 666
 667                 for (i = 0; i <= j; i++) {
 668                         if ((dir_entry = readdir(directory)) == NULL) {
 669                                 fatal(myrank,
 670                                       "could not read entry number %d in directory %s.\n",
 671                                       i, dir);
 672                         }
 673                 }
 674
 675                 timestamp = time(0);
 676                 dmesg("%d: index %d, filename %s, offset %ld: %s initialization complete: %s",
 677                       myrank, i, dir_entry->d_name, telldir(directory),
 678                       hostname, ctime(&timestamp));
 679         }
 680
 681         if (seconds) {
 682                 act.sa_handler = sigalrm_handler;
 683                 (void)sigemptyset(&act.sa_mask);
 684                 act.sa_flags = 0;
 685                 sigaction(SIGALRM, &act, NULL);
 686                 alarm(seconds);
 687         }
 688
 689         rc = MPI_Barrier(MPI_COMM_WORLD);
 690         if (rc != MPI_SUCCESS)
 691                 fatal(myrank, "prep MPI_Barrier failed: %d\n", rc);
 692
 693         startTime = MPI_Wtime();
 694         lastTime = MPI_Wtime();
 695         nops = 0;
 696         lastOps = 0;
 697
 698         switch (mode) {
 699         case CREATE:
 700                 for (; begin <= end && !alarm_caught; begin += dirthreads) {
 701                         snprintf(filename, sizeof(filename), filefmt, begin);
 702                         fd = open(filename, openflags, 0644);
 703                         if (fd < 0) {
 704                                 rc = errno;
 705                                 if (rc == EINTR && alarm_caught)
 706                                         break;
 707                                 fatal(myrank, "open(%s) error: %s\n",
 708                                       filename, strerror(rc));
 709                         }
 710
 711                         if (with_xattr) {
 712                                 rc = fsetxattr(fd, xattrname, xattrbuf,
 713                                                xattrlen, XATTR_CREATE);
 714                                 if (rc) {
 715                                         rc = errno;
 716                                         if (rc == EINTR && alarm_caught)
 717                                                 break;
 718                                         fatal(myrank,
 719                                               "setxattr(%s) error: %s\n",
 720                                               filename, strerror(rc));
 721                                 }
 722                         }
 723                         if (smallwrite) {
 724                                 rc = write(fd, xattrbuf, xattrlen);
 725                                 if (rc < 0) {
 726                                         rc = errno;
 727                                         if (rc == EINTR && alarm_caught)
 728                                                 break;
 729                                         fatal(myrank,
 730                                               "write(%s) error: %s\n",
 731                                               filename, strerror(rc));
 732                                 }
 733                         }
 734
 735                         close(fd);
 736                         nops++;
 737                         DISPLAY_PROGRESS();
 738                 }
 739
 740                 dmesg("%d: created %d files, last file '%s'.\n",
 741                       myrank, nops, filename);
 742                 break;
 743         case LOOKUP:
 744                 fd = open(dir, O_RDONLY);
 745                 if (fd < 0) {
 746                         fatal(myrank, "open(dir == '%s') error: %s\n",
 747                               dir, strerror(errno));
 748                 }
 749
 750                 for (; nops < iters && !alarm_caught;) {
 751                         char *filename = next_file();
 752
 753                         rc = llapi_file_lookup(fd, filename);
 754                         if (rc < 0) {
 755                                 if (((rc = errno) == EINTR) && alarm_caught)
 756                                         break;
 757                                 fatal(myrank,
 758                                       "llapi_file_lookup(%s) error: %s\n",
 759                                       filename, strerror(rc));
 760                         }
 761
 762                         nops++;
 763                         DISPLAY_PROGRESS();
 764                 }
 765                 break;
 766         case MKNOD:
 767                 for (; begin <= end && !alarm_caught; begin += dirthreads) {
 768                         snprintf(filename, sizeof(filename), filefmt, begin);
 769                         rc = mknod(filename, S_IFREG | 0644, 0);
 770                         if (rc) {
 771                                 rc = errno;
 772                                 if (rc == EINTR && alarm_caught)
 773                                         break;
 774                                 fatal(myrank, "mknod(%s) error: %s\n",
 775                                       filename, strerror(rc));
 776                         }
 777
 778                         if (with_xattr) {
 779                                 rc = setxattr(filename, xattrname, xattrbuf,
 780                                               xattrlen, XATTR_CREATE);
 781                                 if (rc) {
 782                                         rc = errno;
 783                                         if (rc == EINTR && alarm_caught)
 784                                                 break;
 785                                         fatal(myrank,
 786                                               "setxattr(%s) error: %s\n",
 787                                               filename, strerror(rc));
 788                                 }
 789                         }
 790
 791                         nops++;
 792                         DISPLAY_PROGRESS();
 793                 }
 794                 break;
 795         case OPEN:
 796                 for (; nops < iters && !alarm_caught;) {
 797                         file = next_file();
 798                         if ((fd = open(file, openflags, 0644)) < 0) {
 799                                 if (((rc = errno) == EINTR) && alarm_caught)
 800                                         break;
 801                                 fatal(myrank, "open(%s) error: %s\n",
 802                                       file, strerror(rc));
 803                         }
 804
 805                         close(fd);
 806
 807                         nops++;
 808                         DISPLAY_PROGRESS();
 809                 }
 810                 break;
 811         case STAT:
 812                 for (; begin <= end && !alarm_caught; begin += dirthreads) {
 813                         sprintf(filename, filefmt, begin);
 814                         rc = stat(filename, &statbuf);
 815                         if (rc) {
 816                                 if (((rc = errno) == EINTR) && alarm_caught)
 817                                         break;
 818                                 if (((rc = errno) == ENOENT) && ignore)
 819                                         continue;
 820                                 fatal(myrank, "stat(%s) error: %s\n",
 821                                       filename, strerror(rc));
 822                         }
 823
 824                         nops++;
 825                         DISPLAY_PROGRESS();
 826                 }
 827                 break;
 828         case UNLINK:
 829                 for (; begin <= end && !alarm_caught; begin += dirthreads) {
 830                         sprintf(filename, filefmt, begin);
 831                         rc = unlink(filename);
 832                         if (rc) {
 833                                 if (((rc = errno) == EINTR) && alarm_caught)
 834                                         break;
 835                                 if ((rc = errno) == ENOENT) {
 836                                         if (ignore)
 837                                                 continue;
 838                                         /* no more files to unlink */
 839                                         break;
 840                                 }
 841                                 fatal(myrank, "unlink(%s) error: %s\n",
 842                                       filename, strerror(rc));
 843                         }
 844
 845                         nops++;
 846                         DISPLAY_PROGRESS();
 847                 }
 848                 break;
 849         case SETXATTR:
 850                 for (; begin <= end && !alarm_caught; begin += dirthreads) {
 851                         snprintf(filename, sizeof(filename), filefmt, begin);
 852                         rc = setxattr(filename, xattrname, xattrbuf, xattrlen,
 853                                       XATTR_CREATE);
 854                         if (rc) {
 855                                 rc = errno;
 856                                 if (rc == EINTR && alarm_caught)
 857                                         break;
 858                                 if (rc == ENOENT && ignore)
 859                                         continue;
 860                                 fatal(myrank, "setxattr(%s) error: %s\n",
 861                                       filename, strerror(rc));
 862                         }
 863
 864                         nops++;
 865                         DISPLAY_PROGRESS();
 866                 }
 867                 break;
 868         }
 869
 870         rc = MPI_Barrier(MPI_COMM_WORLD);
 871         if (rc != MPI_SUCCESS)
 872                 fatal(myrank, "prep MPI_Barrier failed: %d\n", rc);
 873         curTime = MPI_Wtime();
 874         interval = curTime - startTime;
 875         rate = (double)(nops) / interval;
 876
 877         rc = MPI_Reduce(&nops, &ag_ops, 1, MPI_INT, MPI_SUM, 0,
 878                         MPI_COMM_WORLD);
 879         if (rc != MPI_SUCCESS)
 880                 fatal(myrank, "Failure in MPI_Reduce of total ops.\n");
 881
 882         rc = MPI_Reduce(&interval, &ag_interval, 1, MPI_DOUBLE, MPI_SUM, 0,
 883                         MPI_COMM_WORLD);
 884         if (rc != MPI_SUCCESS)
 885                 fatal(myrank, "Failure in MPI_Reduce of total interval.\n");
 886
 887         rc = MPI_Reduce(&rate, &ag_rate, 1, MPI_DOUBLE, MPI_SUM, 0,
 888                         MPI_COMM_WORLD);
 889         if (rc != MPI_SUCCESS)
 890                 fatal(myrank, "Failure in MPI_Reduce of aggregated rate.\n");
 891
 892         if (myrank == 0) {
 893                 curTime = MPI_Wtime();
 894                 interval = curTime - startTime;
 895                 effective_rate = (double)ag_ops / interval;
 896                 avg_rate = (double)ag_ops / ag_interval;
 897
 898                 printf("Rate: %.2f eff %.2f aggr %.2f avg client %ss/sec (total: %d threads %d %ss %d dirs %d threads/dir %.2f secs)\n",
 899                        effective_rate, ag_rate, avg_rate, cmd, nthreads, ag_ops,
 900                        cmd, ndirs, dirthreads, interval);
 901                 if (mode == UNLINK && !recreate && !ignore && ag_ops != nfiles)
 902                         printf("Warning: only unlinked %d files instead of %d\n",
 903                                ag_ops, nfiles);
 904         }
 905
 906         if (recreate) {
 907                 for (begin = beginsave; begin <= end; begin += dirthreads) {
 908                         sprintf(filename, filefmt, begin);
 909                         if ((fd = open(filename, openflags, 0644)) < 0) {
 910                                 rc = errno;
 911                                 if (rc == EEXIST)
 912                                         break;
 913                                 fatal(myrank, "recreate open(%s) error: %s\n",
 914                                       filename, strerror(rc));
 915                         }
 916
 917                         close(fd);
 918                 }
 919         }
 920
 921         timestamp = time(0);
 922         if ((myrank == 0) || debug) {
 923                 printf("%d: %s finished at %s",
 924                        myrank, hostname, ctime(&timestamp));
 925         }
 926
 927         MPI_Finalize();
 928         return 0;
 929 }