Whamcloud - gitweb
LU-6210 tests: Change positional struct initializers to C99
[fs/lustre-release.git] / lustre / tests / mpi / mdsrate.c
1 /*
2  * 2003, Copyright, Hewlett-Packard Development Compnay, LP.
3  *
4  * Developed under the sponsorship of the U.S. Government
5  *     under Subcontract No. B514193
6  */
7
8 /*
9  * Copyright (c) 2008, 2010, Oracle and/or its affiliates. All rights reserved.
10  * Use is subject to license terms.
11  *
12  * Copyright (c) 2012, 2015, Intel Corporation.
13  */
14
15 #include <stdio.h>
16 #include <getopt.h>
17 #include <libgen.h>
18 #include <sys/types.h>
19 #include <sys/stat.h>
20 #include <time.h>
21 #include <limits.h>
22 #include <errno.h>
23 #include <string.h>
24 #include <fcntl.h>
25 #include <unistd.h>
26 #include <stdlib.h>
27 #include <stdarg.h>
28 #include <signal.h>
29 #include <sys/ioctl.h>
30 #include <dirent.h>
31 #include <sys/xattr.h>
32
33 #include "mpi.h"
34
35 /* lustre */
36 #include <lustre/lustreapi.h>        /* for O_LOV_DELAY_CREATE */
37
38 #define CHECK_COUNT 10000
39 #define DISPLAY_COUNT (CHECK_COUNT * 10)
40 #define DISPLAY_TIME 100
41
42 enum {
43         CREATE          = 'c',
44         LOOKUP          = 'l',
45         MKNOD           = 'm',
46         OPEN            = 'o',
47         STAT            = 's',
48         UNLINK          = 'u',
49         BEGIN           = 'b',
50         ITERS           = 'i',
51         TIME            = 't',
52         DIRFMT          = 'd',
53         NDIRS           = 'D',
54         FILEFMT         = 'f',
55         NFILES          = 'F',
56         NOEXCL          = 'X',
57         STRIPES         = 'S',
58         SEED            = 'r',
59         SEEDFILE        = 'R',
60         RANDOM          = 'A',
61         READDIR         = 'B',
62         RECREATE        = 'C',
63         SETXATTR        = 'x',
64         SMALLWRITE      = 'w',
65         IGNORE          = 'E',
66         VERBOSE         = 'V',
67         DEBUG           = 'v',
68         HELP            = 'h',
69         MNT             = 'M',
70         MNTCOUNT        = 'N',
71         MDTCOUNT        = 'T',
72 };
73
74 struct option longOpts[] = {
75         { .name = "create", .has_arg = no_argument, .val = CREATE },
76         { .name = "lookup", .has_arg = no_argument, .val = LOOKUP },
77         { .name = "mknod", .has_arg = no_argument, .val = MKNOD },
78         { .name = "open", .has_arg = no_argument, .val = OPEN },
79         { .name = "stat", .has_arg = no_argument, .val = STAT },
80         { .name = "unlink", .has_arg = no_argument, .val = UNLINK },
81         { .name = "begin", .has_arg = required_argument, .val = BEGIN },
82         { .name = "iters", .has_arg = required_argument, .val = ITERS },
83         /* time is in seconds */
84         { .name = "time", .has_arg = required_argument, .val = TIME },
85         { .name = "dirfmt", .has_arg = required_argument, .val = DIRFMT },
86         { .name = "ndirs", .has_arg = required_argument, .val = NDIRS },
87         { .name = "filefmt", .has_arg = required_argument, .val = FILEFMT },
88         { .name = "nfiles", .has_arg = required_argument, .val = NFILES },
89         { .name = "noexcl", .has_arg = no_argument, .val = NOEXCL },
90         { .name = "stripes", .has_arg = required_argument, .val = STRIPES },
91         { .name = "seed", .has_arg = required_argument, .val = SEED },
92         { .name = "seedfile", .has_arg = required_argument, .val = SEEDFILE },
93         { .name = "random_order", .has_arg = no_argument, .val = RANDOM },
94         { .name = "readdir_order", .has_arg = no_argument, .val = READDIR },
95         { .name = "recreate", .has_arg = no_argument, .val = RECREATE },
96         { .name = "setxattr", .has_arg = no_argument, .val = SETXATTR },
97         { .name = "smallwrite", .has_arg = no_argument, .val = SMALLWRITE },
98         { .name = "ignore", .has_arg = no_argument, .val = IGNORE },
99         { .name = "verbose", .has_arg = no_argument, .val = VERBOSE },
100         { .name = "debug", .has_arg = no_argument, .val = DEBUG },
101         { .name = "help", .has_arg = no_argument, .val = HELP },
102         { .name = "mdtcount", .has_arg = required_argument, .val = MDTCOUNT },
103         { .name = "mntcount", .has_arg = required_argument, .val = MNTCOUNT },
104         { .name = "mntfmt", .has_arg = required_argument, .val = MNT },
105         { .name = NULL }
106 };
107
108 int foo1, foo2;
109
110 char   shortOpts[128];
111 int    myrank = -1;
112 int    nthreads = -1;
113 char * prog;
114 char   hostname[512] = "unknown";
115 char   mode;
116 char * cmd;
117 int    openflags = O_RDWR|O_CREAT|O_EXCL;
118 int    ndirs = 1;
119 char * dirfmt;
120 char   dir[PATH_MAX];
121 char   mkdir_cmd[PATH_MAX+14];
122 int    dirthreads;
123 int    dirnum;
124 DIR *  directory;
125 struct dirent *dir_entry;
126 int    nfiles;
127 char   filefmt[PATH_MAX];
128 char   filename[PATH_MAX];
129 char   path[PATH_MAX];
130 int    stripes = -1;
131 int    begin;
132 int    beginsave;
133 int    end;
134 int    iters;
135 int    seconds;
136 int    alarm_caught;
137 struct sigaction act;
138 int    order = RANDOM;
139 int    seed;
140 int    recreate;
141 int    ignore;
142 int    verbose;
143 int    debug;
144 struct stat statbuf;
145 bool   with_xattr;
146 char   xattrname[] = "user.mdsrate";
147 char   xattrbuf[4096];
148 /* max xattr name + value length is block size, use 4000 here to avoid ENOSPC */
149 int    xattrlen = 4000;
150 bool   smallwrite;
151 int    mnt_count = -1;
152 int    mdt_count = 1;
153 char  *mntfmt;
154
155 #define dmesg if (debug) printf
156
157 #define DISPLAY_PROGRESS() {                                                \
158         if (verbose && (nops % CHECK_COUNT == 0)) {                         \
159                 curTime = MPI_Wtime();                                      \
160                 interval = curTime - lastTime;                              \
161                 if (interval > DISPLAY_TIME || nops % DISPLAY_COUNT == 0) { \
162                         rate = (double)(nops - lastOps)/interval;           \
163                         printf("Rank %d: %.2f %ss/sec %.2f secs "           \
164                                "(total: %d %ss %.2f secs)\n",               \
165                                myrank, rate, cmd, interval,                 \
166                                nops, cmd, curTime - startTime);             \
167                         lastOps = nops;                                     \
168                         lastTime = curTime;                                 \
169                 }                                                           \
170         }                                                                   \
171 }
172
173 char *usage_msg = "usage: %s\n"
174                   "    { --create [ --noexcl | --setxattr | --smallwrite ] |\n"
175                   "      --lookup | --mknod [ --setxattr ] | --open |\n"
176                   "      --stat | --unlink [ --recreate ] [ --ignore ] |\n"
177                   "      --setxattr }\n"
178                   "    [ --help ] [ --verbose ] [ --debug ]\n"
179                   "    { [ --begin <num> ] --nfiles <num> }\n"
180                   "    [ --iters <num> ] [ --time <secs> ]\n"
181                   "    [ --dirfmt <str> ] [ --ndirs  <num> ]\n"
182                   "    [ --filefmt <str> ] [ --stripes <num> ]\n"
183                   "    [ --random_order [--seed <num> | --seedfile <file>] ]\n"
184                   "    [ --readdir_order ] [ --mntfmt <str> ]\n"
185                   "    [ --mntcount <num> ] [ --mdtcount <num> ]\n"
186                   "    [ --setxattr ] }\n";
187
188 static void
189 usage(FILE *stream, char *fmt, ...)
190 {
191         if (myrank == 0) {
192                 if (fmt != NULL) {
193                         va_list       ap;
194
195                         fprintf(stream, "%s: ", prog);
196                         va_start(ap, fmt);
197                         vfprintf(stderr, fmt, ap);
198                         va_end(ap);
199                 }
200                 fprintf(stream, usage_msg, prog);
201         }
202
203         MPI_Finalize();
204         exit(stream == stderr);
205 }
206
207 /* Print process myrank and message, and exit (i.e. a fatal error) */
208 static int
209 fatal(int rank, const char *fmt, ...)
210 {
211         if (rank == myrank) {
212                 va_list       ap;
213
214                 fprintf(stderr, "rank %d: ", rank);
215                 va_start(ap, fmt);
216                 vfprintf(stderr, fmt, ap);
217                 va_end(ap);
218         }
219
220         MPI_Abort(MPI_COMM_WORLD, 1);
221         exit(1);
222 }
223
224 static void
225 sigalrm_handler(int signum)
226 {
227         alarm_caught++;
228 }
229
230 /* HAVE_LLAPI_FILE_LOOKUP is defined by liblustreapi.h if this function is
231  * defined therein.  Otherwise we can do the equivalent operation via ioctl
232  * if we have access to a complete lustre build tree to get the various
233  * definitions - then compile with USE_MDC_LOOKUP defined. */
234 #if defined(HAVE_LLAPI_FILE_LOOKUP)
235 #define HAVE_MDC_LOOKUP
236 #elif defined(USE_MDC_LOOKUP)
237 #include <config.h>
238 #include <lustre_ioctl.h>
239
240 int llapi_file_lookup(int dirfd, const char *name)
241 {
242         struct obd_ioctl_data data = { 0 };
243         char rawbuf[8192];
244         char *buf = rawbuf;
245         int rc;
246
247         if (dirfd < 0 || name == NULL)
248                 return -EINVAL;
249
250         data.ioc_version = OBD_IOCTL_VERSION;
251         data.ioc_len = sizeof(data);
252         data.ioc_inlbuf1 = name;
253         data.ioc_inllen1 = strlen(name) + 1;
254
255         rc = obd_ioctl_pack(&data, &buf, sizeof(rawbuf));
256         if (rc) {
257                 fatal(myrank, "ioctl_pack failed: rc = %d\n", rc);
258                 return rc;
259         }
260
261         return ioctl(fd, IOC_MDC_LOOKUP, buf);
262 }
263 #define HAVE_MDC_LOOKUP
264 #endif
265
266 static void
267 process_args(int argc, char *argv[])
268 {
269         char   *cp, *endptr;
270         int    i, index, offset, tmpend, rc;
271         char   tmp[16];
272         FILE * seed_file;
273         struct option *opt;
274
275         setbuf(stdout, 0);
276         setbuf(stderr, 0);
277         prog = basename(argv[0]);
278         strcpy(filefmt, "f%d");
279         gethostname(hostname, sizeof(hostname));
280
281         /* auto create shortOpts rather than maintaining a static string. */
282         for (opt = longOpts, cp = shortOpts; opt->name != NULL; opt++, cp++) {
283                 *cp = opt->val;
284                 if (opt->has_arg)
285                         *++cp = ':';
286         }
287
288         while ((rc = getopt_long(argc,argv, shortOpts, longOpts,&index)) != -1) {
289                 switch (rc) {
290                 case OPEN:
291                         openflags &= ~(O_CREAT|O_EXCL);
292                 case CREATE:
293 #ifdef HAVE_MDC_LOOKUP
294                 case LOOKUP:
295 #endif
296                 case MKNOD:
297                 case STAT:
298                 case UNLINK:
299                         if (cmd != NULL) {
300                                 fatal(0, "Invalid - more than one operation "
301                                            "specified: --%s\n",
302                                         longOpts[index].name);
303                         }
304                         mode = rc;
305                         cmd = (char *)longOpts[index].name;
306                         break;
307                 case NOEXCL:
308                         if (mode != CREATE && mode != MKNOD) {
309                                 usage(stderr, "--noexcl only applies to "
310                                               "--create or --mknod.\n");
311                         }
312                         openflags &= ~O_EXCL;
313                         break;
314                 case RECREATE:
315                         if (mode != UNLINK) {
316                                 usage(stderr, "--recreate only makes sense"
317                                               "with --unlink.\n");
318                         }
319                         recreate++;
320                         break;
321                 case SETXATTR:
322                         if (cmd == NULL) {
323                                 mode = SETXATTR;
324                                 cmd = (char *)longOpts[index].name;
325                         } else if (mode == CREATE || mode == MKNOD) {
326                                 with_xattr = true;
327                         } else {
328                                 usage(stderr, "--setxattr only makes sense "
329                                       "with --create, --mknod or alone.\n");
330                         }
331                         break;
332                 case SMALLWRITE:
333                         if (mode != CREATE)
334                                 usage(stderr, "--smallwrite only applies to "
335                                               "--create.\n");
336                         smallwrite = true;
337                         break;
338                 case BEGIN:
339                         begin = strtol(optarg, &endptr, 0);
340                         if ((*endptr != 0) || (begin < 0)) {
341                                 fatal(0, "Invalid --start value.\n");
342                         }
343                         break;
344                 case ITERS:
345                         iters = strtol(optarg, &endptr, 0);
346                         if ((*endptr != 0) || (iters <= 0)) {
347                                 fatal(0, "Invalid --iters value.\n");
348                         }
349                         if (mode != LOOKUP && mode != OPEN) {
350                                 usage(stderr, "--iters only makes sense with "
351                                               "--lookup or --open.\n");
352                         }
353                         break;
354                 case TIME:
355                         seconds = strtol(optarg, &endptr, 0);
356                         if ((*endptr != 0) || (seconds <= 0)) {
357                                 fatal(0, "Invalid --time value.\n");
358                         }
359                         break;
360                 case DIRFMT:
361                         if (strlen(optarg) > (PATH_MAX - 16)) {
362                                 fatal(0, "--dirfmt too long\n");
363                         }
364                         dirfmt = optarg;
365                         break;
366                 case NDIRS:
367                         ndirs = strtol(optarg, &endptr, 0);
368                         if ((*endptr != 0) || (ndirs <= 0)) {
369                                 fatal(0, "Invalid --ndirs value.\n");
370                         }
371                         if ((ndirs > nthreads) &&
372                             ((mode == CREATE) || (mode == MKNOD))) {
373                                 fatal(0, "--ndirs=%d must be less than or "
374                                       "equal to the number of threads (%d).\n",
375                                       ndirs, nthreads);
376                         }
377                         break;
378                 case FILEFMT:
379                         if (strlen(optarg) > 4080) {
380                                 fatal(0, "--filefmt too long\n");
381                         }
382
383                         /* Use %%d where you want the file # in the name. */
384                         sprintf(filefmt, optarg, myrank);
385                         break;
386                 case NFILES:
387                         nfiles = strtol(optarg, &endptr, 0);
388                         if ((*endptr != 0) || (nfiles <= 0)) {
389                                 fatal(0, "Invalid --nfiles value.\n");
390                         }
391                         break;
392                 case STRIPES:
393                         stripes = strtol(optarg, &endptr, 0);
394                         if ((*endptr != 0) || (stripes < 0)) {
395                                 fatal(0, "Invalid --stripes value.\n");
396                         }
397
398                         if (stripes == 0) {
399                                 openflags |= O_LOV_DELAY_CREATE;
400                         } else {
401                                 fatal(0, "non-zero --stripes value "
402                                          "not yet supported.\n");
403                         }
404
405                         break;
406                 case SEED:
407                         seed = strtoul(optarg, &endptr, 0);
408                         if (*endptr) {
409                                 fatal(0, "bad --seed option %s\n", optarg);
410                         }
411                         break;
412                 case SEEDFILE:
413                         seed_file = fopen(optarg, "r");
414                         if (!seed_file) {
415                               fatal(myrank, "fopen(%s) error: %s\n",
416                                       optarg, strerror(errno));
417                         }
418
419                         for (i = -1; fgets(tmp, 16, seed_file) != NULL;) {
420                                 if (++i == myrank)
421                                         break;
422                         }
423
424                         if (i == myrank) {
425                                 rc = sscanf(tmp, "%d", &seed);
426                                 if ((rc != 1) || (seed < 0)) {
427                                         fatal(myrank, "Invalid seed value '%s' "
428                                               "at line %d in %s.\n",
429                                               tmp, i, optarg);
430                                 }
431                         } else {
432                                 fatal(myrank, "File '%s' too short. Does not "
433                                       "contain a seed for thread %d.\n",
434                                       optarg, myrank);
435                         }
436
437                         fclose(seed_file);
438                         break;
439                 case RANDOM:
440                 case READDIR:
441                         if (mode != LOOKUP && mode != OPEN)  {
442                                 fatal(0, "--%s can only be specified with "
443                                          "--lookup, or --open.\n",
444                                       (char *)longOpts[index].name);
445                         }
446                         order = rc;
447                         break;
448                 case IGNORE:
449                         ++ignore;
450                         break;
451                 case DEBUG:
452                         ++debug;
453                 case VERBOSE:
454                         ++verbose;
455                         break;
456                 case HELP:
457                         usage(stdout, NULL);
458                         break;
459                 case MNT:
460                         if (strlen(optarg) > (PATH_MAX - 16))
461                                 fatal(0, "--mnt too long\n");
462                         mntfmt = optarg;
463                         break;
464                 case MNTCOUNT:
465                         mnt_count = strtol(optarg, &endptr, 0);
466                         if ((*endptr != 0) || (mnt_count <= 0)) {
467                                 fatal(0, "Invalid --mnt_count value %s.\n",
468                                       optarg);
469                         }
470                         break;
471                 case MDTCOUNT:
472                         mdt_count = strtol(optarg, &endptr, 0);
473                         if ((*endptr != 0) || (mdt_count <= 0)) {
474                                 fatal(0, "Invalid --mdt_count value %s.\n",
475                                       optarg);
476                         }
477                         break;
478                 default:
479                         usage(stderr, "unrecognized option: '%c'.\n", optopt);
480                 }
481         }
482
483         if (optind < argc) {
484                 usage(stderr, "too many arguments %d >= %d.\n", optind, argc);
485         }
486
487         if ((mnt_count != -1 && mntfmt == NULL) ||
488             (mnt_count == -1 && mntfmt != NULL)) {
489                 usage(stderr, "mnt_count and mntfmt must be specified at the "
490                              "same time\n");
491         }
492
493         if (mode == CREATE || mode == MKNOD || mode == UNLINK ||
494             mode == STAT || mode == SETXATTR) {
495                 if (seconds != 0) {
496                         if (nfiles == 0)
497                                 nfiles = INT_MAX;
498                 } else if (nfiles == 0) {
499                         usage(stderr, "--nfiles or --time must be specified "
500                                       "with %s.\n", cmd);
501                 }
502         } else if (mode == LOOKUP || mode == OPEN) {
503                 if (seconds != 0) {
504                         if (iters == 0)
505                                 iters = INT_MAX;
506                 } else if (iters == 0) {
507                         usage(stderr, "--iters or --time must be specifed "
508                                       "with %s.\n", cmd);
509                 }
510
511                 if (nfiles == 0) {
512                         usage(stderr, "--nfiles must be specifed with --%s.\n",
513                               cmd);
514                 }
515
516                 if (seed == 0) {
517                         int fd = open("/dev/urandom", O_RDONLY);
518
519                         if (fd >= 0) {
520                                 if (read(fd, &seed, sizeof(seed)) <
521                                     sizeof(seed))
522                                         seed = time(0);
523                                 close(fd);
524                         } else {
525                                 seed = time(0);
526                         }
527                 }
528
529                 srand(seed);
530
531                 dmesg("%s: rank %d seed %d (%s).\n", prog, myrank, seed,
532                       (order == RANDOM) ? "random_order" : "readdir_order");
533         } else {
534                 usage(stderr, "one --create, --mknod, --open, --stat,"
535 #ifdef HAVE_MDC_LOOKUP
536                       " --lookup,"
537 #endif
538                       " --unlink or --setxattr must be specifed.");
539         }
540
541         /* support for multiple threads in a dir, set begin/end appropriately.*/
542         dirnum = myrank % ndirs;
543         dirthreads = nthreads / ndirs;
544         if (nthreads > (ndirs * dirthreads + dirnum))
545                 ++dirthreads;
546
547         offset = myrank / ndirs;
548
549         tmpend = begin + nfiles - 1;
550         if (tmpend <= 0)
551                 tmpend = INT_MAX;
552
553         end = begin + (nfiles / dirthreads) * dirthreads + offset;
554         if ((end > tmpend) || (end <= 0))
555                 end -= dirthreads;
556
557         /* make sure mnt_count <= nthreads, otherwise it might div 0 in
558          * the following test */
559         if (mnt_count > nthreads)
560                 mnt_count = nthreads;
561
562         begin += offset;
563         if (begin < 0)
564                 begin = INT_MAX;
565
566         beginsave = begin;
567
568         dmesg("%d: iters %d nfiles %d time %d begin %d end %d dirthreads %d."
569               "\n", myrank, iters, nfiles, seconds, begin, end, dirthreads);
570
571         if (dirfmt == NULL) {
572                 strcpy(dir, ".");
573         } else {
574                 int dir_len = 0;
575
576                 if (mntfmt != NULL) {
577                         sprintf(dir, mntfmt, (myrank / (nthreads/mnt_count)));
578                         strcat(dir, "/");
579                         dir_len = strlen(dir);
580                 }
581                 sprintf(dir + dir_len, dirfmt, dirnum);
582
583                 if (mdt_count > 1) {
584                         struct stat sb;
585                         if (stat(dir, &sb) == 0) {
586                                 if (!S_ISDIR(sb.st_mode))
587                                         fatal(myrank, "'%s' is not dir\n", dir);
588                         } else if (errno == ENOENT) {
589                                 sprintf(mkdir_cmd, "lfs mkdir -i %d %s",
590                                         myrank % mdt_count, dir);
591                         } else {
592                                 fatal(myrank, "'%s' stat failed\n", dir);
593                         }
594                 } else {
595                         sprintf(mkdir_cmd, "mkdir -p %s", dir);
596                 }
597
598                 dmesg("%d: %s\n", myrank, mkdir_cmd);
599 #ifdef _LIGHTWEIGHT_KERNEL
600                 printf("NOTICE: not running system(%s)\n", mkdir_cmd);
601 #else
602                 rc = system(mkdir_cmd);
603                 if (rc)
604                         fatal(myrank, "'%s' failed.\n", mkdir_cmd);
605 #endif
606
607                 rc = chdir(dir);
608                 if (rc) {
609                         fatal(myrank, "unable to chdir to '%s'.\n", dir);
610                 }
611         }
612 }
613
614 static inline char *next_file()
615 {
616         if (order == RANDOM) {
617                 sprintf(filename, filefmt, random() % nfiles);
618                 return(filename);
619         }
620
621         /* readdir order */
622
623         dir_entry = readdir(directory);
624         if (dir_entry == NULL) {
625                 rewinddir(directory);
626                 while ((dir_entry = readdir(directory)) != NULL) {
627                         if (dir_entry->d_name[0] != '.')
628                                 return(dir_entry->d_name);
629                 }
630
631                 fatal(myrank, "unable to read directory %s (%s).\n",
632                       dir, strerror(errno));
633         }
634
635         return(dir_entry->d_name);
636 }
637
638 int
639 main(int argc, char *argv[])
640 {
641         int    i, j, fd, rc, nops, lastOps;
642         int ag_ops = 0;
643         double ag_interval = 0;
644         double ag_rate = 0;
645         double rate, avg_rate, effective_rate;
646         double startTime, curTime, lastTime, interval;
647         time_t timestamp;
648         char * file;
649
650         rc = MPI_Init(&argc, &argv);
651         if (rc != MPI_SUCCESS)
652                 fatal(myrank, "MPI_Init failed: %d\n", rc);
653
654         rc = MPI_Comm_size(MPI_COMM_WORLD, &nthreads);
655         if (rc != MPI_SUCCESS)
656                 fatal(myrank, "MPI_Comm_size failed: %d\n", rc);
657
658         rc = MPI_Comm_rank(MPI_COMM_WORLD, &myrank);
659         if (rc != MPI_SUCCESS)
660                 fatal(myrank, "MPI_Comm_rank failed: %d\n", rc);
661
662         process_args(argc, argv);
663
664         timestamp = time(0);
665         if ((myrank == 0) || debug) {
666                 printf("%d: %s starting at %s",
667                        myrank, hostname, ctime(&timestamp));
668         }
669
670         /* if we're not measuring creation rates then precreate
671          * the files we're operating on. */
672         if ((mode != CREATE) && (mode != MKNOD) && !ignore &&
673             (mode != UNLINK || recreate)) {
674                 /* create the files in reverse order. When we encounter
675                  * a file that already exists, assume the remainder of 
676                  * the files exist to save time. The timed performance
677                  * test scripts make use of this behavior. */
678                 for (i = end, j = 0; i >= begin; i -= dirthreads) {
679                         sprintf(filename, filefmt, i);
680                         fd = open(filename, openflags, 0644);
681                         if (fd < 0) {
682                                 if (errno == EEXIST)
683                                         break;
684                                 rc = errno;
685                                 fatal(myrank, "precreate open(%s) error: %s\n",
686                                       filename, strerror(rc));
687                         }
688                         j++;
689                         close(fd);
690                 }
691                 dmesg("%d: %s pre-created %d files.\n",myrank,hostname,j);
692
693                 rc = MPI_Barrier(MPI_COMM_WORLD);
694                 if (rc != MPI_SUCCESS)
695                         fatal(myrank, "prep MPI_Barrier failed: %d\n", rc);
696         }
697
698         if (order == READDIR) {
699                 directory = opendir(dir);
700                 if (directory == NULL) {
701                         rc = errno;
702                         fatal(myrank, "opendir(%s) error: %s\n",
703                               dir, strerror(rc));
704                 }
705
706                 timestamp = time(0);
707                 j = random() % nfiles;
708                 dmesg("%d: %s initializing dir offset %u: %s",
709                       myrank, hostname, j, ctime(&timestamp));
710
711                 for (i = 0; i <= j; i++) {
712                         if ((dir_entry = readdir(directory)) == NULL) {
713                                 fatal(myrank, "could not read entry number %d "
714                                       "in directory %s.\n", i, dir);
715                         }
716                 }
717
718                 timestamp = time(0);
719                 dmesg("%d: index %d, filename %s, offset %ld: "
720                       "%s initialization complete: %s",
721                       myrank, i, dir_entry->d_name, telldir(directory),
722                       hostname, ctime(&timestamp));
723         }
724
725         if (seconds) {
726                 act.sa_handler = sigalrm_handler;
727                 (void)sigemptyset(&act.sa_mask);
728                 act.sa_flags = 0;
729                 sigaction(SIGALRM, &act, NULL);
730                 alarm(seconds);
731         }
732
733         rc = MPI_Barrier(MPI_COMM_WORLD);
734         if (rc != MPI_SUCCESS)
735                 fatal(myrank, "prep MPI_Barrier failed: %d\n", rc);
736
737         startTime = lastTime = MPI_Wtime();
738         nops = lastOps = 0;
739
740         switch (mode) {
741         case CREATE:
742                 for (; begin <= end && !alarm_caught; begin += dirthreads) {
743                         snprintf(filename, sizeof(filename), filefmt, begin);
744                         fd = open(filename, openflags, 0644);
745                         if (fd < 0) {
746                                 rc = errno;
747                                 if (rc == EINTR && alarm_caught)
748                                         break;
749                                 fatal(myrank, "open(%s) error: %s\n",
750                                       filename, strerror(rc));
751                         }
752
753                         if (with_xattr) {
754                                 rc = fsetxattr(fd, xattrname, xattrbuf,
755                                                xattrlen, XATTR_CREATE);
756                                 if (rc) {
757                                         rc = errno;
758                                         if (rc == EINTR && alarm_caught)
759                                                 break;
760                                         fatal(myrank,
761                                               "setxattr(%s) error: %s\n",
762                                               filename, strerror(rc));
763                                 }
764                         }
765                         if (smallwrite) {
766                                 rc = write(fd, xattrbuf, xattrlen);
767                                 if (rc < 0) {
768                                         rc = errno;
769                                         if (rc == EINTR && alarm_caught)
770                                                 break;
771                                         fatal(myrank,
772                                               "write(%s) error: %s\n",
773                                               filename, strerror(rc));
774                                 }
775                         }
776
777                         close(fd);
778                         nops++;
779                         DISPLAY_PROGRESS();
780                 }
781
782                 dmesg("%d: created %d files, last file '%s'.\n",
783                       myrank, nops, filename);
784                 break;
785 #ifdef HAVE_MDC_LOOKUP
786         case LOOKUP:
787                 fd = open(dir, O_RDONLY);
788                 if (fd < 0) {
789                         fatal(myrank, "open(dir == '%s') error: %s\n",
790                               dir, strerror(errno));
791                 }
792
793                 for (; nops < iters && !alarm_caught;) {
794                         char *filename = next_file();
795                         rc = llapi_file_lookup(fd, filename);
796                         if (rc < 0) {
797                                 if (((rc = errno) == EINTR) && alarm_caught)
798                                         break;
799                                 fatal(myrank, "llapi_file_lookup(%s) "
800                                       "error: %s\n", filename, strerror(rc));
801                         }
802
803                         nops++;
804                         DISPLAY_PROGRESS();
805                 }
806                 break;
807 #endif
808         case MKNOD:
809                 for (; begin <= end && !alarm_caught; begin += dirthreads) {
810                         snprintf(filename, sizeof(filename), filefmt, begin);
811                         rc = mknod(filename, S_IFREG | 0644, 0);
812                         if (rc) {
813                                 rc = errno;
814                                 if (rc == EINTR && alarm_caught)
815                                         break;
816                                 fatal(myrank, "mknod(%s) error: %s\n",
817                                       filename, strerror(rc));
818                         }
819
820                         if (with_xattr) {
821                                 rc = setxattr(filename, xattrname, xattrbuf,
822                                               xattrlen, XATTR_CREATE);
823                                 if (rc) {
824                                         rc = errno;
825                                         if (rc == EINTR && alarm_caught)
826                                                 break;
827                                         fatal(myrank,
828                                               "setxattr(%s) error: %s\n",
829                                               filename, strerror(rc));
830                                 }
831                         }
832
833                         nops++;
834                         DISPLAY_PROGRESS();
835                 }
836                 break;
837         case OPEN:
838                 for (; nops < iters && !alarm_caught;) {
839                         file = next_file();
840                         if ((fd = open(file, openflags, 0644)) < 0) {
841                                 if (((rc = errno) == EINTR) && alarm_caught)
842                                         break;
843                                 fatal(myrank, "open(%s) error: %s\n",
844                                       file, strerror(rc));
845                         }
846
847                         close(fd);
848
849                         nops++;
850                         DISPLAY_PROGRESS();
851                 }
852                 break;
853         case STAT:
854                 for (; begin <= end && !alarm_caught; begin += dirthreads) {
855                         sprintf(filename, filefmt, begin);
856                         rc = stat(filename, &statbuf);
857                         if (rc) {
858                                 if (((rc = errno) == EINTR) && alarm_caught)
859                                         break;
860                                 if (((rc = errno) == ENOENT) && ignore)
861                                         continue;
862                                 fatal(myrank, "stat(%s) error: %s\n",
863                                       filename, strerror(rc));
864                         }
865
866                         nops++;
867                         DISPLAY_PROGRESS();
868                 }
869                 break;
870         case UNLINK:
871                 for (; begin <= end && !alarm_caught; begin += dirthreads) {
872                         sprintf(filename, filefmt, begin);
873                         rc = unlink(filename);
874                         if (rc) {
875                                 if (((rc = errno) == EINTR) && alarm_caught)
876                                         break;
877                                 if ((rc = errno) == ENOENT) {
878                                         if (ignore)
879                                                 continue;
880                                         /* no more files to unlink */
881                                         break;
882                                 }
883                                 fatal(myrank, "unlink(%s) error: %s\n",
884                                       filename, strerror(rc));
885                         }
886
887                         nops++;
888                         DISPLAY_PROGRESS();
889                 }
890                 break;
891         case SETXATTR:
892                 for (; begin <= end && !alarm_caught; begin += dirthreads) {
893                         snprintf(filename, sizeof(filename), filefmt, begin);
894                         rc = setxattr(filename, xattrname, xattrbuf, xattrlen,
895                                       XATTR_CREATE);
896                         if (rc) {
897                                 rc = errno;
898                                 if (rc == EINTR && alarm_caught)
899                                         break;
900                                 if (rc == ENOENT && ignore)
901                                         continue;
902                                 fatal(myrank, "setxattr(%s) error: %s\n",
903                                       filename, strerror(rc));
904                         }
905
906                         nops++;
907                         DISPLAY_PROGRESS();
908                 }
909                 break;
910         }
911
912         rc = MPI_Barrier(MPI_COMM_WORLD);
913         if (rc != MPI_SUCCESS)
914                fatal(myrank, "prep MPI_Barrier failed: %d\n", rc);
915         curTime = MPI_Wtime();
916         interval = curTime - startTime;
917         rate = (double) (nops) / interval;
918
919         rc = MPI_Reduce(&nops, &ag_ops, 1, MPI_INT, MPI_SUM, 0,
920                         MPI_COMM_WORLD);
921         if (rc != MPI_SUCCESS) {
922                 fatal(myrank, "Failure in MPI_Reduce of total ops.\n");
923         }
924
925         rc = MPI_Reduce(&interval, &ag_interval, 1, MPI_DOUBLE, MPI_SUM, 0,
926                         MPI_COMM_WORLD);
927         if (rc != MPI_SUCCESS) {
928                 fatal(myrank, "Failure in MPI_Reduce of total interval.\n");
929         }
930
931         rc = MPI_Reduce(&rate, &ag_rate, 1, MPI_DOUBLE, MPI_SUM, 0,
932                         MPI_COMM_WORLD);
933         if (rc != MPI_SUCCESS) {
934                 fatal(myrank, "Failure in MPI_Reduce of aggregated rate.\n");
935         }
936
937         if (myrank == 0) {
938                 curTime = MPI_Wtime();
939                 interval = curTime - startTime;
940                 effective_rate = (double) ag_ops / interval;
941                 avg_rate = (double) ag_ops / ag_interval;
942
943                 printf("Rate: %.2f eff %.2f aggr %.2f avg client %ss/sec "
944                        "(total: %d threads %d %ss %d dirs %d threads/dir %.2f secs)\n",
945                        effective_rate, ag_rate, avg_rate, cmd, nthreads, ag_ops,
946                        cmd, ndirs, dirthreads, interval);
947                 if (mode == UNLINK && !recreate && !ignore && ag_ops != nfiles)
948                         printf("Warning: only unlinked %d files instead of %d"
949                                "\n", ag_ops, nfiles);
950         }
951
952         if (recreate) {
953                 for (begin = beginsave; begin <= end; begin += dirthreads) {
954                         sprintf(filename, filefmt, begin);
955                         if ((fd = open(filename, openflags, 0644)) < 0) {
956                                 rc = errno;
957                                 if (rc == EEXIST)
958                                         break;
959                                 fatal(myrank, "recreate open(%s) error: %s\n",
960                                       filename, strerror(rc));
961                         }
962
963                         close(fd);
964                 }
965         }
966
967         timestamp = time(0);
968         if ((myrank == 0) || debug) {
969                 printf("%d: %s finished at %s",
970                        myrank, hostname, ctime(&timestamp));
971         }
972
973         MPI_Finalize();
974         return(0);
975 }