lustre/tests/write_disjoint.c

   1 /* -*- mode: c; c-basic-offset: 8; indent-tabs-mode: nil; -*-
   2  * vim:expandtab:shiftwidth=8:tabstop=8:
   3  *
   4  * Each loop does 3 things:
   5  *   - rank 0 truncates to 0
   6  *   - all ranks agree on a random chunk size
   7  *   - all ranks race to write their pattern to their chunk of the file
   8  *   - rank 0 makes sure that the resulting file size is ranks * chunk size
   9  *   - rank 0 makes sure that everyone's patterns went to the right place
  10  *
  11  * compile: mpicc -g -Wall -o write_disjoint write_disjoint.c
  12  * run:     mpirun -np N -machlist <hostlist file> write_disjoint
  13  *  or:     pdsh -w <N hosts> write_disjoint
  14  *  or:     prun -n N [-N M] write_disjoint
  15  */
  16 #include <stdlib.h>
  17 #include <stdio.h>
  18 #include <sys/types.h>
  19 #include <sys/stat.h>
  20 #include <fcntl.h>
  21 #include <string.h>
  22 #include <errno.h>
  23 #include <unistd.h>
  24 #include <stdarg.h>
  25 #include "mpi.h"
  26
  27 #define CHUNK_MAX_SIZE 123456
  28
  29 void rprintf(int rank, int loop, const char *fmt, ...)
  30 {
  31         va_list       ap;
  32
  33         printf("rank %d, loop %d: ", rank, loop);
  34
  35         va_start(ap, fmt);
  36
  37         vprintf(fmt, ap);
  38
  39         MPI_Abort(MPI_COMM_WORLD, -1); /* This will exit() according to man */
  40 }
  41
  42 #define CHUNK_SIZE(n) chunk_size[(n) % 2]
  43
  44 int main (int argc, char *argv[]) {
  45         int i, n, fd;
  46         unsigned long chunk_size[2];
  47         int rank, noProcessors, done;
  48         int error;
  49         off_t offset;
  50         char **chunk_buf;
  51         char *read_buf, c;
  52         struct stat stat_buf;
  53         ssize_t ret;
  54         char *filename = "/mnt/lustre/write_disjoint";
  55         int numloops = 1000;
  56
  57         error = MPI_Init(&argc, &argv);
  58         if (error != MPI_SUCCESS)
  59                 rprintf(-1, -1, "MPI_Init failed: %d\n", error);
  60         /* Parse command line options */
  61         while ((c = getopt(argc, argv, "f:n:")) != EOF) {
  62                 switch (c) {
  63                 case 'f':
  64                         filename = optarg;
  65                         break;
  66                 case 'n':
  67                         numloops = strtoul(optarg, NULL, 0);
  68                         break;
  69                 }
  70         }
  71
  72         MPI_Comm_size(MPI_COMM_WORLD, &noProcessors);
  73         MPI_Comm_rank(MPI_COMM_WORLD, &rank);
  74
  75         chunk_buf = malloc(noProcessors * sizeof(chunk_buf[0]));
  76         for (i=0; i < noProcessors; i++) {
  77                 chunk_buf[i] = malloc(CHUNK_MAX_SIZE);
  78                 memset(chunk_buf[i], 'A'+ i, CHUNK_MAX_SIZE);
  79         }
  80         read_buf = malloc(noProcessors * CHUNK_MAX_SIZE);
  81
  82         if (rank == 0) {
  83                 fd = open(filename, O_WRONLY|O_CREAT|O_TRUNC, 0666);
  84                 if (fd < 0)
  85                         rprintf(rank, -1, "open() returned %s\n",
  86                                 strerror(errno));
  87         }
  88         MPI_Barrier(MPI_COMM_WORLD);
  89
  90         fd = open(filename, O_RDWR);
  91         if (fd < 0)
  92                 rprintf(rank, -1, "open() returned %s\n", strerror(errno));
  93
  94         for (n = 0; n < numloops; n++) {
  95                 /* reset the environment */
  96                 if (rank == 0) {
  97                         ret = truncate(filename, 0);
  98                         if (ret != 0)
  99                                 rprintf(rank, n, "truncate() returned %s\n",
 100                                         strerror(errno) );
 101                 }
 102                 CHUNK_SIZE(n) = rand() % CHUNK_MAX_SIZE;
 103
 104                 if (n % 1000 == 0 && rank == 0)
 105                         printf("loop %d: chunk_size %lu\n", n, CHUNK_SIZE(n));
 106
 107                 MPI_Barrier(MPI_COMM_WORLD);
 108
 109                 /* Do the race */
 110                 offset = rank * CHUNK_SIZE(n);
 111                 lseek(fd, offset, SEEK_SET);
 112
 113                 done = 0;
 114                 do {
 115                         ret = write(fd, chunk_buf[rank] + done,
 116                                     CHUNK_SIZE(n) - done);
 117                         if (ret < 0)
 118                                 rprintf(rank, n, "write() returned %s\n",
 119                                         strerror(errno));
 120                         done += ret;
 121                 } while (done != CHUNK_SIZE(n));
 122
 123                 MPI_Barrier(MPI_COMM_WORLD);
 124
 125                 /* Check the result */
 126                 if (stat(filename, &stat_buf) < 0)
 127                         rprintf(rank, n, "error stating %s: %s\n",
 128                                 filename, strerror(errno));
 129
 130                 if (stat_buf.st_size != CHUNK_SIZE(n) * noProcessors) {
 131                         if (n > 0)
 132                                 printf("loop %d: chunk_size %lu, "
 133                                        "file size was %lu\n",
 134                                        n - 1, CHUNK_SIZE(n - 1),
 135                                        CHUNK_SIZE(n - 1) *noProcessors);
 136                         rprintf(rank, n, "invalid file size %lu"
 137                                 " instead of %lu = %lu * %u\n",
 138                                 (unsigned long)stat_buf.st_size,
 139                                 CHUNK_SIZE(n) * noProcessors,
 140                                 CHUNK_SIZE(n), noProcessors);
 141                 }
 142
 143                 if (rank == 0) {
 144                         if (lseek(fd, 0, SEEK_SET) < 0)
 145                                 rprintf(rank, n, "error seeking to 0: %s\n",
 146                                         strerror(errno));
 147
 148                         done = 0;
 149                         do {
 150                                 ret = read(fd, read_buf + done,
 151                                            CHUNK_SIZE(n) * noProcessors - done);
 152                                 if (ret < 0)
 153                                         rprintf(rank, n, "read returned %s\n",
 154                                                 strerror(errno));
 155
 156                                 done += ret;
 157                         } while (done != CHUNK_SIZE(n) * noProcessors);
 158
 159                         for (i = 0; i < noProcessors; i++) {
 160                                 char command[4096];
 161                                 int j;
 162                                 if (!memcmp(read_buf + (i * CHUNK_SIZE(n)),
 163                                             chunk_buf[i], CHUNK_SIZE(n)))
 164                                         continue;
 165
 166                                 /* print out previous chunk sizes */
 167                                 if (n > 0)
 168                                         printf("loop %d: chunk_size %lu\n",
 169                                                n - 1, CHUNK_SIZE(n - 1));
 170
 171                                 printf("loop %d: chunk %d corrupted "
 172                                        "with chunk_size %lu, page_size %d\n",
 173                                        n, i, CHUNK_SIZE(n), getpagesize());
 174                                 printf("ranks:\tpage boundry\tchunk boundry\t"
 175                                        "page boundry\n");
 176                                 for (j = 1 ; j < noProcessors; j++) {
 177                                         int b = j * CHUNK_SIZE(n);
 178                                         printf("%c -> %c:\t%d\t%d\t%d\n",
 179                                                'A' + j - 1, 'A' + j,
 180                                                b & ~(getpagesize()-1), b,
 181                                                (b + getpagesize()) &
 182                                                ~(getpagesize()-1));
 183                                 }
 184
 185                                 sprintf(command, "od -Ad -a %s", filename);
 186                                 system(command);
 187                                 rprintf(0, n, "data check error - exiting\n");
 188                         }
 189                 }
 190         }
 191
 192         printf("Finished after %d loops\n", n);
 193         MPI_Finalize();
 194         return 0;
 195 }