lustre/tests/write_disjoint.c

   1 /* -*- mode: c; c-basic-offset: 8; indent-tabs-mode: nil; -*-
   2  * vim:expandtab:shiftwidth=8:tabstop=8:
   3  *
   4  * Each loop does 3 things:
   5  *   - rank 0 truncates to 0
   6  *   - all ranks agree on a random chunk size
   7  *   - all ranks race to write their pattern to their chunk of the file
   8  *   - rank 0 makes sure that the resulting file size is ranks * chunk size
   9  *   - rank 0 makes sure that everyone's patterns went to the right place
  10  *
  11  * compile: mpicc -g -Wall -o write_disjoint write_disjoint.c
  12  * run:     mpirun -np N -machlist <hostlist file> write_disjoint
  13  *  or:     pdsh -w <N hosts> write_disjoint
  14  *  or:     prun -n N [-N M] write_disjoint
  15  */
  16 #include <stdlib.h>
  17 #include <stdio.h>
  18 #include <sys/types.h>
  19 #include <sys/stat.h>
  20 #include <fcntl.h>
  21 #include <string.h>
  22 #include <errno.h>
  23 #include <unistd.h>
  24 #include <stdarg.h>
  25 #include "mpi.h"
  26
  27 #define CHUNK_MAX_SIZE 123456
  28
  29 void rprintf(int rank, int loop, const char *fmt, ...)
  30 {
  31         va_list       ap;
  32
  33         printf("rank %d, loop %d: ", rank, loop);
  34
  35         va_start(ap, fmt);
  36
  37         vprintf(fmt, ap);
  38
  39         MPI_Abort(MPI_COMM_WORLD, 1);
  40         exit(1); /* shouldn't be needed, but some MPI_Abort()s are broken */
  41 }
  42
  43 int main (int argc, char *argv[]) {
  44          int i, n, fd, chunk_size, file_size;
  45          int rank, noProcessors, done;
  46          off_t offset;
  47          char **chunk_buf;
  48          char *read_buf, c;
  49          struct stat stat_buf;
  50          ssize_t ret;
  51          char *filename = "/mnt/lustre/write_disjoint";
  52
  53         /* Parse command line options */
  54         while (1) {
  55                 c = getopt(argc, argv, "f:");
  56                 if (c == -1)
  57                         break;
  58
  59                 switch (c) {
  60                 case 'f':
  61                         filename = optarg;
  62                         break;
  63                 }
  64         }
  65
  66          MPI_Init(&argc, &argv);
  67          MPI_Comm_size(MPI_COMM_WORLD, &noProcessors);
  68          MPI_Comm_rank(MPI_COMM_WORLD, &rank);
  69
  70          chunk_buf = malloc(noProcessors * sizeof(chunk_buf[0]));
  71          for (i=0; i < noProcessors; i++) {
  72                 chunk_buf[i] = malloc(CHUNK_MAX_SIZE);
  73                 memset(chunk_buf[i], 'A'+ i, CHUNK_MAX_SIZE);
  74          }
  75          read_buf = malloc(noProcessors * CHUNK_MAX_SIZE);
  76
  77          if (rank == 0) {
  78                 fd = open(filename, O_WRONLY|O_CREAT|O_TRUNC, 0666);
  79                 if (fd < 0)
  80                         rprintf(rank, -1, "open() returned %s\n",
  81                                 strerror(errno));
  82          }
  83          MPI_Barrier(MPI_COMM_WORLD);
  84
  85          fd = open(filename, O_RDWR);
  86          if (fd < 0)
  87                  rprintf(rank, -1, "open() returned %s\n", strerror(errno));
  88
  89          for (n=0; n < 1000 ; n++) {
  90                  /* reset the environment */
  91                  if (rank == 0) {
  92                          ret = truncate(filename, 0);
  93                          if (ret != 0)
  94                                  rprintf(rank, n, "truncate() returned %s\n",
  95                                          strerror(errno) );
  96                  }
  97                  chunk_size = rand() % CHUNK_MAX_SIZE;
  98
  99                  if (n % 1000 == 0 && rank == 0)
 100                          printf("loop %d: chunk_size %d\n", n, chunk_size);
 101
 102                  MPI_Barrier(MPI_COMM_WORLD);
 103
 104                  /* Do the race */
 105                  offset = rank * chunk_size;
 106                  lseek(fd, offset, SEEK_SET);
 107
 108                  done = 0;
 109                  do {
 110                         ret = write(fd, chunk_buf[rank]+done, chunk_size-done);
 111                         if (ret < 0)
 112                                  rprintf(rank, n, "write() returned %s\n",
 113                                          strerror(errno));
 114                         done += ret;
 115                  } while (done != chunk_size);
 116
 117                  MPI_Barrier(MPI_COMM_WORLD);
 118
 119                  /* Check the result */
 120                  if (rank == 0) {
 121                          lseek(fd, 0, SEEK_SET);
 122
 123                          /* quick check */
 124                          stat(filename, &stat_buf);
 125                          file_size = stat_buf.st_size;
 126                          if (file_size != chunk_size * noProcessors)
 127                                   rprintf(rank, n, "invalid file size %d"
 128                                           " instead of %d\n", file_size,
 129                                           chunk_size * noProcessors);
 130
 131                         done = 0;
 132                         do {
 133                                 ret = read(fd, read_buf + done,
 134                                            (chunk_size * noProcessors) - done);
 135                                 if (ret < 0)
 136                                         rprintf(rank, n, "read returned %s\n",
 137                                                 strerror(errno));
 138
 139                                 done += ret;
 140                         } while (done != chunk_size * noProcessors);
 141
 142                         for (i = 0; i < noProcessors; i++) {
 143                                 char command[4096];
 144                                 int j;
 145                                 if (!memcmp(read_buf + (i * chunk_size),
 146                                            chunk_buf[i], chunk_size))
 147                                         continue;
 148
 149                                 printf("rank %d, loop %d: chunk %d corrupted "
 150                                        "with chunk_size %d\n", rank, n, i,
 151                                        chunk_size);
 152                                 printf("(ranks: page boundry, chunk boundry, "
 153                                        "page boundry)\n");
 154                                 for (j = 1 ; j < noProcessors; j++) {
 155                                         int b = j * chunk_size;
 156                                         printf("\t%c -> %c: %d %d %d\n",
 157                                                'A' + j - 1, 'A' + j,
 158                                                b & ~(4096-1), b,
 159                                                (b + 4096) & ~(4096-1));
 160                                 }
 161
 162                                 sprintf(command, "od -Ad -a %s", filename);
 163                                 system(command);
 164                                 MPI_Finalize();
 165                                 exit(1);
 166                         }
 167                 }
 168         }
 169
 170         printf("Finished after %d loops\n", n);
 171         MPI_Finalize();
 172         return 0;
 173 }