Whamcloud - gitweb
c9ffae697ab9f8457995f5488ec8ddca91b1463a
[fs/lustre-release.git] / lustre / tests / write_disjoint.c
1 /* -*- mode: c; c-basic-offset: 8; indent-tabs-mode: nil; -*-
2  * vim:expandtab:shiftwidth=8:tabstop=8:
3  *
4  * Each loop does 3 things:
5  *   - rank 0 truncates to 0
6  *   - all ranks agree on a random chunk size
7  *   - all ranks race to write their pattern to their chunk of the file
8  *   - rank 0 makes sure that the resulting file size is ranks * chunk size
9  *   - rank 0 makes sure that everyone's patterns went to the right place
10  *
11  * compile: mpicc -g -Wall -o write_disjoint write_disjoint.c
12  * run:     mpirun -np N -machlist <hostlist file> write_disjoint
13  *  or:     pdsh -w <N hosts> write_disjoint 
14  *  or:     prun -n N [-N M] write_disjoint
15  */
16 #include <stdlib.h>
17 #include <stdio.h>
18 #include <sys/types.h>
19 #include <sys/stat.h>
20 #include <fcntl.h>
21 #include <string.h>
22 #include <errno.h>
23 #include <unistd.h>
24 #include <stdarg.h>
25 #include "mpi.h"
26
27 #define CHUNK_MAX_SIZE 123456
28
29 void rprintf(int rank, int loop, const char *fmt, ...)
30 {
31         va_list       ap;
32  
33         printf("rank %d, loop %d: ", rank, loop);
34  
35         va_start(ap, fmt);
36  
37         vprintf(fmt, ap);
38  
39         MPI_Abort(MPI_COMM_WORLD, 1);
40 }
41
42 int main (int argc, char *argv[]) {
43          int i, n, fd, chunk_size, file_size;
44          int rank, noProcessors, done;
45          off_t offset;
46          char **chunk_buf;
47          char *read_buf, c;
48          struct stat stat_buf;
49          ssize_t ret;
50          char *filename = "/mnt/lustre/write_disjoint";
51
52         /* Parse command line options */
53         while (1) {
54                 c = getopt(argc, argv, "f:");
55                 if (c == -1)
56                         break;
57
58                 switch (c) {
59                 case 'f':
60                         filename = optarg;
61                         break;
62                 }
63         }
64
65          MPI_Init(&argc, &argv);
66          MPI_Comm_size(MPI_COMM_WORLD, &noProcessors);
67          MPI_Comm_rank(MPI_COMM_WORLD, &rank);
68                          
69          chunk_buf = malloc(noProcessors * sizeof(chunk_buf[0]));
70          for (i=0; i < noProcessors; i++) {
71                 chunk_buf[i] = malloc(CHUNK_MAX_SIZE);
72                 memset(chunk_buf[i], 'A'+ i, CHUNK_MAX_SIZE);
73          }
74          read_buf = malloc(noProcessors * CHUNK_MAX_SIZE);
75          
76          if (rank == 0) {
77                 fd = open(filename, O_WRONLY|O_CREAT|O_TRUNC, 0666);
78                 if (fd < 0) 
79                         rprintf(rank, -1, "open() returned %s\n", 
80                                 strerror(errno));
81          }
82          MPI_Barrier(MPI_COMM_WORLD);
83
84          fd = open(filename, O_RDWR);
85          if (fd < 0)
86                  rprintf(rank, -1, "open() returned %s\n", strerror(errno));
87          
88          for (n=0; n < 1000 ; n++) {
89                  /* reset the environment */
90                  if (rank == 0) {
91                          ret = truncate(filename, 0);
92                          if (ret != 0)
93                                  rprintf(rank, n, "truncate() returned %s\n", 
94                                          strerror(errno) );
95                  }
96                  chunk_size = rand() % CHUNK_MAX_SIZE;
97
98                  if (n % 1000 == 0 && rank == 0)
99                          printf("loop %d: chunk_size %d\n", n, chunk_size);
100
101                  MPI_Barrier(MPI_COMM_WORLD);
102                  
103                  /* Do the race */
104                  offset = rank * chunk_size;
105                  lseek(fd, offset, SEEK_SET);
106
107                  done = 0;
108                  do {
109                         ret = write(fd, chunk_buf[rank]+done, chunk_size-done);
110                         if (ret < 0) 
111                                  rprintf(rank, n, "write() returned %s\n", 
112                                          strerror(errno));
113                         done += ret;
114                  } while (done != chunk_size);
115
116                  MPI_Barrier(MPI_COMM_WORLD);
117
118                  /* Check the result */
119                  if (rank == 0) {
120                          lseek(fd, 0, SEEK_SET);
121                          
122                          /* quick check */
123                          stat(filename, &stat_buf);
124                          file_size = stat_buf.st_size;
125                          if (file_size != chunk_size * noProcessors)
126                                   rprintf(rank, n, "invalid file size %d"
127                                           " instead of %d\n", file_size, 
128                                           chunk_size * noProcessors);
129
130                         done = 0;
131                         do {
132                                 ret = read(fd, read_buf + done, 
133                                            (chunk_size * noProcessors) - done);
134                                 if (ret < 0) 
135                                         rprintf(rank, n, "read returned %s\n",
136                                                 strerror(errno));
137
138                                 done += ret;
139                         } while (done != chunk_size * noProcessors);
140
141                         for (i = 0; i < noProcessors; i++) {
142                                 char command[4096]; 
143                                 int j;
144                                 if (!memcmp(read_buf + (i * chunk_size), 
145                                            chunk_buf[i], chunk_size))
146                                         continue;
147
148                                 printf("rank %d, loop %d: chunk %d corrupted "
149                                        "with chunk_size %d\n", rank, n, i, 
150                                        chunk_size);
151                                 printf("(ranks: page boundry, chunk boundry, "
152                                        "page boundry)\n");
153                                 for (j = 1 ; j < noProcessors; j++) {
154                                         int b = j * chunk_size;
155                                         printf("\t%c -> %c: %d %d %d\n", 
156                                                'A' + j - 1, 'A' + j, 
157                                                b & ~(4096-1), b, 
158                                                (b + 4096) & ~(4096-1));
159                                 }
160
161                                 sprintf(command, "od -Ad -a %s", filename);
162                                 system(command);
163                                 MPI_Finalize();
164                                 exit(1);
165                         }
166                 }
167         }
168
169         printf("Finished after %d loops\n", n);
170         MPI_Finalize();
171         return 0;
172 }