1 /* -*- mode: c; c-basic-offset: 8; indent-tabs-mode: nil; -*-
2 * vim:expandtab:shiftwidth=8:tabstop=8:
4 * Each loop does 3 things:
5 * - rank 0 truncates to 0
6 * - all ranks agree on a random chunk size
7 * - all ranks race to write their pattern to their chunk of the file
8 * - rank 0 makes sure that the resulting file size is ranks * chunk size
9 * - rank 0 makes sure that everyone's patterns went to the right place
11 * compile: mpicc -g -Wall -o write_disjoint write_disjoint.c
12 * run: mpirun -np N -machlist <hostlist file> write_disjoint
13 * or: pdsh -w <N hosts> write_disjoint
14 * or: prun -n N [-N M] write_disjoint
18 #include <sys/types.h>
27 #define CHUNK_MAX_SIZE 123456
29 void rprintf(int rank, int loop, const char *fmt, ...)
33 printf("rank %d, loop %d: ", rank, loop);
39 MPI_Abort(MPI_COMM_WORLD, -1); /* This will exit() according to man */
42 #define CHUNK_SIZE(n) chunk_size[(n) % 2]
44 int main (int argc, char *argv[]) {
46 unsigned long chunk_size[2];
47 int rank, noProcessors, done;
54 char *filename = "/mnt/lustre/write_disjoint";
57 error = MPI_Init(&argc, &argv);
58 if (error != MPI_SUCCESS)
59 rprintf(-1, -1, "MPI_Init failed: %d\n", error);
60 /* Parse command line options */
61 while ((c = getopt(argc, argv, "f:n:")) != EOF) {
67 numloops = strtoul(optarg, NULL, 0);
72 MPI_Comm_size(MPI_COMM_WORLD, &noProcessors);
73 MPI_Comm_rank(MPI_COMM_WORLD, &rank);
75 chunk_buf = malloc(noProcessors * sizeof(chunk_buf[0]));
76 for (i=0; i < noProcessors; i++) {
77 chunk_buf[i] = malloc(CHUNK_MAX_SIZE);
78 memset(chunk_buf[i], 'A'+ i, CHUNK_MAX_SIZE);
80 read_buf = malloc(noProcessors * CHUNK_MAX_SIZE);
83 fd = open(filename, O_WRONLY|O_CREAT|O_TRUNC, 0666);
85 rprintf(rank, -1, "open() returned %s\n",
88 MPI_Barrier(MPI_COMM_WORLD);
90 fd = open(filename, O_RDWR);
92 rprintf(rank, -1, "open() returned %s\n", strerror(errno));
94 for (n = 0; n < numloops; n++) {
95 /* reset the environment */
97 ret = truncate(filename, 0);
99 rprintf(rank, n, "truncate() returned %s\n",
102 CHUNK_SIZE(n) = rand() % CHUNK_MAX_SIZE;
104 if (n % 1000 == 0 && rank == 0)
105 printf("loop %d: chunk_size %lu\n", n, CHUNK_SIZE(n));
107 MPI_Barrier(MPI_COMM_WORLD);
110 offset = rank * CHUNK_SIZE(n);
111 lseek(fd, offset, SEEK_SET);
115 ret = write(fd, chunk_buf[rank] + done,
116 CHUNK_SIZE(n) - done);
118 rprintf(rank, n, "write() returned %s\n",
121 } while (done != CHUNK_SIZE(n));
123 MPI_Barrier(MPI_COMM_WORLD);
125 /* Check the result */
127 if (lseek(fd, 0, SEEK_SET) < 0)
128 rprintf(rank, n, "error seeking to 0: %s\n",
132 if (stat(filename, &stat_buf) < 0)
133 rprintf(rank, n, "error stating %s: %s\n",
134 filename, strerror(errno));
136 if (stat_buf.st_size != CHUNK_SIZE(n) * noProcessors)
137 rprintf(rank, n, "invalid file size %lu"
139 (unsigned long)stat_buf.st_size,
140 CHUNK_SIZE(n) * noProcessors);
144 ret = read(fd, read_buf + done,
145 CHUNK_SIZE(n) * noProcessors - done);
147 rprintf(rank, n, "read returned %s\n",
151 } while (done != CHUNK_SIZE(n) * noProcessors);
153 for (i = 0; i < noProcessors; i++) {
156 if (!memcmp(read_buf + (i * CHUNK_SIZE(n)),
157 chunk_buf[i], CHUNK_SIZE(n)))
160 /* print out previous chunk sizes */
162 printf("loop %d: chunk_size %lu\n",
163 n - 1, CHUNK_SIZE(n - 1));
165 printf("loop %d: chunk %d corrupted "
166 "with chunk_size %lu, page_size %d\n",
167 n, i, CHUNK_SIZE(n), getpagesize());
168 printf("ranks:\tpage boundry\tchunk boundry\t"
170 for (j = 1 ; j < noProcessors; j++) {
171 int b = j * CHUNK_SIZE(n);
172 printf("%c -> %c:\t%d\t%d\t%d\n",
173 'A' + j - 1, 'A' + j,
174 b & ~(getpagesize()-1), b,
175 (b + getpagesize()) &
179 sprintf(command, "od -Ad -a %s", filename);
181 rprintf(0, n, "data check error - exiting\n");
186 printf("Finished after %d loops\n", n);