1 /* -*- mode: c; c-basic-offset: 8; indent-tabs-mode: nil; -*-
2 * vim:expandtab:shiftwidth=8:tabstop=8:
4 * Each loop does 3 things:
5 * - rank 0 truncates to 0
6 * - all ranks agree on a random chunk size
7 * - all ranks race to write their pattern to their chunk of the file
8 * - rank 0 makes sure that the resulting file size is ranks * chunk size
9 * - rank 0 makes sure that everyone's patterns went to the right place
11 * compile: mpicc -g -Wall -o write_disjoint write_disjoint.c
12 * run: mpirun -np N -machlist <hostlist file> write_disjoint
13 * or: pdsh -w <N hosts> write_disjoint
14 * or: prun -n N [-N M] write_disjoint
18 #include <sys/types.h>
27 #define CHUNK_MAX_SIZE 123456
29 void rprintf(int rank, int loop, const char *fmt, ...)
33 printf("rank %d, loop %d: ", rank, loop);
39 MPI_Abort(MPI_COMM_WORLD, 1);
42 int main (int argc, char *argv[]) {
43 int i, n, fd, chunk_size, file_size;
44 int rank, noProcessors, done;
50 char *filename = "/mnt/lustre/write_disjoint";
52 /* Parse command line options */
54 c = getopt(argc, argv, "f:");
65 MPI_Init(&argc, &argv);
66 MPI_Comm_size(MPI_COMM_WORLD, &noProcessors);
67 MPI_Comm_rank(MPI_COMM_WORLD, &rank);
69 chunk_buf = malloc(noProcessors * sizeof(chunk_buf[0]));
70 for (i=0; i < noProcessors; i++) {
71 chunk_buf[i] = malloc(CHUNK_MAX_SIZE);
72 memset(chunk_buf[i], 'A'+ i, CHUNK_MAX_SIZE);
74 read_buf = malloc(noProcessors * CHUNK_MAX_SIZE);
77 fd = open(filename, O_WRONLY|O_CREAT|O_TRUNC, 0666);
79 rprintf(rank, -1, "open() returned %s\n",
82 MPI_Barrier(MPI_COMM_WORLD);
84 fd = open(filename, O_RDWR);
86 rprintf(rank, -1, "open() returned %s\n", strerror(errno));
88 for (n=0; n < 1000 ; n++) {
89 /* reset the environment */
91 ret = truncate(filename, 0);
93 rprintf(rank, n, "truncate() returned %s\n",
96 chunk_size = rand() % CHUNK_MAX_SIZE;
98 if (n % 1000 == 0 && rank == 0)
99 printf("loop %d: chunk_size %d\n", n, chunk_size);
101 MPI_Barrier(MPI_COMM_WORLD);
104 offset = rank * chunk_size;
105 lseek(fd, offset, SEEK_SET);
109 ret = write(fd, chunk_buf[rank]+done, chunk_size-done);
111 rprintf(rank, n, "write() returned %s\n",
114 } while (done != chunk_size);
116 MPI_Barrier(MPI_COMM_WORLD);
118 /* Check the result */
120 lseek(fd, 0, SEEK_SET);
123 stat(filename, &stat_buf);
124 file_size = stat_buf.st_size;
125 if (file_size != chunk_size * noProcessors)
126 rprintf(rank, n, "invalid file size %d"
127 " instead of %d\n", file_size,
128 chunk_size * noProcessors);
132 ret = read(fd, read_buf + done,
133 (chunk_size * noProcessors) - done);
135 rprintf(rank, n, "read returned %s\n",
139 } while (done != chunk_size * noProcessors);
141 for (i = 0; i < noProcessors; i++) {
144 if (!memcmp(read_buf + (i * chunk_size),
145 chunk_buf[i], chunk_size))
148 printf("rank %d, loop %d: chunk %d corrupted "
149 "with chunk_size %d\n", rank, n, i,
151 printf("(ranks: page boundry, chunk boundry, "
153 for (j = 1 ; j < noProcessors; j++) {
154 int b = j * chunk_size;
155 printf("\t%c -> %c: %d %d %d\n",
156 'A' + j - 1, 'A' + j,
158 (b + 4096) & ~(4096-1));
161 sprintf(command, "od -Ad -a %s", filename);
169 printf("Finished after %d loops\n", n);