4 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
6 * This program is free software; you can redistribute it and/or modify
7 * it under the terms of the GNU General Public License version 2 only,
8 * as published by the Free Software Foundation.
10 * This program is distributed in the hope that it will be useful, but
11 * WITHOUT ANY WARRANTY; without even the implied warranty of
12 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
13 * General Public License version 2 for more details (a copy is included
14 * in the LICENSE file that accompanied this code).
16 * You should have received a copy of the GNU General Public License
17 * version 2 along with this program; If not, see
18 * http://www.gnu.org/licenses/gpl-2.0.html
23 * Copyright (c) 2003, 2010, Oracle and/or its affiliates. All rights reserved.
24 * Use is subject to license terms.
26 * Copyright (c) 2012, Intel Corporation.
29 * This file is part of Lustre, http://www.lustre.org/
30 * Lustre is a trademark of Sun Microsystems, Inc.
32 * lustre/tests/write_disjoint.c
34 * Each loop does 3 things:
35 * - rank 0 truncates to 0
36 * - all ranks agree on a random chunk size
37 * - all ranks race to write their pattern to their chunk of the file
38 * - rank 0 makes sure that the resulting file size is ranks * chunk size
39 * - rank 0 makes sure that everyone's patterns went to the right place
41 * compile: mpicc -g -Wall -o write_disjoint write_disjoint.c
42 * run: mpirun -np N -machlist <hostlist file> write_disjoint
43 * or: pdsh -w <N hosts> write_disjoint
44 * or: prun -n N [-N M] write_disjoint
49 #include <sys/types.h>
59 /* Chosen arbitrarily. Actually running this large will take a long time.*/
60 #define CHUNK_MAX_SIZE (1024*1024*16)
62 void rprintf(int rank, int loop, const char *fmt, ...)
66 printf("rank %d, loop %d: ", rank, loop);
72 MPI_Abort(MPI_COMM_WORLD, -1); /* This will exit() according to man */
75 #define CHUNK_SIZE(n) chunk_size[(n) % 2]
77 int main (int argc, char *argv[]) {
79 unsigned long chunk_size[2];
80 int rank, noProcessors, done;
87 char *filename = "/mnt/lustre/write_disjoint";
89 int max_size = CHUNK_MAX_SIZE;
91 unsigned int seed = 0;
92 int seed_provided = 0;
94 error = MPI_Init(&argc, &argv);
95 if (error != MPI_SUCCESS)
96 rprintf(-1, -1, "MPI_Init failed: %d\n", error);
97 /* Parse command line options */
98 while ((c = getopt(argc, argv, "f:n:m:s:")) != EOF) {
105 numloops = strtoul(optarg, NULL, 0);
108 max_size = strtoul(optarg, NULL, 0);
109 if (max_size > CHUNK_MAX_SIZE)
110 rprintf(-1, -1, "Chunk size larger than %d.\n",
114 seed = strtoul(optarg, NULL, 0);
120 MPI_Comm_size(MPI_COMM_WORLD, &noProcessors);
121 MPI_Comm_rank(MPI_COMM_WORLD, &rank);
123 chunk_buf = malloc(noProcessors * sizeof(chunk_buf[0]));
124 for (i=0; i < noProcessors; i++) {
125 chunk_buf[i] = malloc(max_size);
126 memset(chunk_buf[i], 'A' + i, max_size);
128 read_buf = malloc(noProcessors * max_size);
131 fd = open(filename, O_WRONLY|O_CREAT|O_TRUNC, 0666);
133 rprintf(rank, -1, "open() returned %s\n",
136 MPI_Barrier(MPI_COMM_WORLD);
138 fd = open(filename, O_RDWR);
140 rprintf(rank, -1, "open() returned %s\n", strerror(errno));
144 seed = (unsigned int) time(NULL);
145 printf("random seed: %d\n", seed);
150 for (n = 0; n < numloops; n++) {
151 /* reset the environment */
153 ret = truncate(filename, 0);
155 rprintf(rank, n, "truncate() returned %s\n",
160 MPI_Bcast(&random, 1, MPI_INT, 0, MPI_COMM_WORLD);
161 CHUNK_SIZE(n) = random % max_size;
163 if (n % 1000 == 0 && rank == 0)
164 printf("loop %d: chunk_size %lu\n", n, CHUNK_SIZE(n));
166 if (stat(filename, &stat_buf) < 0)
167 rprintf(rank, n, "error stating %s: %s\n",
168 filename, strerror(errno));
170 if (stat_buf.st_size != 0)
171 rprintf(rank, n, "filesize = %lu. "
172 "Should be zero after truncate\n",
175 MPI_Barrier(MPI_COMM_WORLD);
178 offset = rank * CHUNK_SIZE(n);
179 lseek(fd, offset, SEEK_SET);
183 ret = write(fd, chunk_buf[rank] + done,
184 CHUNK_SIZE(n) - done);
185 if (ret < 0 && errno != EINTR)
186 rprintf(rank, n, "write() returned %s\n",
190 } while (done != CHUNK_SIZE(n));
192 MPI_Barrier(MPI_COMM_WORLD);
194 /* Check the result */
195 if (stat(filename, &stat_buf) < 0)
196 rprintf(rank, n, "error stating %s: %s\n",
197 filename, strerror(errno));
199 if (stat_buf.st_size != CHUNK_SIZE(n) * noProcessors) {
201 printf("loop %d: chunk_size %lu, "
202 "file size was %lu\n",
203 n - 1, CHUNK_SIZE(n - 1),
204 CHUNK_SIZE(n - 1) *noProcessors);
205 rprintf(rank, n, "invalid file size %lu"
206 " instead of %lu = %lu * %u\n",
207 (unsigned long)stat_buf.st_size,
208 CHUNK_SIZE(n) * noProcessors,
209 CHUNK_SIZE(n), noProcessors);
213 if (lseek(fd, 0, SEEK_SET) < 0)
214 rprintf(rank, n, "error seeking to 0: %s\n",
219 ret = read(fd, read_buf + done,
220 CHUNK_SIZE(n) * noProcessors - done);
222 rprintf(rank, n, "read returned %s\n",
226 } while (done != CHUNK_SIZE(n) * noProcessors);
228 for (i = 0; i < noProcessors; i++) {
232 if (!memcmp(read_buf + (i * CHUNK_SIZE(n)),
233 chunk_buf[i], CHUNK_SIZE(n)))
236 /* print out previous chunk sizes */
238 printf("loop %d: chunk_size %lu\n",
239 n - 1, CHUNK_SIZE(n - 1));
241 printf("loop %d: chunk %d corrupted "
242 "with chunk_size %lu, page_size %d\n",
243 n, i, CHUNK_SIZE(n), getpagesize());
244 printf("ranks:\tpage boundry\tchunk boundry\t"
246 for (j = 1 ; j < noProcessors; j++) {
247 int b = j * CHUNK_SIZE(n);
248 printf("%c -> %c:\t%d\t%d\t%d\n",
249 'A' + j - 1, 'A' + j,
250 b & ~(getpagesize()-1), b,
251 (b + getpagesize()) &
255 sprintf(command, "od -Ad -a %s", filename);
256 ret = system(command);
257 rprintf(0, n, "data check error - exiting\n");
260 MPI_Barrier(MPI_COMM_WORLD);
263 printf("Finished after %d loops\n", n);