Whamcloud - gitweb
LU-14264 tests: make PARALLEL available to all suites
[fs/lustre-release.git] / lustre / tests / mpi / write_disjoint.c
1 /*
2  * GPL HEADER START
3  *
4  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
5  *
6  * This program is free software; you can redistribute it and/or modify
7  * it under the terms of the GNU General Public License version 2 only,
8  * as published by the Free Software Foundation.
9  *
10  * This program is distributed in the hope that it will be useful, but
11  * WITHOUT ANY WARRANTY; without even the implied warranty of
12  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
13  * General Public License version 2 for more details (a copy is included
14  * in the LICENSE file that accompanied this code).
15  *
16  * You should have received a copy of the GNU General Public License
17  * version 2 along with this program; If not, see
18  * http://www.gnu.org/licenses/gpl-2.0.html
19  *
20  * GPL HEADER END
21  */
22 /*
23  * Copyright (c) 2003, 2010, Oracle and/or its affiliates. All rights reserved.
24  * Use is subject to license terms.
25  *
26  * Copyright (c) 2012, Intel Corporation.
27  */
28 /*
29  * This file is part of Lustre, http://www.lustre.org/
30  *
31  * lustre/tests/write_disjoint.c
32  *
33  * Each loop does 3 things:
34  *   - rank 0 truncates to 0
35  *   - all ranks agree on a random chunk size
36  *   - all ranks race to write their pattern to their chunk of the file
37  *   - rank 0 makes sure that the resulting file size is ranks * chunk size
38  *   - rank 0 makes sure that everyone's patterns went to the right place
39  *
40  * compile: mpicc -g -Wall -o write_disjoint write_disjoint.c
41  * run:     mpirun -np N -machlist <hostlist file> write_disjoint
42  *  or:     pdsh -w <N hosts> write_disjoint
43  *  or:     prun -n N [-N M] write_disjoint
44  */
45
46 #include <stdlib.h>
47 #include <stdio.h>
48 #include <sys/types.h>
49 #include <sys/stat.h>
50 #include <fcntl.h>
51 #include <string.h>
52 #include <errno.h>
53 #include <unistd.h>
54 #include <stdarg.h>
55 #include <time.h>
56 #include "mpi.h"
57
58 /* Chosen arbitrarily.  Actually running this large will take a long time.*/
59 #define CHUNK_MAX_SIZE (1024 * 1024 * 16)
60
61 void rprintf(int rank, int loop, const char *fmt, ...)
62 {
63         va_list ap;
64
65         printf("rank %d, loop %d: ", rank, loop);
66
67         va_start(ap, fmt);
68
69         vprintf(fmt, ap);
70
71         MPI_Abort(MPI_COMM_WORLD, -1); /* This will exit() according to man */
72 }
73
74 #define CHUNK_SIZE(n) chunk_size[(n) % 2]
75
76 int main(int argc, char *argv[])
77 {
78         int i, n, fd, c;
79         unsigned long chunk_size[2];
80         int rank, noProcessors, done;
81         int error;
82         off_t offset;
83         char **chunk_buf;
84         char *read_buf;
85         struct stat stat_buf;
86         ssize_t ret;
87         char *filename = "/mnt/lustre/write_disjoint";
88         int numloops = 1000;
89         int max_size = CHUNK_MAX_SIZE;
90         int random = 0;
91         unsigned int seed = 0;
92         int seed_provided = 0;
93
94         error = MPI_Init(&argc, &argv);
95         if (error != MPI_SUCCESS)
96                 rprintf(-1, -1, "MPI_Init failed: %d\n", error);
97         /* Parse command line options */
98         while ((c = getopt(argc, argv, "f:n:m:s:")) != EOF) {
99                 errno = 0;
100                 switch (c) {
101                 case 'f':
102                         filename = optarg;
103                         break;
104                 case 'n':
105                         numloops = strtoul(optarg, NULL, 0);
106                         break;
107                 case 'm':
108                         max_size = strtoul(optarg, NULL, 0);
109                         if (max_size > CHUNK_MAX_SIZE)
110                                 rprintf(-1, -1, "Chunk size larger than %d.\n",
111                                         CHUNK_MAX_SIZE);
112                         break;
113                 case 's':
114                         seed = strtoul(optarg, NULL, 0);
115                         seed_provided = 1;
116                         break;
117                 }
118         }
119
120         MPI_Comm_size(MPI_COMM_WORLD, &noProcessors);
121         MPI_Comm_rank(MPI_COMM_WORLD, &rank);
122
123         chunk_buf = malloc(noProcessors * sizeof(chunk_buf[0]));
124         for (i = 0; i < noProcessors; i++) {
125                 chunk_buf[i] = malloc(max_size);
126                 memset(chunk_buf[i], 'A' + i, max_size);
127         }
128         read_buf = malloc(noProcessors * max_size);
129
130         if (rank == 0) {
131                 fd = open(filename, O_WRONLY | O_CREAT | O_TRUNC, 0666);
132                 if (fd < 0)
133                         rprintf(rank, -1, "open() returned %s\n",
134                                 strerror(errno));
135         }
136         MPI_Barrier(MPI_COMM_WORLD);
137
138         fd = open(filename, O_RDWR);
139         if (fd < 0)
140                 rprintf(rank, -1, "open() returned %s\n", strerror(errno));
141
142         if (rank == 0) {
143                 if (!seed_provided)
144                         seed = (unsigned int)time(NULL);
145                 printf("random seed: %d\n", seed);
146                 srand(seed);
147         }
148
149         for (n = 0; n < numloops; n++) {
150                 /* reset the environment */
151                 if (rank == 0) {
152                         ret = truncate(filename, 0);
153                         if (ret != 0)
154                                 rprintf(rank, n, "truncate() returned %s\n",
155                                         strerror(errno));
156
157                         random = rand();
158                 }
159                 MPI_Bcast(&random, 1, MPI_INT, 0, MPI_COMM_WORLD);
160                 CHUNK_SIZE(n) = random % max_size;
161
162                 if (n % 1000 == 0 && rank == 0)
163                         printf("loop %d: chunk_size %lu\n", n, CHUNK_SIZE(n));
164
165                 if (stat(filename, &stat_buf) < 0)
166                         rprintf(rank, n, "error stating %s: %s\n",
167                                 filename, strerror(errno));
168
169                 if (stat_buf.st_size != 0)
170                         rprintf(rank, n,
171                                 "filesize = %lu. Should be zero after truncate\n",
172                                 stat_buf.st_size);
173
174                 MPI_Barrier(MPI_COMM_WORLD);
175
176                 /* Do the race */
177                 offset = rank * CHUNK_SIZE(n);
178                 lseek(fd, offset, SEEK_SET);
179
180                 done = 0;
181                 do {
182                         ret = write(fd, chunk_buf[rank] + done,
183                                     CHUNK_SIZE(n) - done);
184                         if (ret < 0 && errno != EINTR)
185                                 rprintf(rank, n, "write() returned %s\n",
186                                         strerror(errno));
187                         if (ret > 0)
188                                 done += ret;
189                 } while (done != CHUNK_SIZE(n));
190
191                 MPI_Barrier(MPI_COMM_WORLD);
192
193                 /* Check the result */
194                 if (stat(filename, &stat_buf) < 0)
195                         rprintf(rank, n, "error stating %s: %s\n",
196                                 filename, strerror(errno));
197
198                 if (stat_buf.st_size != CHUNK_SIZE(n) * noProcessors) {
199                         if (n > 0)
200                                 printf("loop %d: chunk_size %lu, file size was %lu\n",
201                                        n - 1, CHUNK_SIZE(n - 1),
202                                        CHUNK_SIZE(n - 1) * noProcessors);
203                         rprintf(rank, n,
204                                 "invalid file size %lu instead of %lu = %lu * %u\n",
205                                 (unsigned long)stat_buf.st_size,
206                                 CHUNK_SIZE(n) * noProcessors,
207                                 CHUNK_SIZE(n), noProcessors);
208                 }
209
210                 if (rank == 0) {
211                         if (lseek(fd, 0, SEEK_SET) < 0)
212                                 rprintf(rank, n, "error seeking to 0: %s\n",
213                                         strerror(errno));
214
215                         done = 0;
216                         do {
217                                 ret = read(fd, read_buf + done,
218                                            CHUNK_SIZE(n) * noProcessors - done);
219                                 if (ret < 0)
220                                         rprintf(rank, n, "read returned %s\n",
221                                                 strerror(errno));
222
223                                 done += ret;
224                         } while (done != CHUNK_SIZE(n) * noProcessors);
225
226                         for (i = 0; i < noProcessors; i++) {
227                                 char command[4096];
228                                 int j;
229
230                                 if (!memcmp(read_buf + (i * CHUNK_SIZE(n)),
231                                             chunk_buf[i], CHUNK_SIZE(n)))
232                                         continue;
233
234                                 /* print out previous chunk sizes */
235                                 if (n > 0)
236                                         printf("loop %d: chunk_size %lu\n",
237                                                n - 1, CHUNK_SIZE(n - 1));
238
239                                 printf("loop %d: chunk %d corrupted with chunk_size %lu, page_size %d\n",
240                                        n, i, CHUNK_SIZE(n), getpagesize());
241                                 printf("ranks:\tpage boundry\tchunk boundry\tpage boundry\n");
242                                 for (j = 1 ; j < noProcessors; j++) {
243                                         int b = j * CHUNK_SIZE(n);
244
245                                         printf("%c -> %c:\t%d\t%d\t%d\n",
246                                                'A' + j - 1, 'A' + j,
247                                                b & ~(getpagesize() - 1), b,
248                                                (b + getpagesize()) &
249                                                ~(getpagesize() - 1));
250                                 }
251
252                                 sprintf(command, "od -Ad -a %s", filename);
253                                 ret = system(command);
254                                 rprintf(0, n, "data check error - exiting\n");
255                         }
256                 }
257                 MPI_Barrier(MPI_COMM_WORLD);
258         }
259
260         printf("Finished after %d loops\n", n);
261         MPI_Finalize();
262         return 0;
263 }