Whamcloud - gitweb
LU-11171 tests: set parameters for racer_on_nfs
[fs/lustre-release.git] / lustre / tests / mpi / cascading_rw.c
1 /*
2  * GPL HEADER START
3  *
4  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
5  *
6  * This program is free software; you can redistribute it and/or modify
7  * it under the terms of the GNU General Public License version 2 only,
8  * as published by the Free Software Foundation.
9  *
10  * This program is distributed in the hope that it will be useful, but
11  * WITHOUT ANY WARRANTY; without even the implied warranty of
12  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
13  * General Public License version 2 for more details (a copy is included
14  * in the LICENSE file that accompanied this code).
15  *
16  * You should have received a copy of the GNU General Public License
17  * version 2 along with this program; If not, see
18  * http://www.gnu.org/licenses/gpl-2.0.html
19  *
20  * GPL HEADER END
21  */
22 /*
23  * Copyright (c) 2010, Oracle and/or its affiliates. All rights reserved.
24  * Use is subject to license terms.
25  */
26 /*
27  * This file is part of Lustre, http://www.lustre.org/
28  * Lustre is a trademark of Sun Microsystems, Inc.
29  *
30  * lustre/tests/mpi/cascading_rw.c
31  *
32  * Author: You Feng <youfeng@clusterfs.com>
33  */
34
35 #include <config.h>
36 #include <mpi.h>
37 #include <stdio.h>
38 #include <stdlib.h>
39 #include <string.h>
40 #include <sys/types.h>
41 #include <asm/types.h>
42 #include <sys/stat.h>
43 #include <fcntl.h>
44 #include <unistd.h>
45 #include <sys/ioctl.h>
46 #include <getopt.h>
47 #include <errno.h>
48
49 #include <lustre/lustreapi.h>
50 #include "lp_utils.h"
51 #ifndef _IOWR
52 # include <ioctl.h>
53 #endif
54
55
56 int rank = 0;
57 int size = 0;
58
59 char *testdir = NULL;
60
61 void rw_file(char *name, long stride, unsigned int seed)
62 {
63         char filename[MAX_FILENAME_LEN];
64         char errmsg[MAX_FILENAME_LEN+20];
65         char *buf, *o_buf;
66         struct lov_user_md lum = {0};
67         int fd, rc, i, bad = 0, root = 0;
68         long off;
69         long page_size = sysconf(_SC_PAGESIZE);
70
71         sprintf(filename, "%s/%s", testdir, name);
72
73         if (rank == 0) {
74                 remove_file_or_dir(filename);
75
76                 lum.lmm_magic = LOV_USER_MAGIC;
77                 lum.lmm_stripe_size = 0;
78                 lum.lmm_stripe_count = 0;
79                 lum.lmm_stripe_offset = -1;
80
81                 fd = open(filename, O_CREAT | O_RDWR | O_LOV_DELAY_CREATE,
82                           FILEMODE);
83                 if (fd == -1) {
84                         sprintf(errmsg, "open of file %s", filename);
85                         FAIL(errmsg);
86                 }
87
88                 rc = ioctl(fd, LL_IOC_LOV_SETSTRIPE, &lum);
89                 if (rc == -1) {
90                         sprintf(errmsg, "ioctl SETSTRIPE of file %s", filename);
91                         FAIL(errmsg);
92                 }
93
94                 if (close(fd) == -1) {
95                         sprintf(errmsg, "close of file %s", filename);
96                         FAIL(errmsg);
97                 }
98         }
99
100         MPI_Barrier(MPI_COMM_WORLD);
101
102         if (stride < 0) {
103                 if (rank == 0) {
104                         srandom(seed);
105                         while (stride < page_size/2) {
106                                 stride = random();
107                                 stride -= stride % 16;
108                                 if (stride < 0)
109                                         stride = -stride;
110                                 stride %= 2 * lum.lmm_stripe_size;
111                         }
112                 }
113
114                 MPI_Barrier(MPI_COMM_WORLD);
115
116                 MPI_Bcast(&stride, 1, MPI_LONG, root, MPI_COMM_WORLD);
117         }
118
119         MPI_Barrier(MPI_COMM_WORLD);
120
121         buf = (char *)malloc(stride);
122         if (buf == NULL) {
123                 sprintf(errmsg, "malloc of buf with size %ld", stride);
124                 FAIL(errmsg);
125         }
126
127         if (rank == 0) {
128                 fd = open(filename, O_RDWR);
129                 if (fd == -1) {
130                         sprintf(errmsg, "open of file %s", filename);
131                         FAIL(errmsg);
132                 }
133
134                 off = 0;
135                 fill_stride(buf, stride, 0, off);
136                 rc = write(fd, buf, stride);
137                 if (rc != stride) {
138                         sprintf(errmsg, "write of file %s return %d",
139                                 filename, rc);
140                         FAIL(errmsg);
141                 }
142                 off += stride;
143
144                 while (off < size * stride) {
145                         fill_stride(buf, stride, 0x8080808080808080ULL, off);
146                         rc = write(fd, buf, stride);
147                         if (rc != stride) {
148                                 sprintf(errmsg, "write of file %s return %d",
149                                         filename, rc);
150                                 FAIL(errmsg);
151                         }
152
153                         off += stride;
154                 }
155
156                 if (close(fd) == -1) {
157                         sprintf(errmsg, "close of file %s", filename);
158                         FAIL(errmsg);
159                 }
160         }
161
162         MPI_Barrier(MPI_COMM_WORLD);
163
164         o_buf = (char *)malloc(stride);
165         if (o_buf == NULL) {
166                 sprintf(errmsg, "malloc of o_buf with size %ld", stride);
167                 FAIL(errmsg);
168         }
169
170         fd = open(filename, O_RDWR);
171         if (fd == -1) {
172                 sprintf(errmsg, "open of file %s", filename);
173                 FAIL(errmsg);
174         }
175
176         off = 0;
177         for (i = 1; i < size; ++i) {
178                 if (rank == i) {
179                         rc = lseek(fd, off, SEEK_SET);
180                         if (rc != off) {
181                                 sprintf(errmsg, "lseek of file %s return %d",
182                                         filename, rc);
183                                 FAIL(errmsg);
184                         }
185
186                         rc = read(fd, buf, stride);
187                         if (rc != stride) {
188                                 if (rc > 0) {
189                                         fill_stride(o_buf, rc, i - 1, off);
190                                         dump_diff(o_buf, buf, rc, off);
191                                 }
192                                 sprintf(errmsg, "read of file %s return %d",
193                                         filename, rc);
194                                 FAIL(errmsg);
195                         }
196
197                         fill_stride(o_buf, stride, i - 1, off);
198                         if (memcmp(o_buf, buf, stride) != 0) {
199                                 dump_diff(o_buf, buf, stride, off);
200                                 errno = 0;
201                                 sprintf(errmsg, "Error: diff data read from %s",
202                                         filename);
203                                 FAIL(errmsg);
204                         }
205                 }
206
207                 off += stride;
208
209                 if (rank == i) {
210                         fill_stride(buf, stride, i, off);
211                         rc = write(fd, buf, stride);
212                         if (rc != stride) {
213                                 sprintf(errmsg, "write of file %s return %d",
214                                         filename, rc);
215                                 FAIL(errmsg);
216                         }
217                 }
218
219                 MPI_Barrier(MPI_COMM_WORLD);
220         }
221
222         if (close(fd) == -1) {
223                 sprintf(errmsg, "close of file %s", filename);
224                 FAIL(errmsg);
225         }
226
227         MPI_Barrier(MPI_COMM_WORLD);
228
229         if (rank == 0) {
230                 fd = open(filename, O_RDONLY);
231                 if (fd == -1) {
232                         sprintf(errmsg, "open of file %s", filename);
233                         FAIL(errmsg);
234                 }
235
236                 off = 0;
237                 for (i = 0; i < size; ++i) {
238                         rc = read(fd, buf, stride);
239                         if (rc != stride) {
240                                 if (rc > 0) {
241                                         fill_stride(o_buf, rc, i, off);
242                                         dump_diff(o_buf, buf, rc, off);
243                                 }
244                                 sprintf(errmsg, "read of file %s", filename);
245                                 FAIL(errmsg);
246                         }
247
248                         fill_stride(o_buf, stride, i, off);
249                         if (memcmp(o_buf, buf, stride) != 0) {
250                                 bad = 1;
251                                 dump_diff(o_buf, buf, stride, off);
252                         }
253                         off += stride;
254                 }
255                 if (bad == 1) {
256                         errno = 0;
257                         sprintf(errmsg, "Error: diff data read from %s", filename);
258                         FAIL(errmsg);
259                 }
260         }
261
262         MPI_Barrier(MPI_COMM_WORLD);
263         fprintf(stderr, "passed barrier 5\n");
264
265         free(buf);
266         free(o_buf);
267 }
268
269 void cascading_rw(long stride, unsigned int seed)
270 {
271         begin("setup");
272         end("setup");
273
274         begin("test");
275         rw_file("cascading_rw", stride, seed);
276         end("test");
277
278         begin("cleanup");
279         remove_file("cascading_rw");
280         end("cleanup");
281 }
282
283 void usage(char *proc)
284 {
285         int i;
286
287         if (rank == 0) {
288                 printf("Usage: %s [-h] -d <testdir> [-s \"1024\"]\n", proc);
289                 printf("           [-n \"13\"] [-e \"12345\"]\n"); 
290                 printf("           [-v] [-V #] [-g]\n");
291                 printf("\t-h: prints this help message\n");
292                 printf("\t-d: the directory in which the tests will run\n");
293                 printf("\t-s: process stride size\n");
294                 printf("\t-n: repeat test # times\n");
295                 printf("\t-n: random seed, used to re-create previous runs\n");
296                 printf("\t-v: increase the verbositly level by 1\n");
297                 printf("\t-V: select a specific verbosity level\n");
298                 printf("\t-g: debug mode\n");
299         }
300
301         MPI_Initialized(&i);
302         if (i) MPI_Finalize();
303         exit(0);
304 }
305
306 int main(int argc, char *argv[])
307 {
308         int i, iterations = 16, c;
309         long stride = -1;
310         unsigned int seed = 0;
311
312         /* Check for -h parameter before MPI_Init so the binary can be
313            called directly, without, for instance, mpirun */
314         for (i = 1; i < argc; ++i) {
315                 if (!strcmp(argv[i], "-h") || !strcmp(argv[i], "--help"))
316                         usage(argv[0]);
317         }
318
319         MPI_Init(&argc, &argv);
320         MPI_Comm_rank(MPI_COMM_WORLD, &rank);
321         MPI_Comm_size(MPI_COMM_WORLD, &size);
322
323                 /* Parse command line options */
324         while (1) {
325                 c = getopt(argc, argv, "d:e:ghn:s:vV:");
326                 if (c == -1)
327                         break;
328
329                 switch (c) {
330                 case 'd':
331                         testdir = optarg;
332                         break;
333                 case 'e':
334                         seed = (unsigned int)atoi(optarg);
335                         break;
336                 case 'g':
337                         debug = 1;
338                         break;
339                 case 'h':
340                         usage(argv[0]);
341                         break;
342                 case 'n':
343                         iterations = atoi(optarg);
344                         break;
345                 case 's':
346                         stride = atol(optarg);
347                         break;
348                 case 'v':
349                         verbose += 1;
350                         break;
351                 case 'V':
352                         verbose = atoi(optarg);
353                         break;
354                 }
355         }
356
357         if (rank == 0)
358                 printf("%s is running with %d process(es) %s\n",
359                        argv[0], size, debug ? "in DEBUG mode" : "\b\b");
360
361         if (size < 2) {
362                 fprintf(stderr,
363                         "There should be at least 3 process to run the test!\n");
364                 MPI_Abort(MPI_COMM_WORLD, 2);
365         }
366
367         if (testdir == NULL && rank == 0) {
368                 fprintf(stderr, "Please specify a test directory! (\"%s -h\" for help)\n",
369                        argv[0]);
370                 MPI_Abort(MPI_COMM_WORLD, 2);
371         }
372
373         lp_gethostname();
374
375         for (i = 0; i < iterations; ++i) {
376                 if (rank == 0)
377                         printf("%s: Running test #%s(iter %d)\n",
378                                timestamp(), argv[0], i);
379
380                 cascading_rw(stride, seed);
381                 MPI_Barrier(MPI_COMM_WORLD);
382         }
383
384         if (rank == 0)
385                 printf("%s: All tests passed!\n", timestamp());
386         MPI_Finalize();
387         return 0;
388 }