Whamcloud - gitweb
08b48e8a76e7ef61500f7720cb274adeddf7b5fc
[fs/lustre-release.git] / lustre / tests / mpi / cascading_rw.c
1 /* -*- mode: c; c-basic-offset: 8; indent-tabs-mode: nil; -*-
2  * vim:expandtab:shiftwidth=8:tabstop=8:
3  *
4  * GPL HEADER START
5  *
6  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
7  *
8  * This program is free software; you can redistribute it and/or modify
9  * it under the terms of the GNU General Public License version 2 only,
10  * as published by the Free Software Foundation.
11  *
12  * This program is distributed in the hope that it will be useful, but
13  * WITHOUT ANY WARRANTY; without even the implied warranty of
14  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
15  * General Public License version 2 for more details (a copy is included
16  * in the LICENSE file that accompanied this code).
17  *
18  * You should have received a copy of the GNU General Public License
19  * version 2 along with this program; If not, see
20  * http://www.sun.com/software/products/lustre/docs/GPLv2.pdf
21  *
22  * Please contact Sun Microsystems, Inc., 4150 Network Circle, Santa Clara,
23  * CA 95054 USA or visit www.sun.com if you need additional information or
24  * have any questions.
25  *
26  * GPL HEADER END
27  */
28 /*
29  * Copyright (c) 2010, Oracle and/or its affiliates. All rights reserved.
30  * Use is subject to license terms.
31  */
32 /*
33  * This file is part of Lustre, http://www.lustre.org/
34  * Lustre is a trademark of Sun Microsystems, Inc.
35  *
36  * lustre/tests/mpi/cascading_rw.c
37  *
38  * Author: You Feng <youfeng@clusterfs.com>
39  */
40
41 #include <config.h>
42 #include <mpi.h>
43 #include <stdio.h>
44 #include <stdlib.h>
45 #include <string.h>
46 #include <sys/types.h>
47 #include <asm/types.h>
48 #include <sys/stat.h>
49 #include <fcntl.h>
50 #include <unistd.h>
51 #include <sys/ioctl.h>
52 #include <getopt.h>
53 #include <errno.h>
54
55 #include <libcfs/libcfs.h>
56 #include "lp_utils.h"
57 #ifndef _IOWR
58 # include <ioctl.h>
59 #endif
60
61 #include <lustre/liblustreapi.h>
62
63 int rank = 0;
64 int size = 0;
65
66 char *testdir = NULL;
67
68 void rw_file(char *name, long stride, unsigned int seed)
69 {
70         char filename[MAX_FILENAME_LEN];
71         char errmsg[MAX_FILENAME_LEN+20];
72         char *buf, *o_buf;
73         struct lov_user_md lum = {0};
74         int fd, rc, i, bad = 0, root = 0;
75         long off;
76         long page_size = sysconf(_SC_PAGESIZE);
77
78         sprintf(filename, "%s/%s", testdir, name);
79
80         if (rank == 0) {
81                 remove_file_or_dir(filename);
82
83                 lum.lmm_magic = LOV_USER_MAGIC;
84                 lum.lmm_stripe_size = 0;
85                 lum.lmm_stripe_count = 0;
86                 lum.lmm_stripe_offset = -1;
87
88                 fd = open(filename, O_CREAT | O_RDWR | O_LOV_DELAY_CREATE,
89                           FILEMODE);
90                 if (fd == -1) {
91                         sprintf(errmsg, "open of file %s", filename);
92                         FAIL(errmsg);
93                 }
94
95                 rc = ioctl(fd, LL_IOC_LOV_SETSTRIPE, &lum);
96                 if (rc == -1) {
97                         sprintf(errmsg, "ioctl SETSTRIPE of file %s", filename);
98                         FAIL(errmsg);
99                 }
100
101                 if (close(fd) == -1) {
102                         sprintf(errmsg, "close of file %s", filename);
103                         FAIL(errmsg);
104                 }
105         }
106
107         MPI_Barrier(MPI_COMM_WORLD);
108
109         if (stride < 0) {
110                 if (rank == 0) {
111                         srandom(seed);
112                         while (stride < page_size/2) {
113                                 stride = random();
114                                 stride -= stride % 16;
115                                 if (stride < 0)
116                                         stride = -stride;
117                                 stride %= 2 * lum.lmm_stripe_size;
118                         }
119                 }
120
121                 MPI_Barrier(MPI_COMM_WORLD);
122
123                 MPI_Bcast(&stride, 1, MPI_LONG, root, MPI_COMM_WORLD);
124         }
125
126         MPI_Barrier(MPI_COMM_WORLD);
127
128         buf = (char *)malloc(stride);
129         if (buf == NULL) {
130                 sprintf(errmsg, "malloc of buf with size %ld", stride);
131                 FAIL(errmsg);
132         }
133
134         if (rank == 0) {
135                 fd = open(filename, O_RDWR);
136                 if (fd == -1) {
137                         sprintf(errmsg, "open of file %s", filename);
138                         FAIL(errmsg);
139                 }
140
141                 off = 0;
142                 fill_stride(buf, stride, 0, off);
143                 rc = write(fd, buf, stride);
144                 if (rc != stride) {
145                         sprintf(errmsg, "write of file %s return %d",
146                                 filename, rc);
147                         FAIL(errmsg);
148                 }
149                 off += stride;
150
151                 while (off < size * stride) {
152                         fill_stride(buf, stride, 0x8080808080808080ULL, off);
153                         rc = write(fd, buf, stride);
154                         if (rc != stride) {
155                                 sprintf(errmsg, "write of file %s return %d",
156                                         filename, rc);
157                                 FAIL(errmsg);
158                         }
159
160                         off += stride;
161                 }
162
163                 if (close(fd) == -1) {
164                         sprintf(errmsg, "close of file %s", filename);
165                         FAIL(errmsg);
166                 }
167         }
168
169         MPI_Barrier(MPI_COMM_WORLD);
170
171         o_buf = (char *)malloc(stride);
172         if (o_buf == NULL) {
173                 sprintf(errmsg, "malloc of o_buf with size %ld", stride);
174                 FAIL(errmsg);
175         }
176
177         fd = open(filename, O_RDWR);
178         if (fd == -1) {
179                 sprintf(errmsg, "open of file %s", filename);
180                 FAIL(errmsg);
181         }
182
183         off = 0;
184         for (i = 1; i < size; ++i) {
185                 if (rank == i) {
186                         rc = lseek(fd, off, SEEK_SET);
187                         if (rc != off) {
188                                 sprintf(errmsg, "lseek of file %s return %d",
189                                         filename, rc);
190                                 FAIL(errmsg);
191                         }
192
193                         rc = read(fd, buf, stride);
194                         if (rc != stride) {
195                                 if (rc > 0) {
196                                         fill_stride(o_buf, rc, i - 1, off);
197                                         dump_diff(o_buf, buf, rc, off);
198                                 }
199                                 sprintf(errmsg, "read of file %s return %d",
200                                         filename, rc);
201                                 FAIL(errmsg);
202                         }
203
204                         fill_stride(o_buf, stride, i - 1, off);
205                         if (memcmp(o_buf, buf, stride) != 0) {
206                                 dump_diff(o_buf, buf, stride, off);
207                                 errno = 0;
208                                 sprintf(errmsg, "Error: diff data read from %s",
209                                         filename);
210                                 FAIL(errmsg);
211                         }
212                 }
213
214                 off += stride;
215
216                 if (rank == i) {
217                         fill_stride(buf, stride, i, off);
218                         rc = write(fd, buf, stride);
219                         if (rc != stride) {
220                                 sprintf(errmsg, "write of file %s return %d",
221                                         filename, rc);
222                                 FAIL(errmsg);
223                         }
224                 }
225
226                 MPI_Barrier(MPI_COMM_WORLD);
227         }
228
229         if (close(fd) == -1) {
230                 sprintf(errmsg, "close of file %s", filename);
231                 FAIL(errmsg);
232         }
233
234         MPI_Barrier(MPI_COMM_WORLD);
235
236         if (rank == 0) {
237                 fd = open(filename, O_RDONLY);
238                 if (fd == -1) {
239                         sprintf(errmsg, "open of file %s", filename);
240                         FAIL(errmsg);
241                 }
242
243                 off = 0;
244                 for (i = 0; i < size; ++i) {
245                         rc = read(fd, buf, stride);
246                         if (rc != stride) {
247                                 if (rc > 0) {
248                                         fill_stride(o_buf, rc, i, off);
249                                         dump_diff(o_buf, buf, rc, off);
250                                 }
251                                 sprintf(errmsg, "read of file %s", filename);
252                                 FAIL(errmsg);
253                         }
254
255                         fill_stride(o_buf, stride, i, off);
256                         if (memcmp(o_buf, buf, stride) != 0) {
257                                 bad = 1;
258                                 dump_diff(o_buf, buf, stride, off);
259                         }
260                         off += stride;
261                 }
262                 if (bad == 1) {
263                         errno = 0;
264                         sprintf(errmsg, "Error: diff data read from %s", filename);
265                         FAIL(errmsg);
266                 }
267         }
268
269         MPI_Barrier(MPI_COMM_WORLD);
270         fprintf(stderr, "passed barrier 5\n");
271
272         free(buf);
273         free(o_buf);
274 }
275
276 void cascading_rw(long stride, unsigned int seed)
277 {
278         begin("setup");
279         end("setup");
280
281         begin("test");
282         rw_file("cascading_rw", stride, seed);
283         end("test");
284
285         begin("cleanup");
286         remove_file("cascading_rw");
287         end("cleanup");
288 }
289
290 void usage(char *proc)
291 {
292         int i;
293
294         if (rank == 0) {
295                 printf("Usage: %s [-h] -d <testdir> [-s \"1024\"]\n", proc);
296                 printf("           [-n \"13\"] [-e \"12345\"]\n"); 
297                 printf("           [-v] [-V #] [-g]\n");
298                 printf("\t-h: prints this help message\n");
299                 printf("\t-d: the directory in which the tests will run\n");
300                 printf("\t-s: process stride size\n");
301                 printf("\t-n: repeat test # times\n");
302                 printf("\t-n: random seed, used to re-create previous runs\n");
303                 printf("\t-v: increase the verbositly level by 1\n");
304                 printf("\t-V: select a specific verbosity level\n");
305                 printf("\t-g: debug mode\n");
306         }
307
308         MPI_Initialized(&i);
309         if (i) MPI_Finalize();
310         exit(0);
311 }
312
313 int main(int argc, char *argv[])
314 {
315         int i, iterations = 16, c;
316         long stride = -1;
317         unsigned int seed = 0;
318
319         /* Check for -h parameter before MPI_Init so the binary can be
320            called directly, without, for instance, mpirun */
321         for (i = 1; i < argc; ++i) {
322                 if (!strcmp(argv[i], "-h") || !strcmp(argv[i], "--help"))
323                         usage(argv[0]);
324         }
325
326         MPI_Init(&argc, &argv);
327         MPI_Comm_rank(MPI_COMM_WORLD, &rank);
328         MPI_Comm_size(MPI_COMM_WORLD, &size);
329
330                 /* Parse command line options */
331         while (1) {
332                 c = getopt(argc, argv, "d:e:ghn:s:vV:");
333                 if (c == -1)
334                         break;
335
336                 switch (c) {
337                 case 'd':
338                         testdir = optarg;
339                         break;
340                 case 'e':
341                         seed = (unsigned int)atoi(optarg);
342                         break;
343                 case 'g':
344                         debug = 1;
345                         break;
346                 case 'h':
347                         usage(argv[0]);
348                         break;
349                 case 'n':
350                         iterations = atoi(optarg);
351                         break;
352                 case 's':
353                         stride = atol(optarg);
354                         break;
355                 case 'v':
356                         verbose += 1;
357                         break;
358                 case 'V':
359                         verbose = atoi(optarg);
360                         break;
361                 }
362         }
363
364         if (rank == 0)
365                 printf("%s is running with %d process(es) %s\n",
366                        argv[0], size, debug ? "in DEBUG mode" : "\b\b");
367
368         if (size < 2) {
369                 fprintf(stderr,
370                         "There should be at least 3 process to run the test!\n");
371                 MPI_Abort(MPI_COMM_WORLD, 2);
372         }
373
374         if (testdir == NULL && rank == 0) {
375                 fprintf(stderr, "Please specify a test directory! (\"%s -h\" for help)\n",
376                        argv[0]);
377                 MPI_Abort(MPI_COMM_WORLD, 2);
378         }
379
380         lp_gethostname();
381
382         for (i = 0; i < iterations; ++i) {
383                 if (rank == 0)
384                         printf("%s: Running test #%s(iter %d)\n",
385                                timestamp(), argv[0], i);
386
387                 cascading_rw(stride, seed);
388                 MPI_Barrier(MPI_COMM_WORLD);
389         }
390
391         if (rank == 0)
392                 printf("%s: All tests passed!\n", timestamp());
393         MPI_Finalize();
394         return 0;
395 }