Whamcloud - gitweb
fe91c202070f4ac611ecd01032fca2f775874e1f
[fs/lustre-release.git] / lustre / tests / mpi / cascading_rw.c
1 /* -*- mode: c; c-basic-offset: 8; indent-tabs-mode: nil; -*-
2  * vim:expandtab:shiftwidth=8:tabstop=8:
3  *
4  * GPL HEADER START
5  *
6  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
7  *
8  * This program is free software; you can redistribute it and/or modify
9  * it under the terms of the GNU General Public License version 2 only,
10  * as published by the Free Software Foundation.
11  *
12  * This program is distributed in the hope that it will be useful, but
13  * WITHOUT ANY WARRANTY; without even the implied warranty of
14  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
15  * General Public License version 2 for more details (a copy is included
16  * in the LICENSE file that accompanied this code).
17  *
18  * You should have received a copy of the GNU General Public License
19  * version 2 along with this program; If not, see
20  * http://www.sun.com/software/products/lustre/docs/GPLv2.pdf
21  *
22  * Please contact Sun Microsystems, Inc., 4150 Network Circle, Santa Clara,
23  * CA 95054 USA or visit www.sun.com if you need additional information or
24  * have any questions.
25  *
26  * GPL HEADER END
27  */
28 /*
29  * Copyright (c) 2010, Oracle and/or its affiliates. All rights reserved.
30  * Use is subject to license terms.
31  */
32 /*
33  * This file is part of Lustre, http://www.lustre.org/
34  * Lustre is a trademark of Sun Microsystems, Inc.
35  *
36  * lustre/tests/mpi/cascading_rw.c
37  *
38  * Author: You Feng <youfeng@clusterfs.com>
39  */
40
41 #include <config.h>
42 #include <mpi.h>
43 #include <stdio.h>
44 #include <stdlib.h>
45 #include <string.h>
46 #include <sys/types.h>
47 #include <asm/types.h>
48 #include <sys/stat.h>
49 #include <fcntl.h>
50 #include <unistd.h>
51 #include <sys/ioctl.h>
52 #include <getopt.h>
53 #include <errno.h>
54
55 #include <libcfs/libcfs.h>
56 #include "lp_utils.h"
57 #ifndef _IOWR
58 # include <ioctl.h>
59 #endif
60
61 #include <lustre/liblustreapi.h>
62
63 int rank = 0;
64 int size = 0;
65
66 char *testdir = NULL;
67
68 void rw_file(char *name, long stride, unsigned int seed)
69 {
70         char filename[MAX_FILENAME_LEN];
71         char path[MAX_FILENAME_LEN]; 
72         char errmsg[MAX_FILENAME_LEN+20];
73         char *buf, *o_buf;
74         struct lov_user_md lum = {0};
75         int fd, rc, i, bad = 0, root = 0;
76         long off;
77         long page_size = sysconf(_SC_PAGESIZE);
78
79         sprintf(filename, "%s/%s", testdir, name);
80
81         if (rank == 0) {
82                 remove_file_or_dir(filename);
83
84                 lum.lmm_magic = LOV_USER_MAGIC;
85                 lum.lmm_stripe_size = 0;
86                 lum.lmm_stripe_count = 0;
87                 lum.lmm_stripe_offset = -1;
88
89                 fd = open(filename, O_CREAT | O_RDWR | O_LOV_DELAY_CREATE,
90                           FILEMODE);
91                 if (fd == -1) {
92                         sprintf(errmsg, "open of file %s", filename);
93                         FAIL(errmsg);
94                 }
95
96                 rc = ioctl(fd, LL_IOC_LOV_SETSTRIPE, &lum);
97                 if (rc == -1) {
98                         sprintf(errmsg, "ioctl SETSTRIPE of file %s", filename);
99                         FAIL(errmsg);
100                 }
101
102                 if (close(fd) == -1) {
103                         sprintf(errmsg, "close of file %s", filename);
104                         FAIL(errmsg);
105                 }
106
107                 strncpy(path, filename, MAX_FILENAME_LEN);
108                 rc = llapi_file_get_stripe(path, &lum);
109                 if (rc == -1) {
110                         sprintf(errmsg, "get stripe of file %s", filename);
111                         FAIL(errmsg);
112                 }
113         }
114
115         MPI_Barrier(MPI_COMM_WORLD);
116
117         if (stride < 0) {
118                 if (rank == 0) {
119                         srandom(seed);
120                         while (stride < page_size/2) {
121                                 stride = random();
122                                 stride -= stride % 16;
123                                 if (stride < 0)
124                                         stride = -stride;
125                                 stride %= 2 * lum.lmm_stripe_size;
126                         }
127                 }
128
129                 MPI_Barrier(MPI_COMM_WORLD);
130
131                 MPI_Bcast(&stride, 1, MPI_LONG, root, MPI_COMM_WORLD);
132         }
133
134         MPI_Barrier(MPI_COMM_WORLD);
135
136         buf = (char *)malloc(stride);
137         if (buf == NULL) {
138                 sprintf(errmsg, "malloc of buf with size %ld", stride);
139                 FAIL(errmsg);
140         }
141
142         if (rank == 0) {
143                 fd = open(filename, O_RDWR);
144                 if (fd == -1) {
145                         sprintf(errmsg, "open of file %s", filename);
146                         FAIL(errmsg);
147                 }
148
149                 off = 0;
150                 fill_stride(buf, stride, 0, off);
151                 rc = write(fd, buf, stride);
152                 if (rc != stride) {
153                         sprintf(errmsg, "write of file %s return %d",
154                                 filename, rc);
155                         FAIL(errmsg);
156                 }
157                 off += stride;
158
159                 while (off < size * stride) {
160                         fill_stride(buf, stride, 0x8080808080808080ULL, off);
161                         rc = write(fd, buf, stride);
162                         if (rc != stride) {
163                                 sprintf(errmsg, "write of file %s return %d",
164                                         filename, rc);
165                                 FAIL(errmsg);
166                         }
167
168                         off += stride;
169                 }
170
171                 if (close(fd) == -1) {
172                         sprintf(errmsg, "close of file %s", filename);
173                         FAIL(errmsg);
174                 }
175         }
176
177         MPI_Barrier(MPI_COMM_WORLD);
178
179         o_buf = (char *)malloc(stride);
180         if (o_buf == NULL) {
181                 sprintf(errmsg, "malloc of o_buf with size %ld", stride);
182                 FAIL(errmsg);
183         }
184
185         fd = open(filename, O_RDWR);
186         if (fd == -1) {
187                 sprintf(errmsg, "open of file %s", filename);
188                 FAIL(errmsg);
189         }
190
191         off = 0;
192         for (i = 1; i < size; ++i) {
193                 if (rank == i) {
194                         rc = lseek(fd, off, SEEK_SET);
195                         if (rc != off) {
196                                 sprintf(errmsg, "lseek of file %s return %d",
197                                         filename, rc);
198                                 FAIL(errmsg);
199                         }
200
201                         rc = read(fd, buf, stride);
202                         if (rc != stride) {
203                                 if (rc > 0) {
204                                         fill_stride(o_buf, rc, i - 1, off);
205                                         dump_diff(o_buf, buf, rc, off);
206                                 }
207                                 sprintf(errmsg, "read of file %s return %d",
208                                         filename, rc);
209                                 FAIL(errmsg);
210                         }
211
212                         fill_stride(o_buf, stride, i - 1, off);
213                         if (memcmp(o_buf, buf, stride) != 0) {
214                                 dump_diff(o_buf, buf, stride, off);
215                                 errno = 0;
216                                 sprintf(errmsg, "Error: diff data read from %s",
217                                         filename);
218                                 FAIL(errmsg);
219                         }
220                 }
221
222                 off += stride;
223
224                 if (rank == i) {
225                         fill_stride(buf, stride, i, off);
226                         rc = write(fd, buf, stride);
227                         if (rc != stride) {
228                                 sprintf(errmsg, "write of file %s return %d",
229                                         filename, rc);
230                                 FAIL(errmsg);
231                         }
232                 }
233
234                 MPI_Barrier(MPI_COMM_WORLD);
235         }
236
237         if (close(fd) == -1) {
238                 sprintf(errmsg, "close of file %s", filename);
239                 FAIL(errmsg);
240         }
241
242         MPI_Barrier(MPI_COMM_WORLD);
243
244         if (rank == 0) {
245                 fd = open(filename, O_RDONLY);
246                 if (fd == -1) {
247                         sprintf(errmsg, "open of file %s", filename);
248                         FAIL(errmsg);
249                 }
250
251                 off = 0;
252                 for (i = 0; i < size; ++i) {
253                         rc = read(fd, buf, stride);
254                         if (rc != stride) {
255                                 if (rc > 0) {
256                                         fill_stride(o_buf, rc, i, off);
257                                         dump_diff(o_buf, buf, rc, off);
258                                 }
259                                 sprintf(errmsg, "read of file %s", filename);
260                                 FAIL(errmsg);
261                         }
262
263                         fill_stride(o_buf, stride, i, off);
264                         if (memcmp(o_buf, buf, stride) != 0) {
265                                 bad = 1;
266                                 dump_diff(o_buf, buf, stride, off);
267                         }
268                         off += stride;
269                 }
270                 if (bad == 1) {
271                         errno = 0;
272                         sprintf(errmsg, "Error: diff data read from %s", filename);
273                         FAIL(errmsg);
274                 }
275         }
276
277         MPI_Barrier(MPI_COMM_WORLD);
278         fprintf(stderr, "passed barrier 5\n");
279
280         free(buf);
281         free(o_buf);
282 }
283
284 void cascading_rw(long stride, unsigned int seed)
285 {
286         begin("setup");
287         end("setup");
288
289         begin("test");
290         rw_file("cascading_rw", stride, seed);
291         end("test");
292
293         begin("cleanup");
294         remove_file("cascading_rw");
295         end("cleanup");
296 }
297
298 void usage(char *proc)
299 {
300         int i;
301
302         if (rank == 0) {
303                 printf("Usage: %s [-h] -d <testdir> [-s \"1024\"]\n", proc);
304                 printf("           [-n \"13\"] [-e \"12345\"]\n"); 
305                 printf("           [-v] [-V #] [-g]\n");
306                 printf("\t-h: prints this help message\n");
307                 printf("\t-d: the directory in which the tests will run\n");
308                 printf("\t-s: process stride size\n");
309                 printf("\t-n: repeat test # times\n");
310                 printf("\t-n: random seed, used to re-create previous runs\n");
311                 printf("\t-v: increase the verbositly level by 1\n");
312                 printf("\t-V: select a specific verbosity level\n");
313                 printf("\t-g: debug mode\n");
314         }
315
316         MPI_Initialized(&i);
317         if (i) MPI_Finalize();
318         exit(0);
319 }
320
321 int main(int argc, char *argv[])
322 {
323         int i, iterations = 16, c;
324         long stride = -1;
325         unsigned int seed = 0;
326
327         /* Check for -h parameter before MPI_Init so the binary can be
328            called directly, without, for instance, mpirun */
329         for (i = 1; i < argc; ++i) {
330                 if (!strcmp(argv[i], "-h") || !strcmp(argv[i], "--help"))
331                         usage(argv[0]);
332         }
333
334         MPI_Init(&argc, &argv);
335         MPI_Comm_rank(MPI_COMM_WORLD, &rank);
336         MPI_Comm_size(MPI_COMM_WORLD, &size);
337
338                 /* Parse command line options */
339         while (1) {
340                 c = getopt(argc, argv, "d:e:ghn:s:vV:");
341                 if (c == -1)
342                         break;
343
344                 switch (c) {
345                 case 'd':
346                         testdir = optarg;
347                         break;
348                 case 'e':
349                         seed = (unsigned int)atoi(optarg);
350                         break;
351                 case 'g':
352                         debug = 1;
353                         break;
354                 case 'h':
355                         usage(argv[0]);
356                         break;
357                 case 'n':
358                         iterations = atoi(optarg);
359                         break;
360                 case 's':
361                         stride = atol(optarg);
362                         break;
363                 case 'v':
364                         verbose += 1;
365                         break;
366                 case 'V':
367                         verbose = atoi(optarg);
368                         break;
369                 }
370         }
371
372         if (rank == 0)
373                 printf("%s is running with %d process(es) %s\n",
374                        argv[0], size, debug ? "in DEBUG mode" : "\b\b");
375
376         if (size < 2) {
377                 fprintf(stderr,
378                         "There should be at least 3 process to run the test!\n");
379                 MPI_Abort(MPI_COMM_WORLD, 2);
380         }
381
382         if (testdir == NULL && rank == 0) {
383                 fprintf(stderr, "Please specify a test directory! (\"%s -h\" for help)\n",
384                        argv[0]);
385                 MPI_Abort(MPI_COMM_WORLD, 2);
386         }
387
388         lp_gethostname();
389
390         for (i = 0; i < iterations; ++i) {
391                 if (rank == 0)
392                         printf("%s: Running test #%s(iter %d)\n",
393                                timestamp(), argv[0], i);
394
395                 cascading_rw(stride, seed);
396                 MPI_Barrier(MPI_COMM_WORLD);
397         }
398
399         if (rank == 0)
400                 printf("%s: All tests passed!\n", timestamp());
401         MPI_Finalize();
402         return 0;
403 }