lustre/tests/mpi/cascading_rw.c

   1 /*
   2  * GPL HEADER START
   3  *
   4  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
   5  *
   6  * This program is free software; you can redistribute it and/or modify
   7  * it under the terms of the GNU General Public License version 2 only,
   8  * as published by the Free Software Foundation.
   9  *
  10  * This program is distributed in the hope that it will be useful, but
  11  * WITHOUT ANY WARRANTY; without even the implied warranty of
  12  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  13  * General Public License version 2 for more details (a copy is included
  14  * in the LICENSE file that accompanied this code).
  15  *
  16  * You should have received a copy of the GNU General Public License
  17  * version 2 along with this program; If not, see
  18  * http://www.sun.com/software/products/lustre/docs/GPLv2.pdf
  19  *
  20  * Please contact Sun Microsystems, Inc., 4150 Network Circle, Santa Clara,
  21  * CA 95054 USA or visit www.sun.com if you need additional information or
  22  * have any questions.
  23  *
  24  * GPL HEADER END
  25  */
  26 /*
  27  * Copyright (c) 2010, Oracle and/or its affiliates. All rights reserved.
  28  * Use is subject to license terms.
  29  */
  30 /*
  31  * This file is part of Lustre, http://www.lustre.org/
  32  * Lustre is a trademark of Sun Microsystems, Inc.
  33  *
  34  * lustre/tests/mpi/cascading_rw.c
  35  *
  36  * Author: You Feng <youfeng@clusterfs.com>
  37  */
  38
  39 #include <config.h>
  40 #include <mpi.h>
  41 #include <stdio.h>
  42 #include <stdlib.h>
  43 #include <string.h>
  44 #include <sys/types.h>
  45 #include <asm/types.h>
  46 #include <sys/stat.h>
  47 #include <fcntl.h>
  48 #include <unistd.h>
  49 #include <sys/ioctl.h>
  50 #include <getopt.h>
  51 #include <errno.h>
  52
  53 #include <lustre/lustreapi.h>
  54 #include "lp_utils.h"
  55 #ifndef _IOWR
  56 # include <ioctl.h>
  57 #endif
  58
  59
  60 int rank = 0;
  61 int size = 0;
  62
  63 char *testdir = NULL;
  64
  65 void rw_file(char *name, long stride, unsigned int seed)
  66 {
  67         char filename[MAX_FILENAME_LEN];
  68         char errmsg[MAX_FILENAME_LEN+20];
  69         char *buf, *o_buf;
  70         struct lov_user_md lum = {0};
  71         int fd, rc, i, bad = 0, root = 0;
  72         long off;
  73         long page_size = sysconf(_SC_PAGESIZE);
  74
  75         sprintf(filename, "%s/%s", testdir, name);
  76
  77         if (rank == 0) {
  78                 remove_file_or_dir(filename);
  79
  80                 lum.lmm_magic = LOV_USER_MAGIC;
  81                 lum.lmm_stripe_size = 0;
  82                 lum.lmm_stripe_count = 0;
  83                 lum.lmm_stripe_offset = -1;
  84
  85                 fd = open(filename, O_CREAT | O_RDWR | O_LOV_DELAY_CREATE,
  86                           FILEMODE);
  87                 if (fd == -1) {
  88                         sprintf(errmsg, "open of file %s", filename);
  89                         FAIL(errmsg);
  90                 }
  91
  92                 rc = ioctl(fd, LL_IOC_LOV_SETSTRIPE, &lum);
  93                 if (rc == -1) {
  94                         sprintf(errmsg, "ioctl SETSTRIPE of file %s", filename);
  95                         FAIL(errmsg);
  96                 }
  97
  98                 if (close(fd) == -1) {
  99                         sprintf(errmsg, "close of file %s", filename);
 100                         FAIL(errmsg);
 101                 }
 102         }
 103
 104         MPI_Barrier(MPI_COMM_WORLD);
 105
 106         if (stride < 0) {
 107                 if (rank == 0) {
 108                         srandom(seed);
 109                         while (stride < page_size/2) {
 110                                 stride = random();
 111                                 stride -= stride % 16;
 112                                 if (stride < 0)
 113                                         stride = -stride;
 114                                 stride %= 2 * lum.lmm_stripe_size;
 115                         }
 116                 }
 117
 118                 MPI_Barrier(MPI_COMM_WORLD);
 119
 120                 MPI_Bcast(&stride, 1, MPI_LONG, root, MPI_COMM_WORLD);
 121         }
 122
 123         MPI_Barrier(MPI_COMM_WORLD);
 124
 125         buf = (char *)malloc(stride);
 126         if (buf == NULL) {
 127                 sprintf(errmsg, "malloc of buf with size %ld", stride);
 128                 FAIL(errmsg);
 129         }
 130
 131         if (rank == 0) {
 132                 fd = open(filename, O_RDWR);
 133                 if (fd == -1) {
 134                         sprintf(errmsg, "open of file %s", filename);
 135                         FAIL(errmsg);
 136                 }
 137
 138                 off = 0;
 139                 fill_stride(buf, stride, 0, off);
 140                 rc = write(fd, buf, stride);
 141                 if (rc != stride) {
 142                         sprintf(errmsg, "write of file %s return %d",
 143                                 filename, rc);
 144                         FAIL(errmsg);
 145                 }
 146                 off += stride;
 147
 148                 while (off < size * stride) {
 149                         fill_stride(buf, stride, 0x8080808080808080ULL, off);
 150                         rc = write(fd, buf, stride);
 151                         if (rc != stride) {
 152                                 sprintf(errmsg, "write of file %s return %d",
 153                                         filename, rc);
 154                                 FAIL(errmsg);
 155                         }
 156
 157                         off += stride;
 158                 }
 159
 160                 if (close(fd) == -1) {
 161                         sprintf(errmsg, "close of file %s", filename);
 162                         FAIL(errmsg);
 163                 }
 164         }
 165
 166         MPI_Barrier(MPI_COMM_WORLD);
 167
 168         o_buf = (char *)malloc(stride);
 169         if (o_buf == NULL) {
 170                 sprintf(errmsg, "malloc of o_buf with size %ld", stride);
 171                 FAIL(errmsg);
 172         }
 173
 174         fd = open(filename, O_RDWR);
 175         if (fd == -1) {
 176                 sprintf(errmsg, "open of file %s", filename);
 177                 FAIL(errmsg);
 178         }
 179
 180         off = 0;
 181         for (i = 1; i < size; ++i) {
 182                 if (rank == i) {
 183                         rc = lseek(fd, off, SEEK_SET);
 184                         if (rc != off) {
 185                                 sprintf(errmsg, "lseek of file %s return %d",
 186                                         filename, rc);
 187                                 FAIL(errmsg);
 188                         }
 189
 190                         rc = read(fd, buf, stride);
 191                         if (rc != stride) {
 192                                 if (rc > 0) {
 193                                         fill_stride(o_buf, rc, i - 1, off);
 194                                         dump_diff(o_buf, buf, rc, off);
 195                                 }
 196                                 sprintf(errmsg, "read of file %s return %d",
 197                                         filename, rc);
 198                                 FAIL(errmsg);
 199                         }
 200
 201                         fill_stride(o_buf, stride, i - 1, off);
 202                         if (memcmp(o_buf, buf, stride) != 0) {
 203                                 dump_diff(o_buf, buf, stride, off);
 204                                 errno = 0;
 205                                 sprintf(errmsg, "Error: diff data read from %s",
 206                                         filename);
 207                                 FAIL(errmsg);
 208                         }
 209                 }
 210
 211                 off += stride;
 212
 213                 if (rank == i) {
 214                         fill_stride(buf, stride, i, off);
 215                         rc = write(fd, buf, stride);
 216                         if (rc != stride) {
 217                                 sprintf(errmsg, "write of file %s return %d",
 218                                         filename, rc);
 219                                 FAIL(errmsg);
 220                         }
 221                 }
 222
 223                 MPI_Barrier(MPI_COMM_WORLD);
 224         }
 225
 226         if (close(fd) == -1) {
 227                 sprintf(errmsg, "close of file %s", filename);
 228                 FAIL(errmsg);
 229         }
 230
 231         MPI_Barrier(MPI_COMM_WORLD);
 232
 233         if (rank == 0) {
 234                 fd = open(filename, O_RDONLY);
 235                 if (fd == -1) {
 236                         sprintf(errmsg, "open of file %s", filename);
 237                         FAIL(errmsg);
 238                 }
 239
 240                 off = 0;
 241                 for (i = 0; i < size; ++i) {
 242                         rc = read(fd, buf, stride);
 243                         if (rc != stride) {
 244                                 if (rc > 0) {
 245                                         fill_stride(o_buf, rc, i, off);
 246                                         dump_diff(o_buf, buf, rc, off);
 247                                 }
 248                                 sprintf(errmsg, "read of file %s", filename);
 249                                 FAIL(errmsg);
 250                         }
 251
 252                         fill_stride(o_buf, stride, i, off);
 253                         if (memcmp(o_buf, buf, stride) != 0) {
 254                                 bad = 1;
 255                                 dump_diff(o_buf, buf, stride, off);
 256                         }
 257                         off += stride;
 258                 }
 259                 if (bad == 1) {
 260                         errno = 0;
 261                         sprintf(errmsg, "Error: diff data read from %s", filename);
 262                         FAIL(errmsg);
 263                 }
 264         }
 265
 266         MPI_Barrier(MPI_COMM_WORLD);
 267         fprintf(stderr, "passed barrier 5\n");
 268
 269         free(buf);
 270         free(o_buf);
 271 }
 272
 273 void cascading_rw(long stride, unsigned int seed)
 274 {
 275         begin("setup");
 276         end("setup");
 277
 278         begin("test");
 279         rw_file("cascading_rw", stride, seed);
 280         end("test");
 281
 282         begin("cleanup");
 283         remove_file("cascading_rw");
 284         end("cleanup");
 285 }
 286
 287 void usage(char *proc)
 288 {
 289         int i;
 290
 291         if (rank == 0) {
 292                 printf("Usage: %s [-h] -d <testdir> [-s \"1024\"]\n", proc);
 293                 printf("           [-n \"13\"] [-e \"12345\"]\n");
 294                 printf("           [-v] [-V #] [-g]\n");
 295                 printf("\t-h: prints this help message\n");
 296                 printf("\t-d: the directory in which the tests will run\n");
 297                 printf("\t-s: process stride size\n");
 298                 printf("\t-n: repeat test # times\n");
 299                 printf("\t-n: random seed, used to re-create previous runs\n");
 300                 printf("\t-v: increase the verbositly level by 1\n");
 301                 printf("\t-V: select a specific verbosity level\n");
 302                 printf("\t-g: debug mode\n");
 303         }
 304
 305         MPI_Initialized(&i);
 306         if (i) MPI_Finalize();
 307         exit(0);
 308 }
 309
 310 int main(int argc, char *argv[])
 311 {
 312         int i, iterations = 16, c;
 313         long stride = -1;
 314         unsigned int seed = 0;
 315
 316         /* Check for -h parameter before MPI_Init so the binary can be
 317            called directly, without, for instance, mpirun */
 318         for (i = 1; i < argc; ++i) {
 319                 if (!strcmp(argv[i], "-h") || !strcmp(argv[i], "--help"))
 320                         usage(argv[0]);
 321         }
 322
 323         MPI_Init(&argc, &argv);
 324         MPI_Comm_rank(MPI_COMM_WORLD, &rank);
 325         MPI_Comm_size(MPI_COMM_WORLD, &size);
 326
 327                 /* Parse command line options */
 328         while (1) {
 329                 c = getopt(argc, argv, "d:e:ghn:s:vV:");
 330                 if (c == -1)
 331                         break;
 332
 333                 switch (c) {
 334                 case 'd':
 335                         testdir = optarg;
 336                         break;
 337                 case 'e':
 338                         seed = (unsigned int)atoi(optarg);
 339                         break;
 340                 case 'g':
 341                         debug = 1;
 342                         break;
 343                 case 'h':
 344                         usage(argv[0]);
 345                         break;
 346                 case 'n':
 347                         iterations = atoi(optarg);
 348                         break;
 349                 case 's':
 350                         stride = atol(optarg);
 351                         break;
 352                 case 'v':
 353                         verbose += 1;
 354                         break;
 355                 case 'V':
 356                         verbose = atoi(optarg);
 357                         break;
 358                 }
 359         }
 360
 361         if (rank == 0)
 362                 printf("%s is running with %d process(es) %s\n",
 363                        argv[0], size, debug ? "in DEBUG mode" : "\b\b");
 364
 365         if (size < 2) {
 366                 fprintf(stderr,
 367                         "There should be at least 3 process to run the test!\n");
 368                 MPI_Abort(MPI_COMM_WORLD, 2);
 369         }
 370
 371         if (testdir == NULL && rank == 0) {
 372                 fprintf(stderr, "Please specify a test directory! (\"%s -h\" for help)\n",
 373                        argv[0]);
 374                 MPI_Abort(MPI_COMM_WORLD, 2);
 375         }
 376
 377         lp_gethostname();
 378
 379         for (i = 0; i < iterations; ++i) {
 380                 if (rank == 0)
 381                         printf("%s: Running test #%s(iter %d)\n",
 382                                timestamp(), argv[0], i);
 383
 384                 cascading_rw(stride, seed);
 385                 MPI_Barrier(MPI_COMM_WORLD);
 386         }
 387
 388         if (rank == 0)
 389                 printf("%s: All tests passed!\n", timestamp());
 390         MPI_Finalize();
 391         return 0;
 392 }