lustre/tests/mpi/cascading_rw.c

   1 /*
   2  * GPL HEADER START
   3  *
   4  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
   5  *
   6  * This program is free software; you can redistribute it and/or modify
   7  * it under the terms of the GNU General Public License version 2 only,
   8  * as published by the Free Software Foundation.
   9  *
  10  * This program is distributed in the hope that it will be useful, but
  11  * WITHOUT ANY WARRANTY; without even the implied warranty of
  12  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  13  * General Public License version 2 for more details (a copy is included
  14  * in the LICENSE file that accompanied this code).
  15  *
  16  * You should have received a copy of the GNU General Public License
  17  * version 2 along with this program; If not, see
  18  * http://www.gnu.org/licenses/gpl-2.0.html
  19  *
  20  * GPL HEADER END
  21  */
  22 /*
  23  * Copyright (c) 2010, Oracle and/or its affiliates. All rights reserved.
  24  * Use is subject to license terms.
  25  */
  26 /*
  27  * This file is part of Lustre, http://www.lustre.org/
  28  * Lustre is a trademark of Sun Microsystems, Inc.
  29  *
  30  * lustre/tests/mpi/cascading_rw.c
  31  *
  32  * Author: You Feng <youfeng@clusterfs.com>
  33  */
  34
  35 #include <config.h>
  36 #include <mpi.h>
  37 #include <stdio.h>
  38 #include <stdlib.h>
  39 #include <string.h>
  40 #include <sys/types.h>
  41 #include <asm/types.h>
  42 #include <sys/stat.h>
  43 #include <fcntl.h>
  44 #include <unistd.h>
  45 #include <sys/ioctl.h>
  46 #include <getopt.h>
  47 #include <errno.h>
  48
  49 #include <lustre/lustreapi.h>
  50 #include "lp_utils.h"
  51 #ifndef _IOWR
  52 # include <ioctl.h>
  53 #endif
  54
  55
  56 int rank = 0;
  57 int size = 0;
  58
  59 char *testdir = NULL;
  60
  61 void rw_file(char *name, long stride, unsigned int seed)
  62 {
  63         char filename[MAX_FILENAME_LEN];
  64         char errmsg[MAX_FILENAME_LEN+32];
  65         char *buf, *o_buf;
  66         struct lov_user_md lum = {0};
  67         int fd, rc, i, bad = 0, root = 0;
  68         long off;
  69         long page_size = sysconf(_SC_PAGESIZE);
  70
  71         sprintf(filename, "%s/%s", testdir, name);
  72
  73         if (rank == 0) {
  74                 remove_file_or_dir(filename);
  75
  76                 lum.lmm_magic = LOV_USER_MAGIC;
  77                 lum.lmm_stripe_size = 0;
  78                 lum.lmm_stripe_count = 0;
  79                 lum.lmm_stripe_offset = -1;
  80
  81                 fd = open(filename, O_CREAT | O_RDWR | O_LOV_DELAY_CREATE,
  82                           FILEMODE);
  83                 if (fd == -1) {
  84                         sprintf(errmsg, "open of file %s", filename);
  85                         FAIL(errmsg);
  86                 }
  87
  88                 rc = ioctl(fd, LL_IOC_LOV_SETSTRIPE, &lum);
  89                 if (rc == -1) {
  90                         sprintf(errmsg, "ioctl SETSTRIPE of file %s", filename);
  91                         FAIL(errmsg);
  92                 }
  93
  94                 if (close(fd) == -1) {
  95                         sprintf(errmsg, "close of file %s", filename);
  96                         FAIL(errmsg);
  97                 }
  98         }
  99
 100         MPI_Barrier(MPI_COMM_WORLD);
 101
 102         if (stride < 0) {
 103                 if (rank == 0) {
 104                         srandom(seed);
 105                         while (stride < page_size/2) {
 106                                 stride = random();
 107                                 stride -= stride % 16;
 108                                 if (stride < 0)
 109                                         stride = -stride;
 110                                 stride %= 2 * lum.lmm_stripe_size;
 111                         }
 112                 }
 113
 114                 MPI_Barrier(MPI_COMM_WORLD);
 115
 116                 MPI_Bcast(&stride, 1, MPI_LONG, root, MPI_COMM_WORLD);
 117         }
 118
 119         MPI_Barrier(MPI_COMM_WORLD);
 120
 121         buf = (char *)malloc(stride);
 122         if (buf == NULL) {
 123                 sprintf(errmsg, "malloc of buf with size %ld", stride);
 124                 FAIL(errmsg);
 125         }
 126
 127         if (rank == 0) {
 128                 fd = open(filename, O_RDWR);
 129                 if (fd == -1) {
 130                         sprintf(errmsg, "open of file %s", filename);
 131                         FAIL(errmsg);
 132                 }
 133
 134                 off = 0;
 135                 fill_stride(buf, stride, 0, off);
 136                 rc = write(fd, buf, stride);
 137                 if (rc != stride) {
 138                         sprintf(errmsg, "write of file %s return %d",
 139                                 filename, rc);
 140                         FAIL(errmsg);
 141                 }
 142                 off += stride;
 143
 144                 while (off < size * stride) {
 145                         fill_stride(buf, stride, 0x8080808080808080ULL, off);
 146                         rc = write(fd, buf, stride);
 147                         if (rc != stride) {
 148                                 sprintf(errmsg, "write of file %s return %d",
 149                                         filename, rc);
 150                                 FAIL(errmsg);
 151                         }
 152
 153                         off += stride;
 154                 }
 155
 156                 if (close(fd) == -1) {
 157                         sprintf(errmsg, "close of file %s", filename);
 158                         FAIL(errmsg);
 159                 }
 160         }
 161
 162         MPI_Barrier(MPI_COMM_WORLD);
 163
 164         o_buf = (char *)malloc(stride);
 165         if (o_buf == NULL) {
 166                 sprintf(errmsg, "malloc of o_buf with size %ld", stride);
 167                 FAIL(errmsg);
 168         }
 169
 170         fd = open(filename, O_RDWR);
 171         if (fd == -1) {
 172                 sprintf(errmsg, "open of file %s", filename);
 173                 FAIL(errmsg);
 174         }
 175
 176         off = 0;
 177         for (i = 1; i < size; ++i) {
 178                 if (rank == i) {
 179                         rc = lseek(fd, off, SEEK_SET);
 180                         if (rc != off) {
 181                                 sprintf(errmsg, "lseek of file %s return %d",
 182                                         filename, rc);
 183                                 FAIL(errmsg);
 184                         }
 185
 186                         rc = read(fd, buf, stride);
 187                         if (rc != stride) {
 188                                 if (rc > 0) {
 189                                         fill_stride(o_buf, rc, i - 1, off);
 190                                         dump_diff(o_buf, buf, rc, off);
 191                                 }
 192                                 sprintf(errmsg, "read of file %s return %d",
 193                                         filename, rc);
 194                                 FAIL(errmsg);
 195                         }
 196
 197                         fill_stride(o_buf, stride, i - 1, off);
 198                         if (memcmp(o_buf, buf, stride) != 0) {
 199                                 dump_diff(o_buf, buf, stride, off);
 200                                 errno = 0;
 201                                 sprintf(errmsg, "Error: diff data read from %s",
 202                                         filename);
 203                                 FAIL(errmsg);
 204                         }
 205                 }
 206
 207                 off += stride;
 208
 209                 if (rank == i) {
 210                         fill_stride(buf, stride, i, off);
 211                         rc = write(fd, buf, stride);
 212                         if (rc != stride) {
 213                                 sprintf(errmsg, "write of file %s return %d",
 214                                         filename, rc);
 215                                 FAIL(errmsg);
 216                         }
 217                 }
 218
 219                 MPI_Barrier(MPI_COMM_WORLD);
 220         }
 221
 222         if (close(fd) == -1) {
 223                 sprintf(errmsg, "close of file %s", filename);
 224                 FAIL(errmsg);
 225         }
 226
 227         MPI_Barrier(MPI_COMM_WORLD);
 228
 229         if (rank == 0) {
 230                 fd = open(filename, O_RDONLY);
 231                 if (fd == -1) {
 232                         sprintf(errmsg, "open of file %s", filename);
 233                         FAIL(errmsg);
 234                 }
 235
 236                 off = 0;
 237                 for (i = 0; i < size; ++i) {
 238                         rc = read(fd, buf, stride);
 239                         if (rc != stride) {
 240                                 if (rc > 0) {
 241                                         fill_stride(o_buf, rc, i, off);
 242                                         dump_diff(o_buf, buf, rc, off);
 243                                 }
 244                                 sprintf(errmsg, "read of file %s", filename);
 245                                 FAIL(errmsg);
 246                         }
 247
 248                         fill_stride(o_buf, stride, i, off);
 249                         if (memcmp(o_buf, buf, stride) != 0) {
 250                                 bad = 1;
 251                                 dump_diff(o_buf, buf, stride, off);
 252                         }
 253                         off += stride;
 254                 }
 255                 if (bad == 1) {
 256                         errno = 0;
 257                         sprintf(errmsg, "Error: diff data read from %s", filename);
 258                         FAIL(errmsg);
 259                 }
 260         }
 261
 262         MPI_Barrier(MPI_COMM_WORLD);
 263         fprintf(stderr, "passed barrier 5\n");
 264
 265         free(buf);
 266         free(o_buf);
 267 }
 268
 269 void cascading_rw(long stride, unsigned int seed)
 270 {
 271         begin("setup");
 272         end("setup");
 273
 274         begin("test");
 275         rw_file("cascading_rw", stride, seed);
 276         end("test");
 277
 278         begin("cleanup");
 279         remove_file("cascading_rw");
 280         end("cleanup");
 281 }
 282
 283 void usage(char *proc)
 284 {
 285         int i;
 286
 287         if (rank == 0) {
 288                 printf("Usage: %s [-h] -d <testdir> [-s \"1024\"]\n", proc);
 289                 printf("           [-n \"13\"] [-e \"12345\"]\n");
 290                 printf("           [-v] [-V #] [-g]\n");
 291                 printf("\t-h: prints this help message\n");
 292                 printf("\t-d: the directory in which the tests will run\n");
 293                 printf("\t-s: process stride size\n");
 294                 printf("\t-n: repeat test # times\n");
 295                 printf("\t-n: random seed, used to re-create previous runs\n");
 296                 printf("\t-v: increase the verbositly level by 1\n");
 297                 printf("\t-V: select a specific verbosity level\n");
 298                 printf("\t-g: debug mode\n");
 299         }
 300
 301         MPI_Initialized(&i);
 302         if (i) MPI_Finalize();
 303         exit(0);
 304 }
 305
 306 int main(int argc, char *argv[])
 307 {
 308         int i, iterations = 16, c;
 309         long stride = -1;
 310         unsigned int seed = 0;
 311
 312         /* Check for -h parameter before MPI_Init so the binary can be
 313            called directly, without, for instance, mpirun */
 314         for (i = 1; i < argc; ++i) {
 315                 if (!strcmp(argv[i], "-h") || !strcmp(argv[i], "--help"))
 316                         usage(argv[0]);
 317         }
 318
 319         MPI_Init(&argc, &argv);
 320         MPI_Comm_rank(MPI_COMM_WORLD, &rank);
 321         MPI_Comm_size(MPI_COMM_WORLD, &size);
 322
 323                 /* Parse command line options */
 324         while (1) {
 325                 c = getopt(argc, argv, "d:e:ghn:s:vV:");
 326                 if (c == -1)
 327                         break;
 328
 329                 switch (c) {
 330                 case 'd':
 331                         testdir = optarg;
 332                         break;
 333                 case 'e':
 334                         seed = (unsigned int)atoi(optarg);
 335                         break;
 336                 case 'g':
 337                         debug = 1;
 338                         break;
 339                 case 'h':
 340                         usage(argv[0]);
 341                         break;
 342                 case 'n':
 343                         iterations = atoi(optarg);
 344                         break;
 345                 case 's':
 346                         stride = atol(optarg);
 347                         break;
 348                 case 'v':
 349                         verbose += 1;
 350                         break;
 351                 case 'V':
 352                         verbose = atoi(optarg);
 353                         break;
 354                 }
 355         }
 356
 357         if (rank == 0)
 358                 printf("%s is running with %d process(es) %s\n",
 359                        argv[0], size, debug ? "in DEBUG mode" : "\b\b");
 360
 361         if (size < 2) {
 362                 fprintf(stderr,
 363                         "There should be at least 3 process to run the test!\n");
 364                 MPI_Abort(MPI_COMM_WORLD, 2);
 365         }
 366
 367         if (testdir == NULL && rank == 0) {
 368                 fprintf(stderr, "Please specify a test directory! (\"%s -h\" for help)\n",
 369                        argv[0]);
 370                 MPI_Abort(MPI_COMM_WORLD, 2);
 371         }
 372
 373         lp_gethostname();
 374
 375         for (i = 0; i < iterations; ++i) {
 376                 if (rank == 0)
 377                         printf("%s: Running test #%s(iter %d)\n",
 378                                timestamp(), argv[0], i);
 379
 380                 cascading_rw(stride, seed);
 381                 MPI_Barrier(MPI_COMM_WORLD);
 382         }
 383
 384         if (rank == 0)
 385                 printf("%s: All tests passed!\n", timestamp());
 386         MPI_Finalize();
 387         return 0;
 388 }