lustre/tests/mpi/cascading_rw.c

   1 /*
   2  * GPL HEADER START
   3  *
   4  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
   5  *
   6  * This program is free software; you can redistribute it and/or modify
   7  * it under the terms of the GNU General Public License version 2 only,
   8  * as published by the Free Software Foundation.
   9  *
  10  * This program is distributed in the hope that it will be useful, but
  11  * WITHOUT ANY WARRANTY; without even the implied warranty of
  12  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  13  * General Public License version 2 for more details (a copy is included
  14  * in the LICENSE file that accompanied this code).
  15  *
  16  * You should have received a copy of the GNU General Public License
  17  * version 2 along with this program; If not, see
  18  * http://www.sun.com/software/products/lustre/docs/GPLv2.pdf
  19  *
  20  * Please contact Sun Microsystems, Inc., 4150 Network Circle, Santa Clara,
  21  * CA 95054 USA or visit www.sun.com if you need additional information or
  22  * have any questions.
  23  *
  24  * GPL HEADER END
  25  */
  26 /*
  27  * Copyright (c) 2010, Oracle and/or its affiliates. All rights reserved.
  28  * Use is subject to license terms.
  29  */
  30 /*
  31  * This file is part of Lustre, http://www.lustre.org/
  32  * Lustre is a trademark of Sun Microsystems, Inc.
  33  *
  34  * lustre/tests/mpi/cascading_rw.c
  35  *
  36  * Author: You Feng <youfeng@clusterfs.com>
  37  */
  38
  39 #include <config.h>
  40 #include <mpi.h>
  41 #include <stdio.h>
  42 #include <stdlib.h>
  43 #include <string.h>
  44 #include <sys/types.h>
  45 #include <asm/types.h>
  46 #include <sys/stat.h>
  47 #include <fcntl.h>
  48 #include <unistd.h>
  49 #include <sys/ioctl.h>
  50 #include <getopt.h>
  51 #include <errno.h>
  52
  53 #include <libcfs/libcfs.h>
  54 #include "lp_utils.h"
  55 #ifndef _IOWR
  56 # include <ioctl.h>
  57 #endif
  58
  59 #include <lustre/liblustreapi.h>
  60
  61 int rank = 0;
  62 int size = 0;
  63
  64 char *testdir = NULL;
  65
  66 void rw_file(char *name, long stride, unsigned int seed)
  67 {
  68         char filename[MAX_FILENAME_LEN];
  69         char errmsg[MAX_FILENAME_LEN+20];
  70         char *buf, *o_buf;
  71         struct lov_user_md lum = {0};
  72         int fd, rc, i, bad = 0, root = 0;
  73         long off;
  74         long page_size = sysconf(_SC_PAGESIZE);
  75
  76         sprintf(filename, "%s/%s", testdir, name);
  77
  78         if (rank == 0) {
  79                 remove_file_or_dir(filename);
  80
  81                 lum.lmm_magic = LOV_USER_MAGIC;
  82                 lum.lmm_stripe_size = 0;
  83                 lum.lmm_stripe_count = 0;
  84                 lum.lmm_stripe_offset = -1;
  85
  86                 fd = open(filename, O_CREAT | O_RDWR | O_LOV_DELAY_CREATE,
  87                           FILEMODE);
  88                 if (fd == -1) {
  89                         sprintf(errmsg, "open of file %s", filename);
  90                         FAIL(errmsg);
  91                 }
  92
  93                 rc = ioctl(fd, LL_IOC_LOV_SETSTRIPE, &lum);
  94                 if (rc == -1) {
  95                         sprintf(errmsg, "ioctl SETSTRIPE of file %s", filename);
  96                         FAIL(errmsg);
  97                 }
  98
  99                 if (close(fd) == -1) {
 100                         sprintf(errmsg, "close of file %s", filename);
 101                         FAIL(errmsg);
 102                 }
 103         }
 104
 105         MPI_Barrier(MPI_COMM_WORLD);
 106
 107         if (stride < 0) {
 108                 if (rank == 0) {
 109                         srandom(seed);
 110                         while (stride < page_size/2) {
 111                                 stride = random();
 112                                 stride -= stride % 16;
 113                                 if (stride < 0)
 114                                         stride = -stride;
 115                                 stride %= 2 * lum.lmm_stripe_size;
 116                         }
 117                 }
 118
 119                 MPI_Barrier(MPI_COMM_WORLD);
 120
 121                 MPI_Bcast(&stride, 1, MPI_LONG, root, MPI_COMM_WORLD);
 122         }
 123
 124         MPI_Barrier(MPI_COMM_WORLD);
 125
 126         buf = (char *)malloc(stride);
 127         if (buf == NULL) {
 128                 sprintf(errmsg, "malloc of buf with size %ld", stride);
 129                 FAIL(errmsg);
 130         }
 131
 132         if (rank == 0) {
 133                 fd = open(filename, O_RDWR);
 134                 if (fd == -1) {
 135                         sprintf(errmsg, "open of file %s", filename);
 136                         FAIL(errmsg);
 137                 }
 138
 139                 off = 0;
 140                 fill_stride(buf, stride, 0, off);
 141                 rc = write(fd, buf, stride);
 142                 if (rc != stride) {
 143                         sprintf(errmsg, "write of file %s return %d",
 144                                 filename, rc);
 145                         FAIL(errmsg);
 146                 }
 147                 off += stride;
 148
 149                 while (off < size * stride) {
 150                         fill_stride(buf, stride, 0x8080808080808080ULL, off);
 151                         rc = write(fd, buf, stride);
 152                         if (rc != stride) {
 153                                 sprintf(errmsg, "write of file %s return %d",
 154                                         filename, rc);
 155                                 FAIL(errmsg);
 156                         }
 157
 158                         off += stride;
 159                 }
 160
 161                 if (close(fd) == -1) {
 162                         sprintf(errmsg, "close of file %s", filename);
 163                         FAIL(errmsg);
 164                 }
 165         }
 166
 167         MPI_Barrier(MPI_COMM_WORLD);
 168
 169         o_buf = (char *)malloc(stride);
 170         if (o_buf == NULL) {
 171                 sprintf(errmsg, "malloc of o_buf with size %ld", stride);
 172                 FAIL(errmsg);
 173         }
 174
 175         fd = open(filename, O_RDWR);
 176         if (fd == -1) {
 177                 sprintf(errmsg, "open of file %s", filename);
 178                 FAIL(errmsg);
 179         }
 180
 181         off = 0;
 182         for (i = 1; i < size; ++i) {
 183                 if (rank == i) {
 184                         rc = lseek(fd, off, SEEK_SET);
 185                         if (rc != off) {
 186                                 sprintf(errmsg, "lseek of file %s return %d",
 187                                         filename, rc);
 188                                 FAIL(errmsg);
 189                         }
 190
 191                         rc = read(fd, buf, stride);
 192                         if (rc != stride) {
 193                                 if (rc > 0) {
 194                                         fill_stride(o_buf, rc, i - 1, off);
 195                                         dump_diff(o_buf, buf, rc, off);
 196                                 }
 197                                 sprintf(errmsg, "read of file %s return %d",
 198                                         filename, rc);
 199                                 FAIL(errmsg);
 200                         }
 201
 202                         fill_stride(o_buf, stride, i - 1, off);
 203                         if (memcmp(o_buf, buf, stride) != 0) {
 204                                 dump_diff(o_buf, buf, stride, off);
 205                                 errno = 0;
 206                                 sprintf(errmsg, "Error: diff data read from %s",
 207                                         filename);
 208                                 FAIL(errmsg);
 209                         }
 210                 }
 211
 212                 off += stride;
 213
 214                 if (rank == i) {
 215                         fill_stride(buf, stride, i, off);
 216                         rc = write(fd, buf, stride);
 217                         if (rc != stride) {
 218                                 sprintf(errmsg, "write of file %s return %d",
 219                                         filename, rc);
 220                                 FAIL(errmsg);
 221                         }
 222                 }
 223
 224                 MPI_Barrier(MPI_COMM_WORLD);
 225         }
 226
 227         if (close(fd) == -1) {
 228                 sprintf(errmsg, "close of file %s", filename);
 229                 FAIL(errmsg);
 230         }
 231
 232         MPI_Barrier(MPI_COMM_WORLD);
 233
 234         if (rank == 0) {
 235                 fd = open(filename, O_RDONLY);
 236                 if (fd == -1) {
 237                         sprintf(errmsg, "open of file %s", filename);
 238                         FAIL(errmsg);
 239                 }
 240
 241                 off = 0;
 242                 for (i = 0; i < size; ++i) {
 243                         rc = read(fd, buf, stride);
 244                         if (rc != stride) {
 245                                 if (rc > 0) {
 246                                         fill_stride(o_buf, rc, i, off);
 247                                         dump_diff(o_buf, buf, rc, off);
 248                                 }
 249                                 sprintf(errmsg, "read of file %s", filename);
 250                                 FAIL(errmsg);
 251                         }
 252
 253                         fill_stride(o_buf, stride, i, off);
 254                         if (memcmp(o_buf, buf, stride) != 0) {
 255                                 bad = 1;
 256                                 dump_diff(o_buf, buf, stride, off);
 257                         }
 258                         off += stride;
 259                 }
 260                 if (bad == 1) {
 261                         errno = 0;
 262                         sprintf(errmsg, "Error: diff data read from %s", filename);
 263                         FAIL(errmsg);
 264                 }
 265         }
 266
 267         MPI_Barrier(MPI_COMM_WORLD);
 268         fprintf(stderr, "passed barrier 5\n");
 269
 270         free(buf);
 271         free(o_buf);
 272 }
 273
 274 void cascading_rw(long stride, unsigned int seed)
 275 {
 276         begin("setup");
 277         end("setup");
 278
 279         begin("test");
 280         rw_file("cascading_rw", stride, seed);
 281         end("test");
 282
 283         begin("cleanup");
 284         remove_file("cascading_rw");
 285         end("cleanup");
 286 }
 287
 288 void usage(char *proc)
 289 {
 290         int i;
 291
 292         if (rank == 0) {
 293                 printf("Usage: %s [-h] -d <testdir> [-s \"1024\"]\n", proc);
 294                 printf("           [-n \"13\"] [-e \"12345\"]\n");
 295                 printf("           [-v] [-V #] [-g]\n");
 296                 printf("\t-h: prints this help message\n");
 297                 printf("\t-d: the directory in which the tests will run\n");
 298                 printf("\t-s: process stride size\n");
 299                 printf("\t-n: repeat test # times\n");
 300                 printf("\t-n: random seed, used to re-create previous runs\n");
 301                 printf("\t-v: increase the verbositly level by 1\n");
 302                 printf("\t-V: select a specific verbosity level\n");
 303                 printf("\t-g: debug mode\n");
 304         }
 305
 306         MPI_Initialized(&i);
 307         if (i) MPI_Finalize();
 308         exit(0);
 309 }
 310
 311 int main(int argc, char *argv[])
 312 {
 313         int i, iterations = 16, c;
 314         long stride = -1;
 315         unsigned int seed = 0;
 316
 317         /* Check for -h parameter before MPI_Init so the binary can be
 318            called directly, without, for instance, mpirun */
 319         for (i = 1; i < argc; ++i) {
 320                 if (!strcmp(argv[i], "-h") || !strcmp(argv[i], "--help"))
 321                         usage(argv[0]);
 322         }
 323
 324         MPI_Init(&argc, &argv);
 325         MPI_Comm_rank(MPI_COMM_WORLD, &rank);
 326         MPI_Comm_size(MPI_COMM_WORLD, &size);
 327
 328                 /* Parse command line options */
 329         while (1) {
 330                 c = getopt(argc, argv, "d:e:ghn:s:vV:");
 331                 if (c == -1)
 332                         break;
 333
 334                 switch (c) {
 335                 case 'd':
 336                         testdir = optarg;
 337                         break;
 338                 case 'e':
 339                         seed = (unsigned int)atoi(optarg);
 340                         break;
 341                 case 'g':
 342                         debug = 1;
 343                         break;
 344                 case 'h':
 345                         usage(argv[0]);
 346                         break;
 347                 case 'n':
 348                         iterations = atoi(optarg);
 349                         break;
 350                 case 's':
 351                         stride = atol(optarg);
 352                         break;
 353                 case 'v':
 354                         verbose += 1;
 355                         break;
 356                 case 'V':
 357                         verbose = atoi(optarg);
 358                         break;
 359                 }
 360         }
 361
 362         if (rank == 0)
 363                 printf("%s is running with %d process(es) %s\n",
 364                        argv[0], size, debug ? "in DEBUG mode" : "\b\b");
 365
 366         if (size < 2) {
 367                 fprintf(stderr,
 368                         "There should be at least 3 process to run the test!\n");
 369                 MPI_Abort(MPI_COMM_WORLD, 2);
 370         }
 371
 372         if (testdir == NULL && rank == 0) {
 373                 fprintf(stderr, "Please specify a test directory! (\"%s -h\" for help)\n",
 374                        argv[0]);
 375                 MPI_Abort(MPI_COMM_WORLD, 2);
 376         }
 377
 378         lp_gethostname();
 379
 380         for (i = 0; i < iterations; ++i) {
 381                 if (rank == 0)
 382                         printf("%s: Running test #%s(iter %d)\n",
 383                                timestamp(), argv[0], i);
 384
 385                 cascading_rw(stride, seed);
 386                 MPI_Barrier(MPI_COMM_WORLD);
 387         }
 388
 389         if (rank == 0)
 390                 printf("%s: All tests passed!\n", timestamp());
 391         MPI_Finalize();
 392         return 0;
 393 }