lustre/tests/mpi/cascading_rw.c

   1 /* -*- mode: c; c-basic-offset: 8; indent-tabs-mode: nil; -*-
   2  * vim:expandtab:shiftwidth=8:tabstop=8:
   3  *
   4  * GPL HEADER START
   5  *
   6  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
   7  *
   8  * This program is free software; you can redistribute it and/or modify
   9  * it under the terms of the GNU General Public License version 2 only,
  10  * as published by the Free Software Foundation.
  11  *
  12  * This program is distributed in the hope that it will be useful, but
  13  * WITHOUT ANY WARRANTY; without even the implied warranty of
  14  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  15  * General Public License version 2 for more details (a copy is included
  16  * in the LICENSE file that accompanied this code).
  17  *
  18  * You should have received a copy of the GNU General Public License
  19  * version 2 along with this program; If not, see
  20  * http://www.sun.com/software/products/lustre/docs/GPLv2.pdf
  21  *
  22  * Please contact Sun Microsystems, Inc., 4150 Network Circle, Santa Clara,
  23  * CA 95054 USA or visit www.sun.com if you need additional information or
  24  * have any questions.
  25  *
  26  * GPL HEADER END
  27  */
  28 /*
  29  * Copyright  2008 Sun Microsystems, Inc. All rights reserved
  30  * Use is subject to license terms.
  31  */
  32 /*
  33  * This file is part of Lustre, http://www.lustre.org/
  34  * Lustre is a trademark of Sun Microsystems, Inc.
  35  *
  36  * lustre/tests/mpi/cascading_rw.c
  37  *
  38  * Author: You Feng <youfeng@clusterfs.com>
  39  */
  40
  41 #include <config.h>
  42 #include <mpi.h>
  43 #include <stdio.h>
  44 #include <stdlib.h>
  45 #include <string.h>
  46 #include <sys/types.h>
  47 #include <asm/types.h>
  48 #include <sys/stat.h>
  49 #include <fcntl.h>
  50 #include <unistd.h>
  51 #include <sys/ioctl.h>
  52 #include <getopt.h>
  53 #include <errno.h>
  54
  55 #include <libcfs/libcfs.h>
  56 #include "lp_utils.h"
  57 #ifndef _IOWR
  58 # include <ioctl.h>
  59 #endif
  60
  61 #include <lustre/liblustreapi.h>
  62
  63 int rank = 0;
  64 int size = 0;
  65
  66 char *testdir = NULL;
  67
  68 void rw_file(char *name, long stride, unsigned int seed)
  69 {
  70         char filename[MAX_FILENAME_LEN];
  71         char path[MAX_FILENAME_LEN];
  72         char errmsg[MAX_FILENAME_LEN+20];
  73         char *buf, *o_buf;
  74         struct lov_user_md lum = {0};
  75         int fd, rc, i, bad = 0, root = 0;
  76         long off;
  77         long page_size = sysconf(_SC_PAGESIZE);
  78
  79         sprintf(filename, "%s/%s", testdir, name);
  80
  81         if (rank == 0) {
  82                 remove_file_or_dir(filename);
  83
  84                 lum.lmm_magic = LOV_USER_MAGIC;
  85                 lum.lmm_stripe_size = 0;
  86                 lum.lmm_stripe_count = 0;
  87                 lum.lmm_stripe_offset = -1;
  88
  89                 fd = open(filename, O_CREAT | O_RDWR | O_LOV_DELAY_CREATE,
  90                           FILEMODE);
  91                 if (fd == -1) {
  92                         sprintf(errmsg, "open of file %s", filename);
  93                         FAIL(errmsg);
  94                 }
  95
  96                 rc = ioctl(fd, LL_IOC_LOV_SETSTRIPE, &lum);
  97                 if (rc == -1) {
  98                         sprintf(errmsg, "ioctl SETSTRIPE of file %s", filename);
  99                         FAIL(errmsg);
 100                 }
 101
 102                 if (close(fd) == -1) {
 103                         sprintf(errmsg, "close of file %s", filename);
 104                         FAIL(errmsg);
 105                 }
 106
 107                 strncpy(path, filename, MAX_FILENAME_LEN);
 108                 rc = llapi_file_get_stripe(path, &lum);
 109                 if (rc == -1) {
 110                         sprintf(errmsg, "get stripe of file %s", filename);
 111                         FAIL(errmsg);
 112                 }
 113         }
 114
 115         MPI_Barrier(MPI_COMM_WORLD);
 116
 117         if (stride < 0) {
 118                 if (rank == 0) {
 119                         srandom(seed);
 120                         while (stride < page_size/2) {
 121                                 stride = random();
 122                                 stride -= stride % 16;
 123                                 if (stride < 0)
 124                                         stride = -stride;
 125                                 stride %= 2 * lum.lmm_stripe_size;
 126                         }
 127                 }
 128
 129                 MPI_Barrier(MPI_COMM_WORLD);
 130
 131                 MPI_Bcast(&stride, 1, MPI_LONG, root, MPI_COMM_WORLD);
 132         }
 133
 134         MPI_Barrier(MPI_COMM_WORLD);
 135
 136         buf = (char *)malloc(stride);
 137         if (buf == NULL) {
 138                 sprintf(errmsg, "malloc of buf with size %ld", stride);
 139                 FAIL(errmsg);
 140         }
 141
 142         if (rank == 0) {
 143                 fd = open(filename, O_RDWR);
 144                 if (fd == -1) {
 145                         sprintf(errmsg, "open of file %s", filename);
 146                         FAIL(errmsg);
 147                 }
 148
 149                 off = 0;
 150                 fill_stride(buf, stride, 0, off);
 151                 rc = write(fd, buf, stride);
 152                 if (rc != stride) {
 153                         sprintf(errmsg, "write of file %s return %d",
 154                                 filename, rc);
 155                         FAIL(errmsg);
 156                 }
 157                 off += stride;
 158
 159                 while (off < size * stride) {
 160                         fill_stride(buf, stride, 0x8080808080808080ULL, off);
 161                         rc = write(fd, buf, stride);
 162                         if (rc != stride) {
 163                                 sprintf(errmsg, "write of file %s return %d",
 164                                         filename, rc);
 165                                 FAIL(errmsg);
 166                         }
 167
 168                         off += stride;
 169                 }
 170
 171                 if (close(fd) == -1) {
 172                         sprintf(errmsg, "close of file %s", filename);
 173                         FAIL(errmsg);
 174                 }
 175         }
 176
 177         MPI_Barrier(MPI_COMM_WORLD);
 178
 179         o_buf = (char *)malloc(stride);
 180         if (o_buf == NULL) {
 181                 sprintf(errmsg, "malloc of o_buf with size %ld", stride);
 182                 FAIL(errmsg);
 183         }
 184
 185         fd = open(filename, O_RDWR);
 186         if (fd == -1) {
 187                 sprintf(errmsg, "open of file %s", filename);
 188                 FAIL(errmsg);
 189         }
 190
 191         off = 0;
 192         for (i = 1; i < size; ++i) {
 193                 if (rank == i) {
 194                         rc = lseek(fd, off, SEEK_SET);
 195                         if (rc != off) {
 196                                 sprintf(errmsg, "lseek of file %s return %d",
 197                                         filename, rc);
 198                                 FAIL(errmsg);
 199                         }
 200
 201                         rc = read(fd, buf, stride);
 202                         if (rc != stride) {
 203                                 if (rc > 0) {
 204                                         fill_stride(o_buf, rc, i - 1, off);
 205                                         dump_diff(o_buf, buf, rc, off);
 206                                 }
 207                                 sprintf(errmsg, "read of file %s return %d",
 208                                         filename, rc);
 209                                 FAIL(errmsg);
 210                         }
 211
 212                         fill_stride(o_buf, stride, i - 1, off);
 213                         if (memcmp(o_buf, buf, stride) != 0) {
 214                                 dump_diff(o_buf, buf, stride, off);
 215                                 errno = 0;
 216                                 sprintf(errmsg, "Error: diff data read from %s",
 217                                         filename);
 218                                 FAIL(errmsg);
 219                         }
 220                 }
 221
 222                 off += stride;
 223
 224                 if (rank == i) {
 225                         fill_stride(buf, stride, i, off);
 226                         rc = write(fd, buf, stride);
 227                         if (rc != stride) {
 228                                 sprintf(errmsg, "write of file %s return %d",
 229                                         filename, rc);
 230                                 FAIL(errmsg);
 231                         }
 232                 }
 233
 234                 MPI_Barrier(MPI_COMM_WORLD);
 235         }
 236
 237         if (close(fd) == -1) {
 238                 sprintf(errmsg, "close of file %s", filename);
 239                 FAIL(errmsg);
 240         }
 241
 242         MPI_Barrier(MPI_COMM_WORLD);
 243
 244         if (rank == 0) {
 245                 fd = open(filename, O_RDONLY);
 246                 if (fd == -1) {
 247                         sprintf(errmsg, "open of file %s", filename);
 248                         FAIL(errmsg);
 249                 }
 250
 251                 off = 0;
 252                 for (i = 0; i < size; ++i) {
 253                         rc = read(fd, buf, stride);
 254                         if (rc != stride) {
 255                                 if (rc > 0) {
 256                                         fill_stride(o_buf, rc, i, off);
 257                                         dump_diff(o_buf, buf, rc, off);
 258                                 }
 259                                 sprintf(errmsg, "read of file %s", filename);
 260                                 FAIL(errmsg);
 261                         }
 262
 263                         fill_stride(o_buf, stride, i, off);
 264                         if (memcmp(o_buf, buf, stride) != 0) {
 265                                 bad = 1;
 266                                 dump_diff(o_buf, buf, stride, off);
 267                         }
 268                         off += stride;
 269                 }
 270                 if (bad == 1) {
 271                         errno = 0;
 272                         sprintf(errmsg, "Error: diff data read from %s", filename);
 273                         FAIL(errmsg);
 274                 }
 275         }
 276
 277         MPI_Barrier(MPI_COMM_WORLD);
 278         fprintf(stderr, "passed barrier 5\n");
 279
 280         free(buf);
 281         free(o_buf);
 282 }
 283
 284 void cascading_rw(long stride, unsigned int seed)
 285 {
 286         begin("setup");
 287         end("setup");
 288
 289         begin("test");
 290         rw_file("cascading_rw", stride, seed);
 291         end("test");
 292
 293         begin("cleanup");
 294         remove_file("cascading_rw");
 295         end("cleanup");
 296 }
 297
 298 void usage(char *proc)
 299 {
 300         int i;
 301
 302         if (rank == 0) {
 303                 printf("Usage: %s [-h] -d <testdir> [-s \"1024\"]\n", proc);
 304                 printf("           [-n \"13\"] [-e \"12345\"]\n");
 305                 printf("           [-v] [-V #] [-g]\n");
 306                 printf("\t-h: prints this help message\n");
 307                 printf("\t-d: the directory in which the tests will run\n");
 308                 printf("\t-s: process stride size\n");
 309                 printf("\t-n: repeat test # times\n");
 310                 printf("\t-n: random seed, used to re-create previous runs\n");
 311                 printf("\t-v: increase the verbositly level by 1\n");
 312                 printf("\t-V: select a specific verbosity level\n");
 313                 printf("\t-g: debug mode\n");
 314         }
 315
 316         MPI_Initialized(&i);
 317         if (i) MPI_Finalize();
 318         exit(0);
 319 }
 320
 321 int main(int argc, char *argv[])
 322 {
 323         int i, iterations = 16, c;
 324         long stride = -1;
 325         unsigned int seed = 0;
 326
 327         /* Check for -h parameter before MPI_Init so the binary can be
 328            called directly, without, for instance, mpirun */
 329         for (i = 1; i < argc; ++i) {
 330                 if (!strcmp(argv[i], "-h") || !strcmp(argv[i], "--help"))
 331                         usage(argv[0]);
 332         }
 333
 334         MPI_Init(&argc, &argv);
 335         MPI_Comm_rank(MPI_COMM_WORLD, &rank);
 336         MPI_Comm_size(MPI_COMM_WORLD, &size);
 337
 338                 /* Parse command line options */
 339         while (1) {
 340                 c = getopt(argc, argv, "d:e:ghn:s:vV:");
 341                 if (c == -1)
 342                         break;
 343
 344                 switch (c) {
 345                 case 'd':
 346                         testdir = optarg;
 347                         break;
 348                 case 'e':
 349                         seed = (unsigned int)atoi(optarg);
 350                         break;
 351                 case 'g':
 352                         debug = 1;
 353                         break;
 354                 case 'h':
 355                         usage(argv[0]);
 356                         break;
 357                 case 'n':
 358                         iterations = atoi(optarg);
 359                         break;
 360                 case 's':
 361                         stride = atol(optarg);
 362                         break;
 363                 case 'v':
 364                         verbose += 1;
 365                         break;
 366                 case 'V':
 367                         verbose = atoi(optarg);
 368                         break;
 369                 }
 370         }
 371
 372         if (rank == 0)
 373                 printf("%s is running with %d process(es) %s\n",
 374                        argv[0], size, debug ? "in DEBUG mode" : "\b\b");
 375
 376         if (size < 2) {
 377                 fprintf(stderr,
 378                         "There should be at least 3 process to run the test!\n");
 379                 MPI_Abort(MPI_COMM_WORLD, 2);
 380         }
 381
 382         if (testdir == NULL && rank == 0) {
 383                 fprintf(stderr, "Please specify a test directory! (\"%s -h\" for help)\n",
 384                        argv[0]);
 385                 MPI_Abort(MPI_COMM_WORLD, 2);
 386         }
 387
 388         lp_gethostname();
 389
 390         for (i = 0; i < iterations; ++i) {
 391                 if (rank == 0)
 392                         printf("%s: Running test #%s(iter %d)\n",
 393                                timestamp(), argv[0], i);
 394
 395                 cascading_rw(stride, seed);
 396                 MPI_Barrier(MPI_COMM_WORLD);
 397         }
 398
 399         if (rank == 0)
 400                 printf("%s: All tests passed!\n", timestamp());
 401         MPI_Finalize();
 402         return 0;
 403 }