lustre/tests/mpi/write_disjoint.c

   1 /*
   2  * GPL HEADER START
   3  *
   4  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
   5  *
   6  * This program is free software; you can redistribute it and/or modify
   7  * it under the terms of the GNU General Public License version 2 only,
   8  * as published by the Free Software Foundation.
   9  *
  10  * This program is distributed in the hope that it will be useful, but
  11  * WITHOUT ANY WARRANTY; without even the implied warranty of
  12  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  13  * General Public License version 2 for more details (a copy is included
  14  * in the LICENSE file that accompanied this code).
  15  *
  16  * You should have received a copy of the GNU General Public License
  17  * version 2 along with this program; If not, see
  18  * http://www.gnu.org/licenses/gpl-2.0.html
  19  *
  20  * GPL HEADER END
  21  */
  22 /*
  23  * Copyright (c) 2003, 2010, Oracle and/or its affiliates. All rights reserved.
  24  * Use is subject to license terms.
  25  *
  26  * Copyright (c) 2012, Intel Corporation.
  27  */
  28 /*
  29  * This file is part of Lustre, http://www.lustre.org/
  30  * Lustre is a trademark of Sun Microsystems, Inc.
  31  *
  32  * lustre/tests/write_disjoint.c
  33  *
  34  * Each loop does 3 things:
  35  *   - rank 0 truncates to 0
  36  *   - all ranks agree on a random chunk size
  37  *   - all ranks race to write their pattern to their chunk of the file
  38  *   - rank 0 makes sure that the resulting file size is ranks * chunk size
  39  *   - rank 0 makes sure that everyone's patterns went to the right place
  40  *
  41  * compile: mpicc -g -Wall -o write_disjoint write_disjoint.c
  42  * run:     mpirun -np N -machlist <hostlist file> write_disjoint
  43  *  or:     pdsh -w <N hosts> write_disjoint
  44  *  or:     prun -n N [-N M] write_disjoint
  45  */
  46
  47 #include <stdlib.h>
  48 #include <stdio.h>
  49 #include <sys/types.h>
  50 #include <sys/stat.h>
  51 #include <fcntl.h>
  52 #include <string.h>
  53 #include <errno.h>
  54 #include <unistd.h>
  55 #include <stdarg.h>
  56 #include <time.h>
  57 #include "mpi.h"
  58
  59 /* Chosen arbitrarily.  Actually running this large will take a long time.*/
  60 #define CHUNK_MAX_SIZE (1024*1024*16)
  61
  62 void rprintf(int rank, int loop, const char *fmt, ...)
  63 {
  64         va_list       ap;
  65
  66         printf("rank %d, loop %d: ", rank, loop);
  67
  68         va_start(ap, fmt);
  69
  70         vprintf(fmt, ap);
  71
  72         MPI_Abort(MPI_COMM_WORLD, -1); /* This will exit() according to man */
  73 }
  74
  75 #define CHUNK_SIZE(n) chunk_size[(n) % 2]
  76
  77 int main (int argc, char *argv[]) {
  78         int i, n, fd, c;
  79         unsigned long chunk_size[2];
  80         int rank, noProcessors, done;
  81         int error;
  82         off_t offset;
  83         char **chunk_buf;
  84         char *read_buf;
  85         struct stat stat_buf;
  86         ssize_t ret;
  87         char *filename = "/mnt/lustre/write_disjoint";
  88         int numloops = 1000;
  89         int max_size = CHUNK_MAX_SIZE;
  90         int random = 0;
  91         unsigned int seed = 0;
  92         int seed_provided = 0;
  93
  94         error = MPI_Init(&argc, &argv);
  95         if (error != MPI_SUCCESS)
  96                 rprintf(-1, -1, "MPI_Init failed: %d\n", error);
  97         /* Parse command line options */
  98         while ((c = getopt(argc, argv, "f:n:m:s:")) != EOF) {
  99                 errno = 0;
 100                 switch (c) {
 101                 case 'f':
 102                         filename = optarg;
 103                         break;
 104                 case 'n':
 105                         numloops = strtoul(optarg, NULL, 0);
 106                         break;
 107                 case 'm':
 108                         max_size = strtoul(optarg, NULL, 0);
 109                         if (max_size > CHUNK_MAX_SIZE)
 110                                 rprintf(-1, -1, "Chunk size larger than %d.\n",
 111                                         CHUNK_MAX_SIZE);
 112                         break;
 113                 case 's':
 114                         seed = strtoul(optarg, NULL, 0);
 115                         seed_provided = 1;
 116                         break;
 117                 }
 118         }
 119
 120         MPI_Comm_size(MPI_COMM_WORLD, &noProcessors);
 121         MPI_Comm_rank(MPI_COMM_WORLD, &rank);
 122
 123         chunk_buf = malloc(noProcessors * sizeof(chunk_buf[0]));
 124         for (i=0; i < noProcessors; i++) {
 125                 chunk_buf[i] = malloc(max_size);
 126                 memset(chunk_buf[i], 'A' + i, max_size);
 127         }
 128         read_buf = malloc(noProcessors * max_size);
 129
 130         if (rank == 0) {
 131                 fd = open(filename, O_WRONLY|O_CREAT|O_TRUNC, 0666);
 132                 if (fd < 0)
 133                         rprintf(rank, -1, "open() returned %s\n",
 134                                 strerror(errno));
 135         }
 136         MPI_Barrier(MPI_COMM_WORLD);
 137
 138         fd = open(filename, O_RDWR);
 139         if (fd < 0)
 140                 rprintf(rank, -1, "open() returned %s\n", strerror(errno));
 141
 142         if (rank == 0) {
 143                 if (!seed_provided)
 144                         seed = (unsigned int) time(NULL);
 145                 printf("random seed: %d\n", seed);
 146                 srand(seed);
 147         }
 148
 149
 150         for (n = 0; n < numloops; n++) {
 151                 /* reset the environment */
 152                 if (rank == 0) {
 153                         ret = truncate(filename, 0);
 154                         if (ret != 0)
 155                                 rprintf(rank, n, "truncate() returned %s\n",
 156                                         strerror(errno) );
 157
 158                         random = rand();
 159                 }
 160                 MPI_Bcast(&random, 1, MPI_INT, 0, MPI_COMM_WORLD);
 161                 CHUNK_SIZE(n) = random % max_size;
 162
 163                 if (n % 1000 == 0 && rank == 0)
 164                         printf("loop %d: chunk_size %lu\n", n, CHUNK_SIZE(n));
 165
 166                 if (stat(filename, &stat_buf) < 0)
 167                         rprintf(rank, n, "error stating %s: %s\n",
 168                                 filename, strerror(errno));
 169
 170                 if (stat_buf.st_size != 0)
 171                         rprintf(rank, n, "filesize = %lu. "
 172                                 "Should be zero after truncate\n",
 173                                 stat_buf.st_size);
 174
 175                 MPI_Barrier(MPI_COMM_WORLD);
 176
 177                 /* Do the race */
 178                 offset = rank * CHUNK_SIZE(n);
 179                 lseek(fd, offset, SEEK_SET);
 180
 181                 done = 0;
 182                 do {
 183                         ret = write(fd, chunk_buf[rank] + done,
 184                                     CHUNK_SIZE(n) - done);
 185                         if (ret < 0 && errno != EINTR)
 186                                 rprintf(rank, n, "write() returned %s\n",
 187                                         strerror(errno));
 188                         if (ret > 0)
 189                                 done += ret;
 190                 } while (done != CHUNK_SIZE(n));
 191
 192                 MPI_Barrier(MPI_COMM_WORLD);
 193
 194                 /* Check the result */
 195                 if (stat(filename, &stat_buf) < 0)
 196                         rprintf(rank, n, "error stating %s: %s\n",
 197                                 filename, strerror(errno));
 198
 199                 if (stat_buf.st_size != CHUNK_SIZE(n) * noProcessors) {
 200                         if (n > 0)
 201                                 printf("loop %d: chunk_size %lu, "
 202                                        "file size was %lu\n",
 203                                        n - 1, CHUNK_SIZE(n - 1),
 204                                        CHUNK_SIZE(n - 1) *noProcessors);
 205                         rprintf(rank, n, "invalid file size %lu"
 206                                 " instead of %lu = %lu * %u\n",
 207                                 (unsigned long)stat_buf.st_size,
 208                                 CHUNK_SIZE(n) * noProcessors,
 209                                 CHUNK_SIZE(n), noProcessors);
 210                 }
 211
 212                 if (rank == 0) {
 213                         if (lseek(fd, 0, SEEK_SET) < 0)
 214                                 rprintf(rank, n, "error seeking to 0: %s\n",
 215                                         strerror(errno));
 216
 217                         done = 0;
 218                         do {
 219                                 ret = read(fd, read_buf + done,
 220                                            CHUNK_SIZE(n) * noProcessors - done);
 221                                 if (ret < 0)
 222                                         rprintf(rank, n, "read returned %s\n",
 223                                                 strerror(errno));
 224
 225                                 done += ret;
 226                         } while (done != CHUNK_SIZE(n) * noProcessors);
 227
 228                         for (i = 0; i < noProcessors; i++) {
 229                                 char command[4096];
 230                                 int j;
 231
 232                                 if (!memcmp(read_buf + (i * CHUNK_SIZE(n)),
 233                                             chunk_buf[i], CHUNK_SIZE(n)))
 234                                         continue;
 235
 236                                 /* print out previous chunk sizes */
 237                                 if (n > 0)
 238                                         printf("loop %d: chunk_size %lu\n",
 239                                                n - 1, CHUNK_SIZE(n - 1));
 240
 241                                 printf("loop %d: chunk %d corrupted "
 242                                        "with chunk_size %lu, page_size %d\n",
 243                                        n, i, CHUNK_SIZE(n), getpagesize());
 244                                 printf("ranks:\tpage boundry\tchunk boundry\t"
 245                                        "page boundry\n");
 246                                 for (j = 1 ; j < noProcessors; j++) {
 247                                         int b = j * CHUNK_SIZE(n);
 248                                         printf("%c -> %c:\t%d\t%d\t%d\n",
 249                                                'A' + j - 1, 'A' + j,
 250                                                b & ~(getpagesize()-1), b,
 251                                                (b + getpagesize()) &
 252                                                ~(getpagesize()-1));
 253                                 }
 254
 255                                 sprintf(command, "od -Ad -a %s", filename);
 256                                 ret = system(command);
 257                                 rprintf(0, n, "data check error - exiting\n");
 258                         }
 259                 }
 260                 MPI_Barrier(MPI_COMM_WORLD);
 261         }
 262
 263         printf("Finished after %d loops\n", n);
 264         MPI_Finalize();
 265         return 0;
 266 }