lustre/tests/write_append_truncate.c

   1 /* -*- mode: c; c-basic-offset: 8; indent-tabs-mode: nil; -*-
   2  * vim:expandtab:shiftwidth=8:tabstop=8:
   3  *
   4  * Each loop does 3 things:
   5  *   - truncate file to zero (not via ftruncate though, to test O_APPEND)
   6  *   - append a "chunk" of data (should be at file offset 0 after truncate)
   7  *   - on each of two threads either append or truncate-up the file
   8  *
   9  * If the truncate happened first, we should have a hole in the file.
  10  * If the append happened first, we should have truncated the file down.
  11  *
  12  * We pick the CHUNK_SIZE_MAX and APPEND_SIZE_MAX so that we cross a stripe.
  13  *
  14  * compile: mpicc -g -Wall -o write_append_truncate write_append_truncate.c
  15  * run:     mpirun -np 2 -machlist <hostlist file> write_append_truncate <file>
  16  *  or:     pdsh -w <two hosts> write_append_truncate <file>
  17  *  or:     prun -n 2 [-N 2] write_append_truncate <file>
  18  */
  19 #include <stdlib.h>
  20 #include <stdio.h>
  21 #include <stdarg.h>
  22 #include <sys/types.h>
  23 #include <sys/stat.h>
  24 #include <fcntl.h>
  25 #include <errno.h>
  26 #include <string.h>
  27 #include <unistd.h>
  28 #include "mpi.h"
  29
  30 #define DEFAULT_ITER     50000
  31
  32 #define CHUNK_SIZE_MAX   123456
  33 #define CHUNK_CHAR   'C'
  34
  35 #define APPEND_SIZE_MAX  123456
  36 #define APPEND_CHAR  'A'
  37
  38 #define TRUNC_SIZE_MAX   (CHUNK_SIZE_MAX+APPEND_SIZE_MAX)
  39
  40 #define HOSTNAME_SIZE 50
  41
  42 void usage(char *prog)
  43 {
  44         printf("usage: %s <filename> [nloops]\n", prog);
  45         printf("%s must be run with at least 2 processes\n", prog);
  46
  47         MPI_Finalize();
  48         exit(1);
  49 }
  50
  51 /* Print process rank, loop count, message, and exit (i.e. a fatal error) */
  52 int rprintf(int rank, int loop, const char *fmt, ...)
  53 {
  54         va_list       ap;
  55
  56         printf("rank %d, loop %d: ", rank, loop);
  57
  58         va_start(ap, fmt);
  59
  60         vprintf(fmt, ap);
  61
  62         MPI_Finalize();
  63         exit(1);
  64 }
  65
  66 int main(int argc, char *argv[])
  67 {
  68         int n, nloops = 0, fd;
  69         int rank, size, ret;
  70         int chunk_size, append_size, trunc_offset;
  71         char append_buf[APPEND_SIZE_MAX];
  72         char chunk_buf[CHUNK_SIZE_MAX];
  73         char read_buf[TRUNC_SIZE_MAX+APPEND_SIZE_MAX];
  74         char trunc_buf[TRUNC_SIZE_MAX];
  75         int done;
  76         int error;
  77         char hostname[HOSTNAME_SIZE];
  78         char *fname, *prog;
  79
  80         error = MPI_Init(&argc, &argv);
  81         if (error != MPI_SUCCESS)
  82                 rprintf(-1, -1, "MPI_Init failed: %d\n", error);
  83
  84         prog = strrchr(argv[0], '/');
  85         if (prog == NULL)
  86                 prog = argv[0];
  87         else
  88                 prog++;
  89
  90         if (argc < 2 || argc > 3)
  91                 usage(prog);
  92
  93         error = MPI_Comm_rank(MPI_COMM_WORLD, &rank);
  94         if (error != MPI_SUCCESS)
  95                 rprintf(-1, -1, "MPI_Comm_rank failed: %d\n", error);
  96
  97         error = MPI_Comm_size(MPI_COMM_WORLD, &size);
  98         if (error != MPI_SUCCESS)
  99                 rprintf(rank, -1, "MPI_Comm_size failed: %d\n", error);
 100
 101         if (size < 2)
 102                 rprintf(rank, -1, "%s: must run with at least 2 processes\n",
 103                         prog);
 104
 105         memset(append_buf, APPEND_CHAR, APPEND_SIZE_MAX);
 106         memset(chunk_buf, CHUNK_CHAR, CHUNK_SIZE_MAX);
 107         memset(trunc_buf, 0, TRUNC_SIZE_MAX);
 108
 109         if (gethostname(hostname, HOSTNAME_SIZE) < 0)
 110                 rprintf(rank, -1, "gethostname failed: %s\n", strerror(errno));
 111
 112         fname = argv[1];
 113
 114         if (argc == 3)
 115                 nloops = strtoul(argv[2], NULL, 0);
 116         if (nloops == 0)
 117                 nloops = DEFAULT_ITER;
 118
 119         if (rank == 0) {
 120                 fd = open(fname, O_WRONLY|O_CREAT|O_TRUNC, 0666);
 121                 if (fd < 0)
 122                         rprintf(0, -1, "create %s failed: %s\n", fname,
 123                                 strerror(errno));
 124                 printf("using %s\n", fname);
 125         }
 126         error = MPI_Barrier(MPI_COMM_WORLD);
 127         if (error != MPI_SUCCESS)
 128                 rprintf(rank, -1, "prep MPI_Barrier failed: %d\n", error);
 129
 130         fd = open(fname, O_RDWR | O_APPEND);
 131         if (fd < 0)
 132                 rprintf(rank, -1, "open %s failed: %s\n",fname,strerror(errno));
 133
 134         for (n = 0; n < nloops; n++) {
 135                 /* reset the environment */
 136                 chunk_size = (rand()%(CHUNK_SIZE_MAX-1))+1;
 137                 append_size = (rand()%(APPEND_SIZE_MAX-1))+1;
 138                 trunc_offset = chunk_size + rand()%append_size;
 139                 if (rank == 0) {
 140                         if (n % 1000 == 0)
 141                                 printf("loop %5d: chunk %6d/%#07x, "
 142                                        "append %6d/%#07x, trunc @ %6d/%#07x\n",
 143                                        n, chunk_size, chunk_size, append_size,
 144                                        append_size, trunc_offset, trunc_offset);
 145
 146                         ret = truncate(fname, (off_t)0);
 147                         if (ret < 0)
 148                                 rprintf(0, n, "truncate @ 0: %s\n",
 149                                         strerror(errno));
 150                         done = 0;
 151                         do {
 152                                 ret = write(fd, chunk_buf+done,chunk_size-done);
 153                                 if (ret <= 0) {
 154                                         rprintf(0, n, "chunk @ %d: %s\n",
 155                                                 done, strerror(errno));
 156                                         break;
 157                                 }
 158                                 done += ret;
 159                         } while (done != chunk_size);
 160                 }
 161
 162                 error = MPI_Barrier(MPI_COMM_WORLD);
 163                 if (error != MPI_SUCCESS)
 164                         rprintf(rank, n, "start MPI_Barrier: %d\n",error);
 165
 166                 /* Do the race */
 167                 if (rank == n % size) {
 168                         //
 169                         done = 0;
 170                         do {
 171                                 ret = write(fd, append_buf + done,
 172                                             append_size - done);
 173                                 if (ret < 0) {
 174                                         rprintf(rank, n,
 175                                                 "loop %d: append @ %u: %s\n",
 176                                                 done, strerror(errno));
 177                                         break;
 178                                 }
 179                                 done += ret;
 180                         } while (done != append_size);
 181                 } else if (rank == (n + 1) % size) {
 182                         ret = truncate(fname, (off_t)trunc_offset);
 183                         if (ret != 0)
 184                                 rprintf(rank, n, "truncate @ %u: %s\n",
 185                                         trunc_offset, strerror(errno) );
 186                 }
 187
 188                 error = MPI_Barrier(MPI_COMM_WORLD);
 189                 if (error != MPI_SUCCESS)
 190                         rprintf(rank, n, "end MPI_Barrier: %d\n", error);
 191
 192                 error = 0;
 193
 194                 /* Check the result */
 195                 if (rank == 0) {
 196                         struct stat st;
 197                         if (stat(fname, &st) < 0)
 198                                 rprintf(0, n, "loop %d: stat %s: %s\n",
 199                                         fname, strerror(errno));
 200
 201                         if (lseek(fd, (off_t)0, SEEK_SET) != 0)
 202                                 rprintf(0, n, "lseek fname 0: %s\n", fname,
 203                                         strerror(errno));
 204
 205                         done = 0;
 206                         do {
 207                                 ret = read(fd, read_buf+done, st.st_size-done);
 208                                 if (ret < 0) {
 209                                         rprintf(0, n, "read @ %u: %s\n",
 210                                                done, strerror(errno));
 211                                 }
 212                                 done += ret;
 213                         } while (done != st.st_size);
 214
 215                         if (memcmp(read_buf, chunk_buf, chunk_size)) {
 216                                 printf("loop %d: base chunk bad"
 217                                        " [0-%d]/[0-%#x] != %c\n", n,
 218                                        chunk_size - 1, chunk_size - 1,
 219                                        CHUNK_CHAR);
 220                                 error = 1;
 221                         }
 222
 223                         if (st.st_size == trunc_offset) {
 224                                 /* Check case 1: first append then truncate */
 225                                 error = memcmp(read_buf+chunk_size, append_buf,
 226                                                trunc_offset - chunk_size);
 227                                 if (error) {
 228                                         printf("loop %d: trunc-after-append bad"
 229                                                " [%d-%d]/[%#x-%#x] != %c\n",
 230                                                n, chunk_size, trunc_offset - 1,
 231                                                chunk_size, trunc_offset - 1,
 232                                                APPEND_CHAR);
 233                                 }
 234                         } else {
 235                                 /* Check case 2: first truncate then append */
 236                                 if (memcmp(read_buf+chunk_size, trunc_buf,
 237                                            trunc_offset-chunk_size)) {
 238                                         printf("loop %d: append-after-TRUNC bad"
 239                                                " [%d-%d]/[%#x-%#x] != 0\n",
 240                                                n, chunk_size, trunc_offset - 1,
 241                                                chunk_size, trunc_offset - 1);
 242                                         error = 1;
 243                                 } else if (memcmp(read_buf+trunc_offset,
 244                                                   append_buf, append_size)) {
 245                                         printf("loop %d: APPEND-after-trunc bad"
 246                                                " [%d-%d]/[%#x-%#x] != %c\n",
 247                                                n, trunc_offset, append_size - 1,
 248                                                trunc_offset, append_size - 1,
 249                                                APPEND_CHAR);
 250                                         error = 1;
 251                                 }
 252                         }
 253                 }
 254                 ret = MPI_Bcast(&error, 1, MPI_INT, 0, MPI_COMM_WORLD);
 255                 if (ret != MPI_SUCCESS)
 256                         rprintf(rank, n, "MPI_Bcast: %d\n");
 257
 258                 if (error == 1) {
 259                         if (rank == 0) {
 260                                 char command[4096];
 261
 262                                 printf("loop %5d: chunk %6d/%#07x, "
 263                                        "append %6d/%#07x, trunc @ %6d/%#07x\n",
 264                                        n, chunk_size, chunk_size, append_size,
 265                                        append_size, trunc_offset, trunc_offset);
 266
 267                                 sprintf(command, "od -Ax -a %s", fname);
 268                                 system(command);
 269                         }
 270                         rprintf(rank, n, "on machine %s with pid %d\n",
 271                                 hostname, (int)getpid());
 272                 }
 273         }
 274
 275         printf("rank %d, loop %d: finished\n", rank, n);
 276         close(fd);
 277
 278         if (rank == 0) {
 279                 error = unlink(fname);
 280                 if (error < 0)
 281                         rprintf(0, n, "unlink %s failed: %s\n",
 282                                 fname, strerror(errno));
 283         }
 284
 285         MPI_Finalize();
 286         return 0;
 287 }