lustre/tests/write_append_truncate.c

   1 /* -*- mode: c; c-basic-offset: 8; indent-tabs-mode: nil; -*-
   2  * vim:expandtab:shiftwidth=8:tabstop=8:
   3  *
   4  * Each loop does 3 things:
   5  *   - truncate file to zero (not via ftruncate though, to test O_APPEND)
   6  *   - append a "chunk" of data (should be at file offset 0 after truncate)
   7  *   - on each of two threads either append or truncate-up the file
   8  *
   9  * If the truncate happened first, we should have a hole in the file.
  10  * If the append happened first, we should have truncated the file down.
  11  *
  12  * We pick the CHUNK_SIZE_MAX and APPEND_SIZE_MAX so that we cross a stripe.
  13  *
  14  * compile: mpicc -g -Wall -o write_append_truncate write_append_truncate.c
  15  * run:     mpirun -np 2 -machlist <hostlist file> write_append_truncate <file>
  16  *  or:     pdsh -w <two hosts> write_append_truncate <file>
  17  *  or:     prun -n 2 [-N 2] write_append_truncate <file>
  18  */
  19 #include <stdlib.h>
  20 #include <stdio.h>
  21 #include <stdarg.h>
  22 #include <sys/types.h>
  23 #include <sys/stat.h>
  24 #include <fcntl.h>
  25 #include <errno.h>
  26 #include <string.h>
  27 #include <unistd.h>
  28 #include "mpi.h"
  29
  30 #define DEFAULT_ITER     50000
  31
  32 #define CHUNK_SIZE_MAX   123456
  33 #define CHUNK_CHAR   'C'
  34
  35 #define APPEND_SIZE_MAX  123456
  36 #define APPEND_CHAR  'A'
  37
  38 #define TRUNC_SIZE_MAX   (CHUNK_SIZE_MAX+APPEND_SIZE_MAX)
  39
  40 #define HOSTNAME_SIZE 50
  41
  42 void usage(char *prog)
  43 {
  44         printf("usage: %s <filename> [nloops]\n", prog);
  45         printf("%s must be run with at least 2 processes\n", prog);
  46
  47         MPI_Finalize();
  48         exit(1);
  49 }
  50
  51 /* Print process rank, loop count, message, and exit (i.e. a fatal error) */
  52 void rprintf(int rank, int loop, const char *fmt, ...)
  53 {
  54         va_list       ap;
  55
  56         printf("rank %d, loop %d: ", rank, loop);
  57
  58         va_start(ap, fmt);
  59
  60         vprintf(fmt, ap);
  61
  62         MPI_Abort(MPI_COMM_WORLD, -1);
  63 }
  64
  65 int main(int argc, char *argv[])
  66 {
  67         int n, nloops = 0, fd;
  68         int rank, size, ret;
  69         int chunk_size, append_size, trunc_offset;
  70         char append_buf[APPEND_SIZE_MAX];
  71         char chunk_buf[CHUNK_SIZE_MAX];
  72         char read_buf[TRUNC_SIZE_MAX+APPEND_SIZE_MAX];
  73         char trunc_buf[TRUNC_SIZE_MAX];
  74         int done;
  75         int error;
  76         char hostname[HOSTNAME_SIZE];
  77         char *fname, *prog;
  78
  79         error = MPI_Init(&argc, &argv);
  80         if (error != MPI_SUCCESS)
  81                 rprintf(-1, -1, "MPI_Init failed: %d\n", error);
  82
  83         prog = strrchr(argv[0], '/');
  84         if (prog == NULL)
  85                 prog = argv[0];
  86         else
  87                 prog++;
  88
  89         if (argc < 2 || argc > 3)
  90                 usage(prog);
  91
  92         error = MPI_Comm_rank(MPI_COMM_WORLD, &rank);
  93         if (error != MPI_SUCCESS)
  94                 rprintf(-1, -1, "MPI_Comm_rank failed: %d\n", error);
  95
  96         error = MPI_Comm_size(MPI_COMM_WORLD, &size);
  97         if (error != MPI_SUCCESS)
  98                 rprintf(rank, -1, "MPI_Comm_size failed: %d\n", error);
  99
 100         if (size < 2)
 101                 rprintf(rank, -1, "%s: must run with at least 2 processes\n",
 102                         prog);
 103
 104         memset(append_buf, APPEND_CHAR, APPEND_SIZE_MAX);
 105         memset(chunk_buf, CHUNK_CHAR, CHUNK_SIZE_MAX);
 106         memset(trunc_buf, 0, TRUNC_SIZE_MAX);
 107
 108         if (gethostname(hostname, HOSTNAME_SIZE) < 0)
 109                 rprintf(rank, -1, "gethostname failed: %s\n", strerror(errno));
 110
 111         fname = argv[1];
 112
 113         if (argc == 3)
 114                 nloops = strtoul(argv[2], NULL, 0);
 115         if (nloops == 0)
 116                 nloops = DEFAULT_ITER;
 117
 118         if (rank == 0) {
 119                 fd = open(fname, O_WRONLY|O_CREAT|O_TRUNC, 0666);
 120                 if (fd < 0)
 121                         rprintf(0, -1, "create %s failed: %s\n", fname,
 122                                 strerror(errno));
 123                 printf("using %s\n", fname);
 124         }
 125         error = MPI_Barrier(MPI_COMM_WORLD);
 126         if (error != MPI_SUCCESS)
 127                 rprintf(rank, -1, "prep MPI_Barrier failed: %d\n", error);
 128
 129         fd = open(fname, O_RDWR | O_APPEND);
 130         if (fd < 0)
 131                 rprintf(rank, -1, "open %s failed: %s\n",fname,strerror(errno));
 132
 133         for (n = 0; n < nloops; n++) {
 134                 /* reset the environment */
 135                 chunk_size = (rand()%(CHUNK_SIZE_MAX-1))+1;
 136                 append_size = (rand()%(APPEND_SIZE_MAX-1))+1;
 137                 trunc_offset = chunk_size + rand()%append_size;
 138                 if (rank == 0) {
 139                         if (n % 1000 == 0)
 140                                 printf("loop %5d: chunk %6d/%#07x, "
 141                                        "append %6d/%#07x, trunc @ %6d/%#07x\n",
 142                                        n, chunk_size, chunk_size, append_size,
 143                                        append_size, trunc_offset, trunc_offset);
 144
 145                         ret = truncate(fname, (off_t)0);
 146                         if (ret < 0)
 147                                 rprintf(0, n, "truncate @ 0: %s\n",
 148                                         strerror(errno));
 149                         done = 0;
 150                         do {
 151                                 ret = write(fd, chunk_buf+done,chunk_size-done);
 152                                 if (ret <= 0) {
 153                                         rprintf(0, n, "chunk @ %d: %s\n",
 154                                                 done, strerror(errno));
 155                                         break;
 156                                 }
 157                                 done += ret;
 158                         } while (done != chunk_size);
 159                 }
 160
 161                 error = MPI_Barrier(MPI_COMM_WORLD);
 162                 if (error != MPI_SUCCESS)
 163                         rprintf(rank, n, "start MPI_Barrier: %d\n",error);
 164
 165                 /* Do the race */
 166                 if (rank == n % size) {
 167                         //
 168                         done = 0;
 169                         do {
 170                                 ret = write(fd, append_buf + done,
 171                                             append_size - done);
 172                                 if (ret < 0) {
 173                                         rprintf(rank, n,
 174                                                 "loop %d: append @ %u: %s\n",
 175                                                 done, strerror(errno));
 176                                         break;
 177                                 }
 178                                 done += ret;
 179                         } while (done != append_size);
 180                 } else if (rank == (n + 1) % size) {
 181                         ret = truncate(fname, (off_t)trunc_offset);
 182                         if (ret != 0)
 183                                 rprintf(rank, n, "truncate @ %u: %s\n",
 184                                         trunc_offset, strerror(errno) );
 185                 }
 186
 187                 error = MPI_Barrier(MPI_COMM_WORLD);
 188                 if (error != MPI_SUCCESS)
 189                         rprintf(rank, n, "end MPI_Barrier: %d\n", error);
 190
 191                 error = 0;
 192
 193                 /* Check the result */
 194                 if (rank == 0) {
 195                         struct stat st;
 196                         if (stat(fname, &st) < 0)
 197                                 rprintf(0, n, "loop %d: stat %s: %s\n",
 198                                         fname, strerror(errno));
 199
 200                         if (lseek(fd, (off_t)0, SEEK_SET) != 0)
 201                                 rprintf(0, n, "lseek fname 0: %s\n", fname,
 202                                         strerror(errno));
 203
 204                         done = 0;
 205                         do {
 206                                 ret = read(fd, read_buf+done, st.st_size-done);
 207                                 if (ret < 0) {
 208                                         rprintf(0, n, "read @ %u: %s\n",
 209                                                done, strerror(errno));
 210                                 }
 211                                 done += ret;
 212                         } while (done != st.st_size);
 213
 214                         if (memcmp(read_buf, chunk_buf, chunk_size)) {
 215                                 printf("loop %d: base chunk bad"
 216                                        " [0-%d]/[0-%#x] != %c\n", n,
 217                                        chunk_size - 1, chunk_size - 1,
 218                                        CHUNK_CHAR);
 219                                 error = 1;
 220                         }
 221
 222                         if (st.st_size == trunc_offset) {
 223                                 /* Check case 1: first append then truncate */
 224                                 error = memcmp(read_buf+chunk_size, append_buf,
 225                                                trunc_offset - chunk_size);
 226                                 if (error) {
 227                                         printf("loop %d: trunc-after-append bad"
 228                                                " [%d-%d]/[%#x-%#x] != %c\n",
 229                                                n, chunk_size, trunc_offset - 1,
 230                                                chunk_size, trunc_offset - 1,
 231                                                APPEND_CHAR);
 232                                 }
 233                         } else {
 234                                 /* Check case 2: first truncate then append */
 235                                 if (memcmp(read_buf+chunk_size, trunc_buf,
 236                                            trunc_offset-chunk_size)) {
 237                                         printf("loop %d: append-after-TRUNC bad"
 238                                                " [%d-%d]/[%#x-%#x] != 0\n",
 239                                                n, chunk_size, trunc_offset - 1,
 240                                                chunk_size, trunc_offset - 1);
 241                                         error = 1;
 242                                 } else if (memcmp(read_buf+trunc_offset,
 243                                                   append_buf, append_size)) {
 244                                         printf("loop %d: APPEND-after-trunc bad"
 245                                                " [%d-%d]/[%#x-%#x] != %c\n",
 246                                                n, trunc_offset, append_size - 1,
 247                                                trunc_offset, append_size - 1,
 248                                                APPEND_CHAR);
 249                                         error = 1;
 250                                 }
 251                         }
 252                 }
 253                 ret = MPI_Bcast(&error, 1, MPI_INT, 0, MPI_COMM_WORLD);
 254                 if (ret != MPI_SUCCESS)
 255                         rprintf(rank, n, "MPI_Bcast: %d\n");
 256
 257                 if (error == 1) {
 258                         if (rank == 0) {
 259                                 char command[4096];
 260
 261                                 printf("loop %5d: chunk %6d/%#07x, "
 262                                        "append %6d/%#07x, trunc @ %6d/%#07x\n",
 263                                        n, chunk_size, chunk_size, append_size,
 264                                        append_size, trunc_offset, trunc_offset);
 265
 266                                 sprintf(command, "od -Ax -a %s", fname);
 267                                 system(command);
 268                         }
 269                         rprintf(rank, n, "on machine %s with pid %d\n",
 270                                 hostname, (int)getpid());
 271                 }
 272         }
 273
 274         printf("rank %d, loop %d: finished\n", rank, n);
 275         close(fd);
 276
 277         if (rank == 0) {
 278                 error = unlink(fname);
 279                 if (error < 0)
 280                         rprintf(0, n, "unlink %s failed: %s\n",
 281                                 fname, strerror(errno));
 282         }
 283
 284         MPI_Finalize();
 285         return 0;
 286 }