lustre/tests/rw_seq_cst_vs_drop_caches.c

   1 #include <stdio.h>
   2 #include <stdint.h>
   3 #include <stdlib.h>
   4 #include <assert.h>
   5 #include <fcntl.h>
   6 #include <inttypes.h>
   7 #include <string.h>
   8 #include <unistd.h>
   9 #include <sys/mman.h>
  10 #include <sys/stat.h>
  11 #ifdef HAVE_LIBPTHREAD
  12 #include <pthread.h>
  13 #endif
  14
  15 /*
  16  * Usage: rw_seq_cst_vs_drop_caches [-m] /mnt/lustre/file0 /mnt/lustre2/file0
  17
  18  * Race reads of the same file on two client mounts vs writes and drop
  19  * caches to detect sequential consistency violations. Run
  20  * indefinately.  all abort() if a consistency violation is found in
  21  * which case the wait status ($?) will be 134.
  22 */
  23
  24 int mmap_mode;          /* -m flag */
  25
  26 static void usage(void)
  27 {
  28         fprintf(stdout,
  29                 "%s: rw_seq_cst_vs_drop_caches [-m] /mnt/lustre/file0 /mnt/lustre2/file0\n"
  30 "       -m : use mmap to read/write file\n", __func__);
  31 }
  32
  33 #define handle_error(msg)       \
  34         do { perror(msg); exit(EXIT_FAILURE); } while (0)
  35
  36 static int fd[2] = { -1, -1 };
  37 /* u_max is total number of writes, which are time consumg because they are
  38  * contending with constant reads
  39  */
  40 static uint64_t u, u_max = UINT64_MAX / 2;
  41 char *ptr;
  42
  43 #ifdef HAVE_LIBPTHREAD
  44 static uint64_t v[2];
  45
  46 static void *access_thread_start(void *unused)
  47 {
  48         char *ptr2 = NULL;
  49         ssize_t rc;
  50         int i;
  51
  52         if (mmap_mode) {
  53                 ptr2 = mmap(NULL, sizeof(v[1]), PROT_READ,
  54                             MAP_PRIVATE | MAP_POPULATE, fd[1], 0);
  55                 if (ptr2 == MAP_FAILED)
  56                         handle_error("mmap");
  57         }
  58
  59         do {
  60                 for (i = 0; i < 2; i++) {
  61                         if (mmap_mode) {
  62                                 memcpy(&v[i], i == 0 ? ptr : ptr2,
  63                                        sizeof(v[i]));
  64                         } else {
  65                                 rc = pread(fd[i], &v[i], sizeof(v[i]), 0);
  66                                 if (rc < 0 || rc != sizeof(v[i]))
  67                                         handle_error("pread");
  68                         }
  69                 }
  70         } while (v[0] <= v[1]);
  71
  72         fprintf(stderr, "error: u = %"PRIu64", v = %"PRIu64", %"PRIu64"\n",
  73                 u, v[0], v[1]);
  74
  75         if (mmap_mode)
  76                 munmap(ptr2, sizeof(v[i]));
  77         abort();
  78 }
  79 #endif
  80
  81 static char stderr_buf[4096];
  82
  83 int main(int argc, char *argv[])
  84 {
  85         int drop_caches_fd = -1;
  86         pthread_t access_thread __attribute__ ((unused));
  87         struct stat st[2];
  88         ssize_t rc;
  89         int i, ch;
  90
  91         setvbuf(stderr, stderr_buf, _IOLBF, sizeof(stderr_buf));
  92
  93         while ((ch = getopt(argc, argv, "m")) >= 0) {
  94                 switch (ch) {
  95                 case 'm':
  96                         mmap_mode = 1;
  97                         break;
  98                 default:
  99                         usage();
 100                         exit(EXIT_FAILURE);
 101                 }
 102         }
 103
 104         argc -= optind;
 105         argv += optind;
 106
 107         if (argc != 2) {
 108                 usage();
 109                 exit(EXIT_FAILURE);
 110         }
 111
 112         drop_caches_fd = open("/proc/sys/vm/drop_caches", O_WRONLY);
 113         assert(!(drop_caches_fd < 0));
 114
 115         for (i = 0; i < 2; i++) {
 116                 fd[i] = open(argv[i], O_RDWR|O_CREAT|O_TRUNC, 0666);
 117                 if (fd[i] < 0)
 118                         handle_error("open");
 119
 120                 rc = fstat(fd[i], &st[i]);
 121                 if (rc < 0)
 122                         handle_error("fstat");
 123         }
 124
 125         /* file0 and file1 should be the same file on two different
 126          * client mount points. */
 127         if (st[0].st_dev != st[1].st_dev ||
 128             st[0].st_ino != st[1].st_ino) {
 129                 fprintf(stderr, "file mismatch\n");
 130                 exit(EXIT_FAILURE);
 131         }
 132
 133         if (mmap_mode) {
 134                 if (ftruncate(fd[0], sizeof(u)) < 0)
 135                         handle_error("ftruncate");
 136
 137                 ptr = mmap(NULL, sizeof(u), PROT_READ|PROT_WRITE, MAP_SHARED,
 138                            fd[0], 0);
 139                 if (ptr == MAP_FAILED)
 140                         handle_error("mmap");
 141                 memcpy(ptr, &u, sizeof(u));
 142         } else {
 143                 rc = pwrite(fd[0], &u, sizeof(u), 0);
 144                 if (rc < 0 || rc != sizeof(u))
 145                         handle_error("pwrite");
 146         }
 147
 148 #ifdef HAVE_LIBPTHREAD
 149         rc = pthread_create(&access_thread, NULL, &access_thread_start, NULL);
 150         if (rc != 0)
 151                 handle_error("pthread_create");
 152 #endif
 153
 154         for (u = 1; u <= u_max; u++) {
 155                 if (mmap_mode) {
 156                         memcpy(ptr, &u, sizeof(u));
 157                 } else {
 158                         rc = pwrite(fd[0], &u, sizeof(u), 0);
 159                         if (rc < 0 || rc != sizeof(u))
 160                                 handle_error("pwrite");
 161                 }
 162
 163                 rc = write(drop_caches_fd, "3\n", 2);
 164                 if (rc < 0 || rc != 2)
 165                         handle_error("drop caches");
 166         }
 167
 168 #ifdef HAVE_LIBPTHREAD
 169         rc = pthread_cancel(access_thread);
 170         if (rc != 0)
 171                 handle_error("pthread_cancel");
 172
 173         rc = pthread_join(access_thread, NULL);
 174         if (rc != 0)
 175                 handle_error("pthread_join");
 176 #endif
 177
 178         if (mmap_mode)
 179                 munmap(ptr, sizeof(u));
 180
 181         return 0;
 182 }