lustre/tests/rw_seq_cst_vs_drop_caches.c

   1 #include <stdio.h>
   2 #include <stdint.h>
   3 #include <stdlib.h>
   4 #include <assert.h>
   5 #include <fcntl.h>
   6 #include <inttypes.h>
   7 #include <string.h>
   8 #include <unistd.h>
   9 #include <sys/mman.h>
  10 #include <sys/stat.h>
  11 #include <pthread.h>
  12
  13 /*
  14  * Usage: rw_seq_cst_vs_drop_caches [-m] /mnt/lustre/file0 /mnt/lustre2/file0
  15
  16  * Race reads of the same file on two client mounts vs writes and drop
  17  * caches to detect sequential consistency violations. Run
  18  * indefinately.  all abort() if a consistency violation is found in
  19  * which case the wait status ($?) will be 134.
  20 */
  21
  22 int mmap_mode;          /* -m flag */
  23
  24 static void usage(void)
  25 {
  26         fprintf(stdout,
  27                 "%s: rw_seq_cst_vs_drop_caches [-m] /mnt/lustre/file0 /mnt/lustre2/file0\n"
  28 "       -m : use mmap to read/write file\n", __func__);
  29 }
  30
  31 #define handle_error(msg)       \
  32         do { perror(msg); exit(EXIT_FAILURE); } while (0)
  33
  34 static int fd[2] = { -1, -1 };
  35 /* u_max is total number of writes, which are time consumg because they are
  36  * contending with constant reads
  37  */
  38 static uint64_t u, u_max = UINT64_MAX / 2;
  39 static uint64_t v[2];
  40 char *ptr;
  41
  42 static void *access_thread_start(void *unused)
  43 {
  44         char *ptr2 = NULL;
  45         ssize_t rc;
  46         int i;
  47
  48         if (mmap_mode) {
  49                 ptr2 = mmap(NULL, sizeof(v[1]), PROT_READ,
  50                             MAP_PRIVATE | MAP_POPULATE, fd[1], 0);
  51                 if (ptr2 == MAP_FAILED)
  52                         handle_error("mmap");
  53         }
  54
  55         do {
  56                 for (i = 0; i < 2; i++) {
  57                         if (mmap_mode) {
  58                                 memcpy(&v[i], i == 0 ? ptr : ptr2,
  59                                        sizeof(v[i]));
  60                         } else {
  61                                 rc = pread(fd[i], &v[i], sizeof(v[i]), 0);
  62                                 if (rc < 0 || rc != sizeof(v[i]))
  63                                         handle_error("pread");
  64                         }
  65                 }
  66         } while (v[0] <= v[1]);
  67
  68         fprintf(stderr, "error: u = %"PRIu64", v = %"PRIu64", %"PRIu64"\n",
  69                 u, v[0], v[1]);
  70
  71         if (mmap_mode)
  72                 munmap(ptr2, sizeof(v[i]));
  73         abort();
  74 }
  75
  76 static char stderr_buf[4096];
  77
  78 int main(int argc, char *argv[])
  79 {
  80         int drop_caches_fd = -1;
  81         pthread_t access_thread;
  82         struct stat st[2];
  83         ssize_t rc;
  84         int i, ch;
  85
  86         setvbuf(stderr, stderr_buf, _IOLBF, sizeof(stderr_buf));
  87
  88         while ((ch = getopt(argc, argv, "m")) >= 0) {
  89                 switch (ch) {
  90                 case 'm':
  91                         mmap_mode = 1;
  92                         break;
  93                 default:
  94                         usage();
  95                         exit(EXIT_FAILURE);
  96                 }
  97         }
  98
  99         argc -= optind;
 100         argv += optind;
 101
 102         if (argc != 2) {
 103                 usage();
 104                 exit(EXIT_FAILURE);
 105         }
 106
 107         drop_caches_fd = open("/proc/sys/vm/drop_caches", O_WRONLY);
 108         assert(!(drop_caches_fd < 0));
 109
 110         for (i = 0; i < 2; i++) {
 111                 fd[i] = open(argv[i], O_RDWR|O_CREAT|O_TRUNC, 0666);
 112                 if (fd[i] < 0)
 113                         handle_error("open");
 114
 115                 rc = fstat(fd[i], &st[i]);
 116                 if (rc < 0)
 117                         handle_error("fstat");
 118         }
 119
 120         /* file0 and file1 should be the same file on two different
 121          * client mount points. */
 122         if (st[0].st_dev != st[1].st_dev ||
 123             st[0].st_ino != st[1].st_ino) {
 124                 fprintf(stderr, "file mismatch\n");
 125                 exit(EXIT_FAILURE);
 126         }
 127
 128         if (mmap_mode) {
 129                 if (ftruncate(fd[0], sizeof(u)) < 0)
 130                         handle_error("ftruncate");
 131
 132                 ptr = mmap(NULL, sizeof(u), PROT_READ|PROT_WRITE, MAP_SHARED,
 133                            fd[0], 0);
 134                 if (ptr == MAP_FAILED)
 135                         handle_error("mmap");
 136                 memcpy(ptr, &u, sizeof(u));
 137         } else {
 138                 rc = pwrite(fd[0], &u, sizeof(u), 0);
 139                 if (rc < 0 || rc != sizeof(u))
 140                         handle_error("pwrite");
 141         }
 142
 143         rc = pthread_create(&access_thread, NULL, &access_thread_start, NULL);
 144         if (rc != 0)
 145                 handle_error("pthread_create");
 146
 147         for (u = 1; u <= u_max; u++) {
 148                 if (mmap_mode) {
 149                         memcpy(ptr, &u, sizeof(u));
 150                 } else {
 151                         rc = pwrite(fd[0], &u, sizeof(u), 0);
 152                         if (rc < 0 || rc != sizeof(u))
 153                                 handle_error("pwrite");
 154                 }
 155
 156                 rc = write(drop_caches_fd, "3\n", 2);
 157                 if (rc < 0 || rc != 2)
 158                         handle_error("drop caches");
 159         }
 160
 161         rc = pthread_cancel(access_thread);
 162         if (rc != 0)
 163                 handle_error("pthread_cancel");
 164
 165         rc = pthread_join(access_thread, NULL);
 166         if (rc != 0)
 167                 handle_error("pthread_join");
 168
 169         if (mmap_mode)
 170                 munmap(ptr, sizeof(u));
 171
 172         return 0;
 173 }