1 /* -*- mode: c; c-basic-offset: 8; indent-tabs-mode: nil; -*-
2 * vim:expandtab:shiftwidth=8:tabstop=8:
4 * Copyright (C) 2002 Cluster File Systems, Inc.
5 * Author: You Feng <youfeng@clusterfs.com>
7 * This file is part of Lustre, http://www.lustre.org.
9 * Lustre is free software; you can redistribute it and/or
10 * modify it under the terms of version 2 of the GNU General Public
11 * License as published by the Free Software Foundation.
13 * Lustre is distributed in the hope that it will be useful,
14 * but WITHOUT ANY WARRANTY; without even the implied warranty of
15 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
16 * GNU General Public License for more details.
18 * You should have received a copy of the GNU General Public License
19 * along with Lustre; if not, write to the Free Software
20 * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
27 #include <sys/types.h>
30 #include <sys/ioctl.h>
34 #include <lustre/lustre_user.h>
37 #define LPGL_FILEN 700000
38 #define LPGL_TEST_ITEMS 7
42 /* waiting time in 0.1 s */
43 #define MAX_WAITING_TIME 20
50 * process1 attempts CW(gid=1) -- granted immediately
51 * process2 attempts PR -- blocked, goes on waiting list
52 * process3 attempts CW(gid=1) -> should be granted, but may go on
55 void grouplock_test1(char *filename, int fd, char *errmsg)
57 int rc, count, gid = 1;
59 char zeros[LPGL_FILEN];
60 MPI_Request req1, req2;
64 if ((rc = ioctl(fd, LL_IOC_GROUP_LOCK, gid)) == -1) {
65 sprintf(errmsg, "ioctl GROUP_LOCK of file %s return %d",
71 MPI_Barrier(MPI_COMM_WORLD);
74 memset(zeros, 0x0, sizeof(zeros));
75 lseek(fd, 0, SEEK_SET);
77 MPI_Send(&gid, 1, MPI_INT, 2, 1, MPI_COMM_WORLD);
78 count = read(fd, buf, sizeof(buf));
79 if (count != sizeof(buf)) {
81 dump_diff(zeros, buf, count, 0);
82 sprintf(errmsg, "read of file %s return %d",
86 MPI_Send(&gid, 1, MPI_INT, 0, 1, MPI_COMM_WORLD);
92 /* Wait for reading task to progress, this is probably somewhat
93 racey, though, may be adding usleep here would make things
96 MPI_Recv(&temp, 1, MPI_INT, 1, 1, MPI_COMM_WORLD,
98 if ((rc = ioctl(fd, LL_IOC_GROUP_LOCK, gid)) == -1) {
99 sprintf(errmsg, "ioctl GROUP_LOCK of file %s return %d",
103 MPI_Send(&gid, 1, MPI_INT, 0, 1, MPI_COMM_WORLD);
107 int iter = MAX_WAITING_TIME;
110 /* reading task will tell us when it completes */
111 MPI_Irecv(&temp1, 1, MPI_INT, 1, 1, MPI_COMM_WORLD, &req1);
112 /* 2nd locking task will tell us when it completes */
113 MPI_Irecv(&temp2, 1, MPI_INT, 2, 1, MPI_COMM_WORLD, &req2);
118 FAIL("2nd locking task is not progressing\n");
121 MPI_Test(&req2, &flag2, MPI_STATUS_IGNORE);
122 MPI_Test(&req1, &flag1, MPI_STATUS_IGNORE);
124 FAIL("PR task progressed even though GROUP lock"
130 /* Now we need to release the lock */
132 if (rank == 0 || rank == 2) {
133 if ((rc = ioctl(fd, LL_IOC_GROUP_UNLOCK, gid)) == -1) {
134 sprintf(errmsg, "ioctl GROUP_UNLOCK of file %s return %d",
141 int iter = MAX_WAITING_TIME;
147 FAIL("reading task is not progressing even "
148 "though GROUP lock was released\n");
152 MPI_Test(&req1, &flag1, MPI_STATUS_IGNORE);
156 MPI_Barrier(MPI_COMM_WORLD);
161 * process1 attempts CW(gid=1) -- granted immediately
162 * process2 attempts CW(gid=2) -- blocked
163 * process3 attempts PR -- blocked
164 * process4 attempts CW(gid=2) -- blocked
165 * process1 releases CW(gid=1) -- this allows process2's CW lock to be granted
166 process3 remains blocked
168 void grouplock_test2(char *filename, int fd, char *errmsg)
170 int rc, count, gid = 1;
171 char buf[LPGL_FILEN];
172 char zeros[LPGL_FILEN];
173 MPI_Request req1, req2, req3;
174 int temp1, temp2, temp3;
177 if ((rc = ioctl(fd, LL_IOC_GROUP_LOCK, gid)) == -1) {
178 sprintf(errmsg, "ioctl GROUP_LOCK of file %s return %d",
184 MPI_Barrier(MPI_COMM_WORLD);
186 if (rank == 1 || rank == 3) {
189 MPI_Recv(&temp1, 1, MPI_INT, 2, 1, MPI_COMM_WORLD,
193 if ((rc = ioctl(fd, LL_IOC_GROUP_LOCK, gid)) == -1) {
194 sprintf(errmsg, "ioctl GROUP_LOCK of file %s return %d",
198 MPI_Send(&gid, 1, MPI_INT, 0, 1, MPI_COMM_WORLD);
202 memset(zeros, 0x0, sizeof(zeros));
203 lseek(fd, 0, SEEK_SET);
205 MPI_Send(&gid, 1, MPI_INT, 3, 1, MPI_COMM_WORLD);
206 count = read(fd, buf, sizeof(buf));
207 if (count != sizeof(buf)) {
209 dump_diff(zeros, buf, count, 0);
210 sprintf(errmsg, "read of file %s return %d",
214 MPI_Send(&gid, 1, MPI_INT, 0, 1, MPI_COMM_WORLD);
218 int iter = MAX_WAITING_TIME;
219 int flag1, flag2, flag3;
221 /* 2nd locking task will tell us when it completes */
222 MPI_Irecv(&temp1, 1, MPI_INT, 1, 1, MPI_COMM_WORLD, &req1);
223 /* 3nd locking task will tell us when it completes */
224 MPI_Irecv(&temp2, 1, MPI_INT, 3, 1, MPI_COMM_WORLD, &req2);
225 /* reading task will tell us when it completes */
226 MPI_Irecv(&temp3, 1, MPI_INT, 2, 1, MPI_COMM_WORLD, &req3);
231 MPI_Test(&req2, &flag2, MPI_STATUS_IGNORE);
232 MPI_Test(&req1, &flag1, MPI_STATUS_IGNORE);
233 MPI_Test(&req3, &flag3, MPI_STATUS_IGNORE);
235 FAIL("PR task progressed even though GROUP lock"
238 if (flag1 || flag2) {
239 FAIL("GROUP (gid=2) task progressed even though"
240 " GROUP (gid=1) lock is held\n");
245 /* Now let's release first lock */
246 if ((rc = ioctl(fd, LL_IOC_GROUP_UNLOCK, gid)) == -1) {
247 sprintf(errmsg, "ioctl GROUP_UNLOCK of file %s return %d",
251 iter = MAX_WAITING_TIME;
255 FAIL("GROUP(gid=2) tasks are not progressing\n");
258 MPI_Test(&req2, &flag2, MPI_STATUS_IGNORE);
259 MPI_Test(&req1, &flag1, MPI_STATUS_IGNORE);
260 MPI_Test(&req3, &flag3, MPI_STATUS_IGNORE);
262 fprintf(stderr, "task1 %d, task3 %d\n", flag1,
264 FAIL("PR task progressed even though GROUP lock"
265 " was on the queue task\n");
267 } while (!(flag1 && flag2));
268 MPI_Send(&gid, 1, MPI_INT, 1, 1, MPI_COMM_WORLD);
269 MPI_Send(&gid, 1, MPI_INT, 3, 1, MPI_COMM_WORLD);
272 if (rank == 1 || rank == 3) {
273 /* Do not release the locks until task 0 is ready to watch
274 for reading task only */
275 MPI_Recv(&temp1, 1, MPI_INT, 0, 1, MPI_COMM_WORLD,
277 if ((rc = ioctl(fd, LL_IOC_GROUP_UNLOCK, gid)) == -1) {
278 sprintf(errmsg, "ioctl GROUP_UNLOCK of file %s return %d",
285 int iter = MAX_WAITING_TIME;
291 FAIL("reading task is not progressing even "
292 "though GROUP locks are released\n");
296 MPI_Test(&req3, &flag3, MPI_STATUS_IGNORE);
300 MPI_Barrier(MPI_COMM_WORLD);
305 * process1 attempts CW(gid=1) -- granted
306 * process2 attempts PR -- blocked
307 * process3 attempts CW(gid=1) -> should be granted
308 * process3 releases CW(gid=1)
309 * process2 should remain blocked
310 * process1 releases CW(gid=1)
311 * process2's PR should be granted
313 * This is a lot like test1.
315 void grouplock_test3(char *filename, int fd, char *errmsg)
317 int rc, count, gid = 1;
318 char buf[LPGL_FILEN];
319 char zeros[LPGL_FILEN];
320 MPI_Request req1, req2;
324 if ((rc = ioctl(fd, LL_IOC_GROUP_LOCK, gid)) == -1) {
325 sprintf(errmsg, "ioctl GROUP_LOCK of file %s return %d",
331 MPI_Barrier(MPI_COMM_WORLD);
334 memset(zeros, 0x0, sizeof(zeros));
335 lseek(fd, 0, SEEK_SET);
337 MPI_Send(&gid, 1, MPI_INT, 2, 1, MPI_COMM_WORLD);
338 count = read(fd, buf, sizeof(buf));
339 if (count != sizeof(buf)) {
341 dump_diff(zeros, buf, count, 0);
342 sprintf(errmsg, "read of file %s return %d",
346 MPI_Send(&gid, 1, MPI_INT, 0, 1, MPI_COMM_WORLD);
352 /* Wait for reading task to progress, this is probably somewhat
353 racey, though, may be adding usleep here would make things
356 MPI_Recv(&temp, 1, MPI_INT, 1, 1, MPI_COMM_WORLD,
358 if ((rc = ioctl(fd, LL_IOC_GROUP_LOCK, gid)) == -1) {
359 sprintf(errmsg, "ioctl GROUP_LOCK of file %s return %d",
363 MPI_Send(&gid, 1, MPI_INT, 0, 1, MPI_COMM_WORLD);
367 int iter = MAX_WAITING_TIME;
370 /* reading task will tell us when it completes */
371 MPI_Irecv(&temp1, 1, MPI_INT, 1, 1, MPI_COMM_WORLD, &req1);
372 /* 2nd locking task will tell us when it completes */
373 MPI_Irecv(&temp2, 1, MPI_INT, 2, 1, MPI_COMM_WORLD, &req2);
378 FAIL("2nd locking task is not progressing\n");
381 MPI_Test(&req2, &flag2, MPI_STATUS_IGNORE);
382 MPI_Test(&req1, &flag1, MPI_STATUS_IGNORE);
384 FAIL("PR task progressed even though GROUP lock"
390 /* Now we need to release the lock */
393 if ((rc = ioctl(fd, LL_IOC_GROUP_UNLOCK, gid)) == -1) {
394 sprintf(errmsg, "ioctl GROUP_UNLOCK of file %s return %d",
401 int iter = MAX_WAITING_TIME;
407 MPI_Test(&req1, &flag1, MPI_STATUS_IGNORE);
408 } while (!flag1 && iter);
410 FAIL("reading task is progressing even "
411 "though GROUP lock was not fully released\n");
414 iter = MAX_WAITING_TIME;
416 if ((rc = ioctl(fd, LL_IOC_GROUP_UNLOCK, gid)) == -1) {
417 sprintf(errmsg, "ioctl GROUP_UNLOCK of file %s return %d",
425 FAIL("reading task is not progressing even "
426 "though GROUP lock was released\n");
430 MPI_Test(&req1, &flag1, MPI_STATUS_IGNORE);
434 MPI_Barrier(MPI_COMM_WORLD);
439 * process1 attempts CW(gid=1) -- granted
440 * process2 attempts PR on non-blocking fd -> should return -EWOULDBLOCK
441 * process3 attempts CW(gid=2) on non-blocking fd -> should return -EWOULDBLOCK
443 void grouplock_test4(char *filename, int fd, char *errmsg)
445 int rc, count, gid = 1;
446 char buf[LPGL_FILEN];
447 char zeros[LPGL_FILEN];
450 if ((rc = ioctl(fd, LL_IOC_GROUP_LOCK, gid)) == -1) {
451 sprintf(errmsg, "ioctl GROUP_LOCK of file %s return %d",
457 MPI_Barrier(MPI_COMM_WORLD);
460 memset(zeros, 0x0, sizeof(zeros));
461 lseek(fd, 0, SEEK_SET);
463 count = read(fd, buf, sizeof(buf));
464 if (count != sizeof(buf)) {
465 if (count == -1 && errno == EWOULDBLOCK) {
466 MPI_Send(&gid, 1, MPI_INT, 0, 1, MPI_COMM_WORLD);
471 dump_diff(zeros, buf, count, 0);
472 sprintf(errmsg, "read of file %s return %d",
476 FAIL("PR lock succeed while incompatible "
477 "GROUP LOCK (gid=1) is still held\n");
483 if ((rc = ioctl(fd, LL_IOC_GROUP_LOCK, gid)) == -1) {
484 if (errno == EWOULDBLOCK) {
485 MPI_Send(&gid, 1, MPI_INT, 0, 1, MPI_COMM_WORLD);
489 sprintf(errmsg, "ioctl GROUP_LOCK of file %s return %d",
493 FAIL("GROUP_LOCK (gid=2) succeed while incompatible "
494 "GROUP LOCK (gid=1) is still held\n");
500 int iter = MAX_WAITING_TIME;
502 MPI_Request req1, req2;
505 /* reading task will tell us when it completes */
506 MPI_Irecv(&temp1, 1, MPI_INT, 1, 1, MPI_COMM_WORLD, &req1);
507 /* 2nd locking task will tell us when it completes */
508 MPI_Irecv(&temp2, 1, MPI_INT, 2, 1, MPI_COMM_WORLD, &req2);
513 FAIL("non-blocking tasks are not progressing\n");
516 MPI_Test(&req2, &flag2, MPI_STATUS_IGNORE);
517 MPI_Test(&req1, &flag1, MPI_STATUS_IGNORE);
518 } while (!(flag2 && flag1));
520 if ((rc = ioctl(fd, LL_IOC_GROUP_UNLOCK, gid)) == -1) {
521 sprintf(errmsg, "ioctl GROUP_UNLOCK of file %s", filename);
528 * process1 attempts CW(gid=1) -- granted
529 * process2 attempts CW(gid=2) -- blocked
530 * process3 attempts CW(gid=2) -- blocked
531 * process1 releases CW(gid=1)
532 * process2's CW(gid=2) should be granted
533 * process3's CW(gid=2) should be granted
535 * This is pretty much like test 3
537 void grouplock_test5(char *filename, int fd, char *errmsg)
539 int rc, count, gid = 1;
540 char buf[LPGL_FILEN];
541 char zeros[LPGL_FILEN];
542 MPI_Request req1, req2;
546 if ((rc = ioctl(fd, LL_IOC_GROUP_LOCK, gid)) == -1) {
547 sprintf(errmsg, "ioctl GROUP_LOCK of file %s return %d",
553 MPI_Barrier(MPI_COMM_WORLD);
555 if (rank == 2 || rank == 1) {
557 if ((rc = ioctl(fd, LL_IOC_GROUP_LOCK, gid)) == -1) {
558 sprintf(errmsg, "ioctl GROUP_LOCK of file %s return %d",
562 MPI_Send(&gid, 1, MPI_INT, 0, 1, MPI_COMM_WORLD);
566 int iter = MAX_WAITING_TIME;
569 /* 3rd locking task will tell us when it completes */
570 MPI_Irecv(&temp1, 1, MPI_INT, 1, 1, MPI_COMM_WORLD, &req1);
571 /* 2nd locking task will tell us when it completes */
572 MPI_Irecv(&temp2, 1, MPI_INT, 2, 1, MPI_COMM_WORLD, &req2);
577 MPI_Test(&req2, &flag2, MPI_STATUS_IGNORE);
578 MPI_Test(&req1, &flag1, MPI_STATUS_IGNORE);
579 } while (!flag2 && !flag1 && iter);
581 FAIL("incomptible locking tasks are progressing\n");
585 /* Now we need to release the lock */
588 int iter = MAX_WAITING_TIME;
590 if ((rc = ioctl(fd, LL_IOC_GROUP_UNLOCK, gid)) == -1) {
591 sprintf(errmsg, "ioctl GROUP_UNLOCK of file %s return %d",
599 FAIL("locking tasks are not progressing even "
600 "though incompatible lock released\n");
603 MPI_Test(&req1, &flag1, MPI_STATUS_IGNORE);
604 MPI_Test(&req2, &flag2, MPI_STATUS_IGNORE);
605 } while (!(flag1 && flag2));
609 if ( rank == 1 || rank == 2) {
610 if ((rc = ioctl(fd, LL_IOC_GROUP_UNLOCK, gid)) == -1) {
611 sprintf(errmsg, "ioctl GROUP_UNLOCK of file %s return %d",
617 MPI_Barrier(MPI_COMM_WORLD);
623 * process1 attempts CW(gid=1) -- granted
624 * process2 attempts PW -- blocked
625 * process2 attempts CW(gid=2) -- blocked
626 * process3 attempts CW(gid=2) -- blocked
627 * process1 releases CW(gid=1)
628 * process2's CW(gid=2) should be granted
629 * process3's CW(gid=2) should be granted
631 * after process1 release CW(gid=1), there are two pathes:
632 * path 1. process2 get PW
633 * path 2. process3 get CW(gid=2)
635 * green: Also about test6 - by definition if P* and CW lock are waiting,
636 * CW lock have bigger priority and should be granted first when it becomes
637 * possible. So after process1 releases its CW lock, process3 should always
638 * get CW lock, and when it will release it, process 2 will proceed with read
639 * and then with getting CW lock
641 * XXX This test does not make any sence at all the way it is described right
642 * now, hence disabled.
644 void grouplock_test6(char *filename, int fd, char *errmsg)
648 /* Just test some error paths with invalid requests */
649 void grouplock_errorstest(char *filename, int fd, char *errmsg)
654 /* To not do lots of separate tests with lots of fd opening/closing,
655 different parts of this test are performed in different processes */
657 if (rank == 0 || rank == 1 ) {
658 if ((rc = ioctl(fd, LL_IOC_GROUP_LOCK, gid)) == -1) {
659 sprintf(errmsg, "ioctl GROUP_LOCK of file %s return %d",
665 /* second group lock on same fd, same gid */
667 if ((rc = ioctl(fd, LL_IOC_GROUP_LOCK, gid)) == -1) {
668 if (errno != EINVAL) {
669 sprintf(errmsg, "Double GROUP lock failed with errno %d instead of EINVAL\n", errno);
673 FAIL("Taking second GROUP lock on same fd succeed\n");
677 /* second group lock on same fd, different gid */
679 if ((rc = ioctl(fd, LL_IOC_GROUP_LOCK, gid + 1)) == -1) {
680 if (errno != EINVAL) {
681 sprintf(errmsg, "Double GROUP lock different gid failed with errno %d instead of EINVAL\n", errno);
685 FAIL("Taking second GROUP lock on same fd, different gid, succeed\n");
689 /* GROUP unlock with wrong gid */
690 if (rank == 0 || rank == 1) {
691 if ((rc = ioctl(fd, LL_IOC_GROUP_UNLOCK, gid + 1)) == -1) {
692 if (errno != EINVAL) {
693 sprintf(errmsg, "GROUP unlock with wrong gid failed with errno %d instead of EINVAL\n",
698 FAIL("GROUP unlock with wrong gid succeed\n");
702 if (rank == 0 || rank == 1) {
703 if ((rc = ioctl(fd, LL_IOC_GROUP_UNLOCK, gid)) == -1) {
704 sprintf(errmsg, "ioctl GROUP_UNLOCK of file %s return %d",
710 /* unlock of never locked fd */
712 if ((rc = ioctl(fd, LL_IOC_GROUP_UNLOCK, gid)) == -1) {
713 if (errno != EINVAL) {
714 sprintf(errmsg, "GROUP unlock on never locked fd failed with errno %d instead of EINVAL\n",
719 FAIL("GROUP unlock on never locked fd succeed\n");
724 void grouplock_file(char *name, int items)
727 char filename[MAX_FILENAME_LEN];
728 char errmsg[MAX_FILENAME_LEN+20];
730 sprintf(filename, "%s/%s", testdir, name);
733 if ((fd = open(filename, O_RDWR | O_NONBLOCK)) == -1) {
734 sprintf(errmsg, "open of file %s", filename);
737 } else if ((fd = open(filename, O_RDWR)) == -1) {
738 sprintf(errmsg, "open of file %s", filename);
742 MPI_Barrier(MPI_COMM_WORLD);
746 grouplock_test1(filename, fd, errmsg);
749 grouplock_test2(filename, fd, errmsg);
752 grouplock_test3(filename, fd, errmsg);
755 grouplock_test4(filename, fd, errmsg);
758 grouplock_test5(filename, fd, errmsg);
761 grouplock_test6(filename, fd, errmsg);
764 grouplock_errorstest(filename, fd, errmsg);
767 sprintf(errmsg, "wrong test case number %d (should be <= %d)",
768 items, LPGL_TEST_ITEMS);
772 MPI_Barrier(MPI_COMM_WORLD);
774 if (close(fd) == -1) {
775 sprintf(errmsg, "close of file %s", filename);
781 void parallel_grouplock(void)
785 for (i = 1;i <= LPGL_TEST_ITEMS;++i) {
787 create_file("parallel_grouplock", LPGL_FILEN, 0);
791 grouplock_file("parallel_grouplock", i);
795 remove_file("parallel_grouplock");
800 void usage(char *proc)
805 printf("Usage: %s [-h] -d <testdir>\n", proc);
806 printf(" [-n \"13\"] [-v] [-V #] [-g]\n");
807 printf("\t-h: prints this help message\n");
808 printf("\t-d: the directory in which the tests will run\n");
809 printf("\t-n: repeat test # times\n");
810 printf("\t-v: increase the verbositly level by 1\n");
811 printf("\t-V: select a specific verbosity level\n");
812 printf("\t-g: debug mode\n");
816 if (i) MPI_Finalize();
820 int main(int argc, char *argv[])
823 int i, iterations = 1;
826 /* Check for -h parameter before MPI_Init so the binary can be
827 called directly, without, for instance, mpirun */
828 for (i = 1; i < argc; ++i) {
829 if (!strcmp(argv[i], "-h") || !strcmp(argv[i], "--help"))
833 MPI_Init(&argc, &argv);
834 MPI_Comm_rank(MPI_COMM_WORLD, &rank);
835 MPI_Comm_size(MPI_COMM_WORLD, &size);
837 // MPI_Comm_set_attr(MPI_COMM_WORLD, MPI_WTIME_IS_GLOBAL, &tr);
839 /* Parse command line options */
841 c = getopt(argc, argv, "d:ghn:vV:");
856 iterations = atoi(optarg);
862 verbose = atoi(optarg);
868 printf("%s is running with %d process(es) %s\n",
869 argv[0], size, debug ? "in DEBUG mode" : "\b\b");
871 if (size < MAX_GLHOST) {
872 fprintf(stderr, "Error: "
873 "should be at least four processes to run the test!\n");
874 MPI_Abort(MPI_COMM_WORLD, 2);
877 if (testdir == NULL && rank == 0) {
878 fprintf(stderr, "Please specify a test directory! (\"%s -h\" for help)\n",
880 MPI_Abort(MPI_COMM_WORLD, 2);
885 for (i = 0; i < iterations; ++i) {
887 printf("%s: Running test #%s(iter %d)\n",
888 timestamp(), argv[0], i);
890 parallel_grouplock();
891 MPI_Barrier(MPI_COMM_WORLD);
895 printf("%s: All tests passed!\n", timestamp());