From 36dc1ab9e0e6dbb31c0c4c86120d568bb968f20f Mon Sep 17 00:00:00 2001 From: green Date: Thu, 13 May 2004 19:36:10 +0000 Subject: [PATCH] b=2657 Added group lock tests. Those are not built by default. --- lustre/tests/Makefile.am | 2 + lustre/tests/lp_utils.c | 265 +++++++++++ lustre/tests/lp_utils.h | 103 +++++ lustre/tests/parallel_grouplock.c | 899 ++++++++++++++++++++++++++++++++++++++ 4 files changed, 1269 insertions(+) create mode 100644 lustre/tests/lp_utils.c create mode 100644 lustre/tests/lp_utils.h create mode 100644 lustre/tests/parallel_grouplock.c diff --git a/lustre/tests/Makefile.am b/lustre/tests/Makefile.am index 4d812e0..1430099 100644 --- a/lustre/tests/Makefile.am +++ b/lustre/tests/Makefile.am @@ -72,6 +72,8 @@ sleeptest_SOURCES = sleeptest.c #write_append_truncate_CC=mpicc #createmany_mpi_SOURCES=createmany_mpi.c #createmany_mpi_CC=mpicc +#parallel_grouplock_SOURCES=parallel_grouplock.c lp_utils.c +#parallel_grouplock_CC=mpicc copy_attr_SOURCES= copy_attr.c copy_attr_LDADD= -lattr diff --git a/lustre/tests/lp_utils.c b/lustre/tests/lp_utils.c new file mode 100644 index 0000000..0ca7a85 --- /dev/null +++ b/lustre/tests/lp_utils.c @@ -0,0 +1,265 @@ +/* -*- mode: c; c-basic-offset: 8; indent-tabs-mode: nil; -*- + * vim:expandtab:shiftwidth=8:tabstop=8: + * + * Copyright (C) 2002 Cluster File Systems, Inc. + * Author: You Feng + * + * This file is part of Lustre, http://www.lustre.org. + * + * Lustre is free software; you can redistribute it and/or + * modify it under the terms of version 2 of the GNU General Public + * License as published by the Free Software Foundation. + * + * Lustre is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with Lustre; if not, write to the Free Software + * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA. + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include "lustre/lustre_user.h" +#include "lp_utils.h" + +#define MAX_PROCESSES 8 + +int verbose = 0; +int debug = 0; + +char hostname[1024]; + +struct timeval t1, t2; + +char *timestamp() { + static char datestring[80]; + time_t timestamp; + + fflush(stdout); + timestamp = time(NULL); + strftime(datestring, 80, "%T", localtime(×tamp)); + + return datestring; +} + +inline void begin(char *str) { + if (verbose > 0 && rank == 0) { + gettimeofday(&t1, NULL); + printf("%s:\tBeginning %s\n", timestamp(), str); + fflush(stdout); + } +} + +inline void end(char *str) { + float elapsed; + + MPI_Barrier(MPI_COMM_WORLD); + if (verbose > 0 && rank == 0) { + gettimeofday(&t2, NULL); + elapsed = (t2.tv_sec + ((float)t2.tv_usec/1000000)) + - (t1.tv_sec + ((float)t1.tv_usec/1000000)); + if (elapsed >= 60) { + printf("%s:\tFinished %-15s(%.2f min)\n", + timestamp(), str, elapsed / 60); + } else { + printf("%s:\tFinished %-15s(%.3f sec)\n", + timestamp(), str, elapsed); + + } + fflush(stdout); + } +} + +void dump_diff(char *orig_buf, char *buf, int size, long _off) +{ + int i, diff, off; + char *p, *end; + + printf("commpared buf size %d, at offset %lu\n\n", size, _off); + + if (orig_buf) { + printf("original buf:\n"); + p = orig_buf; + end = orig_buf + size; + i = 1; + while (p < end) { + printf(" %8lx", *(long *)p); + p += sizeof(long); + if (i++%8 == 0) + printf("\n"); + } + if (i%8) printf("\n\n"); + else printf("\n"); + } + + if (buf) { + printf("different data: diff_data(orig_data)\n"); + diff = 0; + off = 0; + i = 1; + p = buf; + end = buf + size; + while (p < end) { + if (memcmp(p, orig_buf + off, sizeof(long)) != 0) { + printf("\toff: %5d,\tdata: %8lx (%8lx)\n", off, + *(unsigned long *)p, + *(unsigned long *)(orig_buf + off)); + diff++; + } + off += sizeof(long); + p += sizeof(long); + } + printf("\n %d total differents found\n\n", diff); + } +} + +void lp_gethostname(void) +{ + if (gethostname(hostname, 1024) == -1) { + fprintf(stderr, "gethostname: (%d)%s", errno, strerror(errno)); + MPI_Abort(MPI_COMM_WORLD, 2); + } +} + +/* This function does not FAIL if the requested "name" does not exit. + * This is just to clean up any files or directories left over from + * previous runs + */ +void remove_file_or_dir(char *name) +{ + struct stat statbuf; + char errmsg[MAX_FILENAME_LEN + 20]; + + if (stat(name, &statbuf) != -1) { + if (S_ISREG(statbuf.st_mode)) { + printf("stale file found\n"); + if (unlink(name) == -1) { + sprintf(errmsg, "unlink of %s", name); + FAIL(errmsg); + } + } + if (S_ISDIR(statbuf.st_mode)) { + printf("stale directory found\n"); + if (rmdir(name) == -1) { + sprintf(errmsg, "rmdir of %s", name); + FAIL(errmsg); + } + } + } +} + +void create_file(char *name, long filesize, int fill) +{ + static char filename[MAX_FILENAME_LEN]; + char errmsg[MAX_FILENAME_LEN + 20]; + char buf[1024 * 8]; + char c = 'A' + size; + int fd, rc; + short zero = 0; + long left = filesize; + + /* Process 0 creates the test file(s) */ + if (rank == 0) { + sprintf(filename, "%s/%s", testdir, name); + remove_file_or_dir(filename); + if ((fd = creat(filename, FILEMODE)) == -1) { + sprintf(errmsg, "create of file %s", filename); + FAIL(errmsg); + } + if (filesize > 0) { + if (lseek(fd, filesize - 1, SEEK_SET) == -1) { + close(fd); + sprintf(errmsg, "lseek of file %s", filename); + FAIL(errmsg); + } + if (write(fd, &zero, 1) == -1) { + close(fd); + sprintf(errmsg, "write of file %s", filename); + FAIL(errmsg); + } + } + if (filesize > 0 && fill) { + if (lseek(fd, 0, SEEK_SET) == -1) { + close(fd); + sprintf(errmsg, "lseek of file %s", filename); + FAIL(errmsg); + } + memset(buf, c, 1024); + while (left > 0) { + if ((rc = write(fd, buf, + left > (1024 * 8) ? (1024 * 8) : left)) + == -1) { + close(fd); + sprintf(errmsg, "write of file %s", filename); + FAIL(errmsg); + } + left -= rc; + } + } + if (close(fd) == -1) { + sprintf(errmsg, "close of file %s", filename); + FAIL(errmsg); + } + } +} + +void check_stat(char *filename, struct stat *state, struct stat *old_state) +{ + char errmsg[MAX_FILENAME_LEN+20]; + + if (stat(filename, state) == -1) { + sprintf(errmsg, "stat of file %s", filename); + FAIL(errmsg); + } + + if (memcmp(state, old_state, sizeof(struct stat)) != 0) { + errno = 0; + sprintf(errmsg, LP_STAT_FMT, LP_STAT_ARGS); + FAIL(errmsg); + } +} + +void remove_file(char *name) +{ + char filename[MAX_FILENAME_LEN]; + char errmsg[MAX_FILENAME_LEN + 20]; + + /* Process 0 remove the file(s) */ + if (rank == 0) { + sprintf(filename, "%s/%s", testdir, name); + if (unlink(filename) == -1) { + sprintf(errmsg, "unlink of file %s", filename); + FAIL(errmsg); + } + } +} + +void fill_stride(char *buf, int buf_size, long long rank, long long _off) +{ + char *p = buf; + long long off, data[2]; + int cp, left = buf_size; + + data[0] = rank; + off = _off; + while (left > 0) { + data[1] = off; + cp = left > sizeof(data) ? sizeof(data) : left; + memcpy(p, data, cp); + off += cp; + p += cp; + left -= cp; + } +} diff --git a/lustre/tests/lp_utils.h b/lustre/tests/lp_utils.h new file mode 100644 index 0000000..993253a --- /dev/null +++ b/lustre/tests/lp_utils.h @@ -0,0 +1,103 @@ +/* -*- mode: c; c-basic-offset: 8; indent-tabs-mode: nil; -*- + * vim:expandtab:shiftwidth=8:tabstop=8: + * + * Copyright (C) 2002 Cluster File Systems, Inc. + * Author: You Feng + * + * This file is part of Lustre, http://www.lustre.org. + * + * Lustre is free software; you can redistribute it and/or + * modify it under the terms of version 2 of the GNU General Public + * License as published by the Free Software Foundation. + * + * Lustre is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with Lustre; if not, write to the Free Software + * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA. + */ + +#ifndef __LP_UTILS_H__ +#define __LP_UTILS_H__ + +#include "lustre/lustre_user.h" + +#define FAIL(msg) \ + \ +do { \ + printf("%s: Process %d (%s)\n", timestamp(), rank, hostname); \ + if (debug) \ + printf("\tFAILED in %s:%d:%s()\n", __FILE__, __LINE__, __func__); \ + else \ + printf("\tFAILED in %s()\n", __func__); \ + printf("%s", msg); \ + fflush(stdout); \ + MPI_Abort(MPI_COMM_WORLD, 1); \ +} while(0) + +#define FILEMODE S_IRUSR|S_IWUSR|S_IRGRP|S_IWGRP|S_IROTH +#define MAX_FILENAME_LEN 512 + +extern int verbose; +extern int debug; + +extern int rank; +extern int size; + +extern char hostname[]; +extern char *timestamp(); +extern char *testdir; + +extern inline void begin(char *str); +extern inline void end(char *str); + +extern void dump_diff(char *orig_buf, char *buf, int len, long off); +extern void lp_gethostname(void); + +extern void create_file(char *name, long filesize, int fill); +extern void fill_file(char *name, long filesize); + +#define LP_STAT_FMT \ + \ +"Stat error:\n \ +\tfields\t\tvalue\told value\n \ +\tst_dev\t\t%d\t%d\n \ +\tst_ino\t\t%d\t%d\n \ +\tst_mode\t\t%o\t%o\n \ +\tst_nlink\t%d\t%d\n \ +\tst_uid\t\t%d\t%d\n \ +\tst_gid\t\t%d\t%d\n \ +\tst_rdev\t\t%x.%x\t%x.%x\n \ +\tst_size\t\t%lu\t%lu\n \ +\tst_blksize\t%d\t%d\n \ +\tst_blocks\t%u\t%u\n \ +\tst_atime\t%d\t%d\n \ +\tst_mtime\t%d\t%d\n \ +\tst_ctime\t%d\t%d\n" + +#define LP_STAT_ARGS \ + \ +(int)state->st_dev, (int)old_state->st_dev, \ +(int)state->st_ino, (int)old_state->st_ino, \ +state->st_mode & 07777, old_state->st_mode & 07777, \ +(int)state->st_nlink, (int)old_state->st_nlink, \ +state->st_uid, old_state->st_uid, \ +state->st_gid, old_state->st_gid, \ +(int)((state->st_rdev >> 8) & 0xff), (int)(state->st_rdev & 0xff), \ +(int)((old_state->st_rdev >> 8) & 0xff), (int)(old_state->st_rdev & 0xff), \ +(unsigned long)state->st_size, (unsigned long)old_state->st_size, \ +(int)state->st_blksize, (int)old_state->st_blksize, \ +(unsigned int)state->st_blocks, (unsigned int)old_state->st_blocks, \ +(int)state->st_atime, (int)old_state->st_atime, \ +(int)state->st_mtime, (int)old_state->st_mtime, \ +(int)state->st_ctime, (int)old_state->st_ctime + +extern void check_stat(char *filename, struct stat *state, struct stat *old_state); +extern void remove_file(char *name); +extern void remove_file_or_dir(char *name); +extern void fill_stride(char *buf, int buf_size, long long rank, long long _off); + +#endif /* __LP_UTILS_H__ */ diff --git a/lustre/tests/parallel_grouplock.c b/lustre/tests/parallel_grouplock.c new file mode 100644 index 0000000..2369209 --- /dev/null +++ b/lustre/tests/parallel_grouplock.c @@ -0,0 +1,899 @@ +/* -*- mode: c; c-basic-offset: 8; indent-tabs-mode: nil; -*- + * vim:expandtab:shiftwidth=8:tabstop=8: + * + * Copyright (C) 2002 Cluster File Systems, Inc. + * Author: You Feng + * + * This file is part of Lustre, http://www.lustre.org. + * + * Lustre is free software; you can redistribute it and/or + * modify it under the terms of version 2 of the GNU General Public + * License as published by the Free Software Foundation. + * + * Lustre is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with Lustre; if not, write to the Free Software + * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA. + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include "lp_utils.h" + +#define LPGL_FILEN 700000 +#define LPGL_TEST_ITEMS 7 + +#define MAX_GLHOST 4 + +/* waiting time in 0.1 s */ +#define MAX_WAITING_TIME 20 +int rank = 0; +int size = 0; + +char *testdir = NULL; + +/* + * process1 attempts CW(gid=1) -- granted immediately + * process2 attempts PR -- blocked, goes on waiting list + * process3 attempts CW(gid=1) -> should be granted, but may go on + * the waiting list + */ +void grouplock_test1(char *filename, int fd, char *errmsg) +{ + int rc, count, gid = 1; + char buf[LPGL_FILEN]; + char zeros[LPGL_FILEN]; + MPI_Request req1, req2; + int temp1, temp2; + + if (rank == 0) { + if ((rc = ioctl(fd, LL_IOC_GROUP_LOCK, gid)) == -1) { + sprintf(errmsg, "ioctl GROUP_LOCK of file %s return %d", + filename, rc); + FAIL(errmsg); + } + } + + MPI_Barrier(MPI_COMM_WORLD); + + if (rank == 1) { + memset(zeros, 0x0, sizeof(zeros)); + lseek(fd, 0, SEEK_SET); + + MPI_Send(&gid, 1, MPI_INT, 2, 1, MPI_COMM_WORLD); + count = read(fd, buf, sizeof(buf)); + if (count != sizeof(buf)) { + if (count > 0) + dump_diff(zeros, buf, count, 0); + sprintf(errmsg, "read of file %s return %d", + filename, count); + FAIL(errmsg); + } + MPI_Send(&gid, 1, MPI_INT, 0, 1, MPI_COMM_WORLD); + } + + if (rank == 2) { + int temp; + + /* Wait for reading task to progress, this is probably somewhat + racey, though, may be adding usleep here would make things + better here. */ + usleep(100); + MPI_Recv(&temp, 1, MPI_INT, 1, 1, MPI_COMM_WORLD, + MPI_STATUS_IGNORE); + if ((rc = ioctl(fd, LL_IOC_GROUP_LOCK, gid)) == -1) { + sprintf(errmsg, "ioctl GROUP_LOCK of file %s return %d", + filename, rc); + FAIL(errmsg); + } + MPI_Send(&gid, 1, MPI_INT, 0, 1, MPI_COMM_WORLD); + } + + if (rank == 0) { + int iter = MAX_WAITING_TIME; + int flag1, flag2; + + /* reading task will tell us when it completes */ + MPI_Irecv(&temp1, 1, MPI_INT, 1, 1, MPI_COMM_WORLD, &req1); + /* 2nd locking task will tell us when it completes */ + MPI_Irecv(&temp2, 1, MPI_INT, 2, 1, MPI_COMM_WORLD, &req2); + + do { + iter--; + if (!iter) { + FAIL("2nd locking task is not progressing\n"); + } + usleep(100); + MPI_Test(&req2, &flag2, MPI_STATUS_IGNORE); + MPI_Test(&req1, &flag1, MPI_STATUS_IGNORE); + if (flag1) { + FAIL("PR task progressed even though GROUP lock" + " is held\n"); + } + } while (!flag2); + } + + /* Now we need to release the lock */ + + if (rank == 0 || rank == 2) { + if ((rc = ioctl(fd, LL_IOC_GROUP_UNLOCK, gid)) == -1) { + sprintf(errmsg, "ioctl GROUP_UNLOCK of file %s return %d", + filename, rc); + FAIL(errmsg); + } + } + + if (rank == 0) { + int iter = MAX_WAITING_TIME; + int flag1; + + do { + iter--; + if (!iter) { + FAIL("reading task is not progressing even " + "though GROUP lock was released\n"); + break; + } + usleep(100); + MPI_Test(&req1, &flag1, MPI_STATUS_IGNORE); + } while (!flag1); + } + + MPI_Barrier(MPI_COMM_WORLD); + +} + +/* + * process1 attempts CW(gid=1) -- granted immediately + * process2 attempts CW(gid=2) -- blocked + * process3 attempts PR -- blocked + * process4 attempts CW(gid=2) -- blocked + * process1 releases CW(gid=1) -- this allows process2's CW lock to be granted + process3 remains blocked + */ +void grouplock_test2(char *filename, int fd, char *errmsg) +{ + int rc, count, gid = 1; + char buf[LPGL_FILEN]; + char zeros[LPGL_FILEN]; + MPI_Request req1, req2, req3; + int temp1, temp2, temp3; + + if (rank == 0) { + if ((rc = ioctl(fd, LL_IOC_GROUP_LOCK, gid)) == -1) { + sprintf(errmsg, "ioctl GROUP_LOCK of file %s return %d", + filename, rc); + FAIL(errmsg); + } + } + + MPI_Barrier(MPI_COMM_WORLD); + + if (rank == 1 || rank == 3) { + gid = 2; + if (rank == 3) { + MPI_Recv(&temp1, 1, MPI_INT, 2, 1, MPI_COMM_WORLD, + MPI_STATUS_IGNORE); + usleep(100); + } + if ((rc = ioctl(fd, LL_IOC_GROUP_LOCK, gid)) == -1) { + sprintf(errmsg, "ioctl GROUP_LOCK of file %s return %d", + filename, rc); + FAIL(errmsg); + } + MPI_Send(&gid, 1, MPI_INT, 0, 1, MPI_COMM_WORLD); + } + + if (rank == 2) { + memset(zeros, 0x0, sizeof(zeros)); + lseek(fd, 0, SEEK_SET); + + MPI_Send(&gid, 1, MPI_INT, 3, 1, MPI_COMM_WORLD); + count = read(fd, buf, sizeof(buf)); + if (count != sizeof(buf)) { + if (count > 0) + dump_diff(zeros, buf, count, 0); + sprintf(errmsg, "read of file %s return %d", + filename, count); + FAIL(errmsg); + } + MPI_Send(&gid, 1, MPI_INT, 0, 1, MPI_COMM_WORLD); + } + + if (rank == 0) { + int iter = MAX_WAITING_TIME; + int flag1, flag2, flag3; + + /* 2nd locking task will tell us when it completes */ + MPI_Irecv(&temp1, 1, MPI_INT, 1, 1, MPI_COMM_WORLD, &req1); + /* 3nd locking task will tell us when it completes */ + MPI_Irecv(&temp2, 1, MPI_INT, 3, 1, MPI_COMM_WORLD, &req2); + /* reading task will tell us when it completes */ + MPI_Irecv(&temp3, 1, MPI_INT, 2, 1, MPI_COMM_WORLD, &req3); + + do { + iter--; + usleep(100); + MPI_Test(&req2, &flag2, MPI_STATUS_IGNORE); + MPI_Test(&req1, &flag1, MPI_STATUS_IGNORE); + MPI_Test(&req3, &flag3, MPI_STATUS_IGNORE); + if (flag3) { + FAIL("PR task progressed even though GROUP lock" + " is held\n"); + } + if (flag1 || flag2) { + FAIL("GROUP (gid=2) task progressed even though" + " GROUP (gid=1) lock is held\n"); + } + + } while (iter); + + /* Now let's release first lock */ + if ((rc = ioctl(fd, LL_IOC_GROUP_UNLOCK, gid)) == -1) { + sprintf(errmsg, "ioctl GROUP_UNLOCK of file %s return %d", + filename, rc); + FAIL(errmsg); + } + iter = MAX_WAITING_TIME; + do { + iter--; + if (!iter) { + FAIL("GROUP(gid=2) tasks are not progressing\n"); + } + usleep(100); + MPI_Test(&req2, &flag2, MPI_STATUS_IGNORE); + MPI_Test(&req1, &flag1, MPI_STATUS_IGNORE); + MPI_Test(&req3, &flag3, MPI_STATUS_IGNORE); + if (flag3) { + fprintf(stderr, "task1 %d, task3 %d\n", flag1, + flag2); + FAIL("PR task progressed even though GROUP lock" + " was on the queue task\n"); + } + } while (!(flag1 && flag2)); + MPI_Send(&gid, 1, MPI_INT, 1, 1, MPI_COMM_WORLD); + MPI_Send(&gid, 1, MPI_INT, 3, 1, MPI_COMM_WORLD); + } + + if (rank == 1 || rank == 3) { + /* Do not release the locks until task 0 is ready to watch + for reading task only */ + MPI_Recv(&temp1, 1, MPI_INT, 0, 1, MPI_COMM_WORLD, + MPI_STATUS_IGNORE); + if ((rc = ioctl(fd, LL_IOC_GROUP_UNLOCK, gid)) == -1) { + sprintf(errmsg, "ioctl GROUP_UNLOCK of file %s return %d", + filename, rc); + FAIL(errmsg); + } + } + + if (rank == 0) { + int iter = MAX_WAITING_TIME; + int flag3; + + do { + iter--; + if (!iter) { + FAIL("reading task is not progressing even " + "though GROUP locks are released\n"); + break; + } + usleep(100); + MPI_Test(&req3, &flag3, MPI_STATUS_IGNORE); + } while (!flag3); + } + + MPI_Barrier(MPI_COMM_WORLD); + +} + +/* + * process1 attempts CW(gid=1) -- granted + * process2 attempts PR -- blocked + * process3 attempts CW(gid=1) -> should be granted + * process3 releases CW(gid=1) + * process2 should remain blocked + * process1 releases CW(gid=1) + * process2's PR should be granted + * + * This is a lot like test1. + */ +void grouplock_test3(char *filename, int fd, char *errmsg) +{ + int rc, count, gid = 1; + char buf[LPGL_FILEN]; + char zeros[LPGL_FILEN]; + MPI_Request req1, req2; + int temp1, temp2; + + if (rank == 0) { + if ((rc = ioctl(fd, LL_IOC_GROUP_LOCK, gid)) == -1) { + sprintf(errmsg, "ioctl GROUP_LOCK of file %s return %d", + filename, rc); + FAIL(errmsg); + } + } + + MPI_Barrier(MPI_COMM_WORLD); + + if (rank == 1) { + memset(zeros, 0x0, sizeof(zeros)); + lseek(fd, 0, SEEK_SET); + + MPI_Send(&gid, 1, MPI_INT, 2, 1, MPI_COMM_WORLD); + count = read(fd, buf, sizeof(buf)); + if (count != sizeof(buf)) { + if (count > 0) + dump_diff(zeros, buf, count, 0); + sprintf(errmsg, "read of file %s return %d", + filename, count); + FAIL(errmsg); + } + MPI_Send(&gid, 1, MPI_INT, 0, 1, MPI_COMM_WORLD); + } + + if (rank == 2) { + int temp; + + /* Wait for reading task to progress, this is probably somewhat + racey, though, may be adding usleep here would make things + better here. */ + usleep(100); + MPI_Recv(&temp, 1, MPI_INT, 1, 1, MPI_COMM_WORLD, + MPI_STATUS_IGNORE); + if ((rc = ioctl(fd, LL_IOC_GROUP_LOCK, gid)) == -1) { + sprintf(errmsg, "ioctl GROUP_LOCK of file %s return %d", + filename, rc); + FAIL(errmsg); + } + MPI_Send(&gid, 1, MPI_INT, 0, 1, MPI_COMM_WORLD); + } + + if (rank == 0) { + int iter = MAX_WAITING_TIME; + int flag1, flag2; + + /* reading task will tell us when it completes */ + MPI_Irecv(&temp1, 1, MPI_INT, 1, 1, MPI_COMM_WORLD, &req1); + /* 2nd locking task will tell us when it completes */ + MPI_Irecv(&temp2, 1, MPI_INT, 2, 1, MPI_COMM_WORLD, &req2); + + do { + iter--; + if (!iter) { + FAIL("2nd locking task is not progressing\n"); + } + usleep(100); + MPI_Test(&req2, &flag2, MPI_STATUS_IGNORE); + MPI_Test(&req1, &flag1, MPI_STATUS_IGNORE); + if (flag1) { + FAIL("PR task progressed even though GROUP lock" + " is held\n"); + } + } while (!flag2); + } + + /* Now we need to release the lock */ + + if (rank == 2) { + if ((rc = ioctl(fd, LL_IOC_GROUP_UNLOCK, gid)) == -1) { + sprintf(errmsg, "ioctl GROUP_UNLOCK of file %s return %d", + filename, rc); + FAIL(errmsg); + } + } + + if (rank == 0) { + int iter = MAX_WAITING_TIME; + int flag1; + + do { + iter--; + usleep(100); + MPI_Test(&req1, &flag1, MPI_STATUS_IGNORE); + } while (!flag1 && iter); + if (iter) { + FAIL("reading task is progressing even " + "though GROUP lock was not fully released\n"); + } + + iter = MAX_WAITING_TIME; + + if ((rc = ioctl(fd, LL_IOC_GROUP_UNLOCK, gid)) == -1) { + sprintf(errmsg, "ioctl GROUP_UNLOCK of file %s return %d", + filename, rc); + FAIL(errmsg); + } + + do { + iter--; + if (!iter) { + FAIL("reading task is not progressing even " + "though GROUP lock was released\n"); + break; + } + usleep(100); + MPI_Test(&req1, &flag1, MPI_STATUS_IGNORE); + } while (!flag1); + } + + MPI_Barrier(MPI_COMM_WORLD); + +} + +/* + * process1 attempts CW(gid=1) -- granted + * process2 attempts PR on non-blocking fd -> should return -EWOULDBLOCK + * process3 attempts CW(gid=2) on non-blocking fd -> should return -EWOULDBLOCK + */ +void grouplock_test4(char *filename, int fd, char *errmsg) +{ + int rc, count, gid = 1; + char buf[LPGL_FILEN]; + char zeros[LPGL_FILEN]; + + if (rank == 0) { + if ((rc = ioctl(fd, LL_IOC_GROUP_LOCK, gid)) == -1) { + sprintf(errmsg, "ioctl GROUP_LOCK of file %s return %d", + filename, rc); + FAIL(errmsg); + } + } + + MPI_Barrier(MPI_COMM_WORLD); + + if (rank == 1) { + memset(zeros, 0x0, sizeof(zeros)); + lseek(fd, 0, SEEK_SET); + + count = read(fd, buf, sizeof(buf)); + if (count != sizeof(buf)) { + if (count == -1 && errno == EWOULDBLOCK) { + MPI_Send(&gid, 1, MPI_INT, 0, 1, MPI_COMM_WORLD); + return; + } + + if (count > 0) + dump_diff(zeros, buf, count, 0); + sprintf(errmsg, "read of file %s return %d", + filename, count); + FAIL(errmsg); + } else { + FAIL("PR lock succeed while incompatible " + "GROUP LOCK (gid=1) is still held\n"); + } + } + + if (rank == 2) { + gid = 2; + if ((rc = ioctl(fd, LL_IOC_GROUP_LOCK, gid)) == -1) { + if (errno == EWOULDBLOCK) { + MPI_Send(&gid, 1, MPI_INT, 0, 1, MPI_COMM_WORLD); + return; + } + + sprintf(errmsg, "ioctl GROUP_LOCK of file %s return %d", + filename, rc); + FAIL(errmsg); + } else { + FAIL("GROUP_LOCK (gid=2) succeed while incompatible " + "GROUP LOCK (gid=1) is still held\n"); + } + } + + + if ( rank == 0) { + int iter = MAX_WAITING_TIME; + int flag1, flag2; + MPI_Request req1, req2; + int temp1, temp2; + + /* reading task will tell us when it completes */ + MPI_Irecv(&temp1, 1, MPI_INT, 1, 1, MPI_COMM_WORLD, &req1); + /* 2nd locking task will tell us when it completes */ + MPI_Irecv(&temp2, 1, MPI_INT, 2, 1, MPI_COMM_WORLD, &req2); + + do { + iter--; + if (!iter) { + FAIL("non-blocking tasks are not progressing\n"); + } + usleep(100); + MPI_Test(&req2, &flag2, MPI_STATUS_IGNORE); + MPI_Test(&req1, &flag1, MPI_STATUS_IGNORE); + } while (!(flag2 && flag1)); + + if ((rc = ioctl(fd, LL_IOC_GROUP_UNLOCK, gid)) == -1) { + sprintf(errmsg, "ioctl GROUP_UNLOCK of file %s", filename); + FAIL(errmsg); + } + } +} + +/* + * process1 attempts CW(gid=1) -- granted + * process2 attempts CW(gid=2) -- blocked + * process3 attempts CW(gid=2) -- blocked + * process1 releases CW(gid=1) + * process2's CW(gid=2) should be granted + * process3's CW(gid=2) should be granted + * + * This is pretty much like test 3 + */ +void grouplock_test5(char *filename, int fd, char *errmsg) +{ + int rc, count, gid = 1; + char buf[LPGL_FILEN]; + char zeros[LPGL_FILEN]; + MPI_Request req1, req2; + int temp1, temp2; + + if (rank == 0) { + if ((rc = ioctl(fd, LL_IOC_GROUP_LOCK, gid)) == -1) { + sprintf(errmsg, "ioctl GROUP_LOCK of file %s return %d", + filename, rc); + FAIL(errmsg); + } + } + + MPI_Barrier(MPI_COMM_WORLD); + + if (rank == 2 || rank == 1) { + gid = 2; + if ((rc = ioctl(fd, LL_IOC_GROUP_LOCK, gid)) == -1) { + sprintf(errmsg, "ioctl GROUP_LOCK of file %s return %d", + filename, rc); + FAIL(errmsg); + } + MPI_Send(&gid, 1, MPI_INT, 0, 1, MPI_COMM_WORLD); + } + + if (rank == 0) { + int iter = MAX_WAITING_TIME; + int flag1, flag2; + + /* 3rd locking task will tell us when it completes */ + MPI_Irecv(&temp1, 1, MPI_INT, 1, 1, MPI_COMM_WORLD, &req1); + /* 2nd locking task will tell us when it completes */ + MPI_Irecv(&temp2, 1, MPI_INT, 2, 1, MPI_COMM_WORLD, &req2); + + do { + iter--; + usleep(100); + MPI_Test(&req2, &flag2, MPI_STATUS_IGNORE); + MPI_Test(&req1, &flag1, MPI_STATUS_IGNORE); + } while (!flag2 && !flag1 && iter); + if (iter) { + FAIL("incomptible locking tasks are progressing\n"); + } + } + + /* Now we need to release the lock */ + + if (rank == 0) { + int iter = MAX_WAITING_TIME; + int flag1, flag2; + if ((rc = ioctl(fd, LL_IOC_GROUP_UNLOCK, gid)) == -1) { + sprintf(errmsg, "ioctl GROUP_UNLOCK of file %s return %d", + filename, rc); + FAIL(errmsg); + } + + do { + iter--; + if (!iter) { + FAIL("locking tasks are not progressing even " + "though incompatible lock released\n"); + } + usleep(100); + MPI_Test(&req1, &flag1, MPI_STATUS_IGNORE); + MPI_Test(&req2, &flag2, MPI_STATUS_IGNORE); + } while (!(flag1 && flag2)); + + } + + if ( rank == 1 || rank == 2) { + if ((rc = ioctl(fd, LL_IOC_GROUP_UNLOCK, gid)) == -1) { + sprintf(errmsg, "ioctl GROUP_UNLOCK of file %s return %d", + filename, rc); + FAIL(errmsg); + } + } + + MPI_Barrier(MPI_COMM_WORLD); + + +} + +/* + * process1 attempts CW(gid=1) -- granted + * process2 attempts PW -- blocked + * process2 attempts CW(gid=2) -- blocked + * process3 attempts CW(gid=2) -- blocked + * process1 releases CW(gid=1) + * process2's CW(gid=2) should be granted + * process3's CW(gid=2) should be granted + * + * after process1 release CW(gid=1), there are two pathes: + * path 1. process2 get PW + * path 2. process3 get CW(gid=2) + * + * green: Also about test6 - by definition if P* and CW lock are waiting, + * CW lock have bigger priority and should be granted first when it becomes + * possible. So after process1 releases its CW lock, process3 should always + * get CW lock, and when it will release it, process 2 will proceed with read + * and then with getting CW lock + * + * XXX This test does not make any sence at all the way it is described right + * now, hence disabled. + */ +void grouplock_test6(char *filename, int fd, char *errmsg) +{ +} + +/* Just test some error paths with invalid requests */ +void grouplock_errorstest(char *filename, int fd, char *errmsg) +{ + int gid = 1; + int rc; + + /* To not do lots of separate tests with lots of fd opening/closing, + different parts of this test are performed in different processes */ + + if (rank == 0 || rank == 1 ) { + if ((rc = ioctl(fd, LL_IOC_GROUP_LOCK, gid)) == -1) { + sprintf(errmsg, "ioctl GROUP_LOCK of file %s return %d", + filename, rc); + FAIL(errmsg); + } + } + + /* second group lock on same fd, same gid */ + if (rank == 0) { + if ((rc = ioctl(fd, LL_IOC_GROUP_LOCK, gid)) == -1) { + if (errno != EINVAL) { + sprintf(errmsg, "Double GROUP lock failed with errno %d instead of EINVAL\n", errno); + FAIL(errmsg); + } + } else { + FAIL("Taking second GROUP lock on same fd succeed\n"); + } + } + + /* second group lock on same fd, different gid */ + if (rank == 1) { + if ((rc = ioctl(fd, LL_IOC_GROUP_LOCK, gid + 1)) == -1) { + if (errno != EINVAL) { + sprintf(errmsg, "Double GROUP lock different gid failed with errno %d instead of EINVAL\n", errno); + FAIL(errmsg); + } + } else { + FAIL("Taking second GROUP lock on same fd, different gid, succeed\n"); + } + } + + /* GROUP unlock with wrong gid */ + if (rank == 0 || rank == 1) { + if ((rc = ioctl(fd, LL_IOC_GROUP_UNLOCK, gid + 1)) == -1) { + if (errno != EINVAL) { + sprintf(errmsg, "GROUP unlock with wrong gid failed with errno %d instead of EINVAL\n", + errno); + FAIL(errmsg); + } + } else { + FAIL("GROUP unlock with wrong gid succeed\n"); + } + } + + if (rank == 0 || rank == 1) { + if ((rc = ioctl(fd, LL_IOC_GROUP_UNLOCK, gid)) == -1) { + sprintf(errmsg, "ioctl GROUP_UNLOCK of file %s return %d", + filename, rc); + FAIL(errmsg); + } + } + + /* unlock of never locked fd */ + if (rank == 2) { + if ((rc = ioctl(fd, LL_IOC_GROUP_UNLOCK, gid)) == -1) { + if (errno != EINVAL) { + sprintf(errmsg, "GROUP unlock on never locked fd failed with errno %d instead of EINVAL\n", + errno); + FAIL(errmsg); + } + } else { + FAIL("GROUP unlock on never locked fd succeed\n"); + } + } +} + +void grouplock_file(char *name, int items) +{ + int i, fd; + char filename[MAX_FILENAME_LEN]; + char errmsg[MAX_FILENAME_LEN+20]; + + sprintf(filename, "%s/%s", testdir, name); + + if (items == 4) { + if ((fd = open(filename, O_RDWR | O_NONBLOCK)) == -1) { + sprintf(errmsg, "open of file %s", filename); + FAIL(errmsg); + } + } else if ((fd = open(filename, O_RDWR)) == -1) { + sprintf(errmsg, "open of file %s", filename); + FAIL(errmsg); + } + + MPI_Barrier(MPI_COMM_WORLD); + + switch (items) { + case 1: + grouplock_test1(filename, fd, errmsg); + break; + case 2: + grouplock_test2(filename, fd, errmsg); + break; + case 3: + grouplock_test3(filename, fd, errmsg); + break; + case 4: + grouplock_test4(filename, fd, errmsg); + break; + case 5: + grouplock_test5(filename, fd, errmsg); + break; + case 6: + grouplock_test6(filename, fd, errmsg); + break; + case 7: + grouplock_errorstest(filename, fd, errmsg); + break; + default: + sprintf(errmsg, "wrong test case number %d (should be <= %d)", + items, LPGL_TEST_ITEMS); + FAIL(errmsg); + } + + MPI_Barrier(MPI_COMM_WORLD); + + if (close(fd) == -1) { + sprintf(errmsg, "close of file %s", filename); + FAIL(errmsg); + } + +} + +void parallel_grouplock(void) +{ + int i; + + for (i = 1;i <= LPGL_TEST_ITEMS;++i) { + begin("setup"); + create_file("parallel_grouplock", LPGL_FILEN, 0); + end("setup"); + + begin("test"); + grouplock_file("parallel_grouplock", i); + end("test"); + + begin("cleanup"); + remove_file("parallel_grouplock"); + end("cleanup"); + } +} + +void usage(char *proc) +{ + int i; + + if (rank == 0) { + printf("Usage: %s [-h] -d \n", proc); + printf(" [-n \"13\"] [-v] [-V #] [-g]\n"); + printf("\t-h: prints this help message\n"); + printf("\t-d: the directory in which the tests will run\n"); + printf("\t-n: repeat test # times\n"); + printf("\t-v: increase the verbositly level by 1\n"); + printf("\t-V: select a specific verbosity level\n"); + printf("\t-g: debug mode\n"); + } + + MPI_Initialized(&i); + if (i) MPI_Finalize(); + exit(0); +} + +int main(int argc, char *argv[]) +{ + char c; + int i, iterations = 1; + int tr = 1; + + /* Check for -h parameter before MPI_Init so the binary can be + called directly, without, for instance, mpirun */ + for (i = 1; i < argc; ++i) { + if (!strcmp(argv[i], "-h") || !strcmp(argv[i], "--help")) + usage(argv[0]); + } + + MPI_Init(&argc, &argv); + MPI_Comm_rank(MPI_COMM_WORLD, &rank); + MPI_Comm_size(MPI_COMM_WORLD, &size); + +// MPI_Comm_set_attr(MPI_COMM_WORLD, MPI_WTIME_IS_GLOBAL, &tr); + + /* Parse command line options */ + while (1) { + c = getopt(argc, argv, "d:ghn:vV:"); + if (c == -1) + break; + + switch (c) { + case 'd': + testdir = optarg; + break; + case 'g': + debug = 1; + break; + case 'h': + usage(argv[0]); + break; + case 'n': + iterations = atoi(optarg); + break; + case 'v': + verbose += 1; + break; + case 'V': + verbose = atoi(optarg); + break; + } + } + + if (rank == 0) + printf("%s is running with %d process(es) %s\n", + argv[0], size, debug ? "in DEBUG mode" : "\b\b"); + + if (size < MAX_GLHOST) { + fprintf(stderr, "Error: " + "should be at least four processes to run the test!\n"); + MPI_Abort(MPI_COMM_WORLD, 2); + } + + if (testdir == NULL && rank == 0) { + fprintf(stderr, "Please specify a test directory! (\"%s -h\" for help)\n", + argv[0]); + MPI_Abort(MPI_COMM_WORLD, 2); + } + + lp_gethostname(); + + for (i = 0; i < iterations; ++i) { + if (rank == 0) + printf("%s: Running test #%s(iter %d)\n", + timestamp(), argv[0], i); + + parallel_grouplock(); + MPI_Barrier(MPI_COMM_WORLD); + } + + if (rank == 0) { + printf("%s: All tests passed!\n", timestamp()); + } + MPI_Finalize(); + return 0; +} -- 1.8.3.1