From e73cf72b82c4fe31acee81ffa6934312fd128359 Mon Sep 17 00:00:00 2001 From: Henri Doreau Date: Thu, 25 Sep 2014 08:50:20 +0200 Subject: [PATCH] LU-5666 llapi: LLAPI helpers for group lock. Introduced llapi_group_{lock,unlock} functions to abstract the ioctl implementation to manipulate group lock. Updated lfs accordingly. Signed-off-by: Henri Doreau Change-Id: I9c0c75029cd0d159e353a2c925cecc9b4ba9a6de Reviewed-on: http://review.whamcloud.com/12053 Tested-by: Jenkins Reviewed-by: frank zago Tested-by: Maloo Reviewed-by: Jinshan Xiong Reviewed-by: Oleg Drokin --- lustre/doc/llapi_group_lock.3 | 51 +++++++++++++++++++++++++++ lustre/doc/llapi_group_unlock.3 | 1 + lustre/include/lustre/lustreapi.h | 4 +++ lustre/utils/lfs.c | 73 +++++++++++++++++++++++++-------------- lustre/utils/liblustreapi.c | 42 ++++++++++++++++++++++ 5 files changed, 145 insertions(+), 26 deletions(-) create mode 100644 lustre/doc/llapi_group_lock.3 create mode 100644 lustre/doc/llapi_group_unlock.3 diff --git a/lustre/doc/llapi_group_lock.3 b/lustre/doc/llapi_group_lock.3 new file mode 100644 index 0000000..35f8896 --- /dev/null +++ b/lustre/doc/llapi_group_lock.3 @@ -0,0 +1,51 @@ +.TH llapi_group_lock 3 "2014 Oct 03" "Lustre User API" +.SH NAME +llapi_group_lock, llapi_group_unlock \- get and put a Lustre group lock. +.SH SYNOPSIS +.nf +.B #include +.PP +.BI "int llapi_group_lock(int "fd ", int "gid ); + +.BI "int llapi_group_unlock(int "fd ", int "gid ); +.fi +.SH DESCRIPTION +.PP +The function +.BR llapi_group_lock() +takes a group lock on the file descriptor +.I fd +with group identifier +.IR gid . + +The function +.BR llapi_group_unlock() +releases a group lock identified by group identifier +.I gid +on the file descriptor +.IR fd . + +The group lock is a whole file lock that blocks concurrent I/O originating from descriptors that have not been locked. Multiple processes can acquire a lock by specifying the same group identifier. + +.SH RETURN VALUES +.LP +.B llapi_group_lock(\|) +and +.B llapi_group_unlock(\|) +return 0 on success or a negative errno value on failure. +.SH ERRORS +.TP 15 +.SM -EBADF +.I fd +is not a valid file descriptor. +.TP +.SM -ENOTTY +.I fd +does not describe an object suitable for this request. +.TP +.SM -EINVAL +.I fd +is already group locked with a different group identifier. +.TP +.SH "SEE ALSO" +.BR liblustreapi (7) diff --git a/lustre/doc/llapi_group_unlock.3 b/lustre/doc/llapi_group_unlock.3 new file mode 100644 index 0000000..9650fff --- /dev/null +++ b/lustre/doc/llapi_group_unlock.3 @@ -0,0 +1 @@ +.so man3/llapi_group_lock.3 diff --git a/lustre/include/lustre/lustreapi.h b/lustre/include/lustre/lustreapi.h index 380dfc5..392ba5d 100644 --- a/lustre/include/lustre/lustreapi.h +++ b/lustre/include/lustre/lustreapi.h @@ -380,6 +380,10 @@ extern int llapi_lease_get(int fd, int mode); extern int llapi_lease_check(int fd); extern int llapi_lease_put(int fd); +/* Group lock */ +int llapi_group_lock(int fd, int gid); +int llapi_group_unlock(int fd, int gid); + /** @} llapi */ /* llapi_layout user interface */ diff --git a/lustre/utils/lfs.c b/lustre/utils/lfs.c index 3b2ee15..773976e 100644 --- a/lustre/utils/lfs.c +++ b/lustre/utils/lfs.c @@ -345,6 +345,36 @@ command_t cmdlist[] = { { 0, 0, 0, NULL } }; +/* Generate a random id for the grouplock */ +static int random_group_id(int *gid) +{ + int fd; + int rc; + size_t sz = sizeof(*gid); + + fd = open("/dev/urandom", O_RDONLY); + if (fd < 0) { + rc = -errno; + fprintf(stderr, "cannot open /dev/urandom: %s\n", + strerror(-rc)); + goto out; + } + + rc = read(fd, gid, sz); + if (rc < sz) { + rc = -errno; + fprintf(stderr, "cannot read %zu bytes from /dev/urandom: %s\n", + sz, strerror(-rc)); + goto out; + } + +out: + if (fd >= 0) + close(fd); + + return rc; +} + #define MIGRATION_BLOCKS 1 static int lfs_migrate(char *name, unsigned long long stripe_size, @@ -365,7 +395,7 @@ static int lfs_migrate(char *name, unsigned long long stripe_size, void *buf = NULL; int rsize, wsize; __u64 rpos, wpos, bufoff; - int gid = 0, sz; + int gid; int have_gl = 0; struct stat st, stv; @@ -393,21 +423,10 @@ static int lfs_migrate(char *name, unsigned long long stripe_size, } if (migration_flags & MIGRATION_BLOCKS) { - /* generate a random id for the grouplock */ - fd = open("/dev/urandom", O_RDONLY); - if (fd == -1) { - rc = -errno; - fprintf(stderr, "cannot open /dev/urandom (%s)\n", - strerror(-rc)); - goto free; - } - sz = sizeof(gid); - rc = read(fd, &gid, sz); - close(fd); - if (rc < sz) { - rc = -errno; - fprintf(stderr, "cannot read %d bytes from" - " /dev/urandom (%s)\n", sz, strerror(-rc)); + rc = random_group_id(&gid); + if (rc < 0) { + fprintf(stderr, "%s: cannot get random group ID: %s\n", + name, strerror(-rc)); goto free; } } @@ -503,8 +522,8 @@ static int lfs_migrate(char *name, unsigned long long stripe_size, * be implemented (see LU-2919) */ /* group lock is taken after data version read because it * blocks data version call */ - if (ioctl(fd, LL_IOC_GROUP_LOCK, gid) == -1) { - rc = -errno; + rc = llapi_group_lock(fd, gid); + if (rc < 0) { fprintf(stderr, "cannot get group lock on %s (%s)\n", name, strerror(-rc)); goto error; @@ -551,11 +570,10 @@ static int lfs_migrate(char *name, unsigned long long stripe_size, if (migration_flags & MIGRATION_BLOCKS) { /* give back group lock */ - if (ioctl(fd, LL_IOC_GROUP_UNLOCK, gid) == -1) { - rc = -errno; + rc = llapi_group_unlock(fd, gid); + if (rc < 0) fprintf(stderr, "cannot put group lock on %s (%s)\n", name, strerror(-rc)); - } have_gl = 0; } @@ -580,11 +598,14 @@ static int lfs_migrate(char *name, unsigned long long stripe_size, error: /* give back group lock */ - if ((migration_flags & MIGRATION_BLOCKS) && have_gl && - (ioctl(fd, LL_IOC_GROUP_UNLOCK, gid) == -1)) { - /* we keep in rc the original error */ - fprintf(stderr, "cannot put group lock on %s (%s)\n", - name, strerror(-errno)); + if ((migration_flags & MIGRATION_BLOCKS) && have_gl) { + int rc2; + + /* we keep the original error in rc */ + rc2 = llapi_group_unlock(fd, gid); + if (rc2 < 0) + fprintf(stderr, "cannot put group lock on %s (%s)\n", + name, strerror(-rc2)); } close(fdv); diff --git a/lustre/utils/liblustreapi.c b/lustre/utils/liblustreapi.c index 2cf1001..2631807 100644 --- a/lustre/utils/liblustreapi.c +++ b/lustre/utils/liblustreapi.c @@ -4653,3 +4653,45 @@ int llapi_open_by_fid(const char *lustre_dir, const lustre_fid *fid, int flags) snprintf(path, sizeof(path), "%s/.lustre/fid/"DFID, mntdir, PFID(fid)); return open(path, flags); } + +/** + * Take group lock. + * + * \param fd File to lock. + * \param gid Group Identifier. + * + * \retval 0 on success. + * \retval -errno on failure. + */ +int llapi_group_lock(int fd, int gid) +{ + int rc; + + rc = ioctl(fd, LL_IOC_GROUP_LOCK, gid); + if (rc < 0) { + rc = -errno; + llapi_error(LLAPI_MSG_ERROR, rc, "cannot get group lock"); + } + return rc; +} + +/** + * Put group lock. + * + * \param fd File to unlock. + * \param gid Group Identifier. + * + * \retval 0 on success. + * \retval -errno on failure. + */ +int llapi_group_unlock(int fd, int gid) +{ + int rc; + + rc = ioctl(fd, LL_IOC_GROUP_UNLOCK, gid); + if (rc < 0) { + rc = -errno; + llapi_error(LLAPI_MSG_ERROR, rc, "cannot put group lock"); + } + return rc; +} -- 1.8.3.1