b=20529

r=jay,wangdi

- fix memory map issue of group lock
- revise liblustre group lock implementation

diff --git a/lustre/lclient/lcommon_cl.c b/lustre/lclient/lcommon_cl.c
index 30e2488..dc2be2f 100644 (file)
--- a/lustre/lclient/lcommon_cl.c
+++ b/lustre/lclient/lcommon_cl.c
@@ -728,7 +728,13 @@ int ccc_io_one_lock_index(const struct lu_env *env, struct cl_io *io,
         CDEBUG(D_VFSTRACE, "lock: %i [%lu, %lu]\n", mode, start, end);
 
         memset(&cio->cui_link, 0, sizeof cio->cui_link);
-        descr->cld_mode  = mode;
+
+        if (cio->cui_fd && (cio->cui_fd->fd_flags & LL_FILE_GROUP_LOCKED)) {
+                descr->cld_mode = CLM_GROUP;
+                descr->cld_gid  = cio->cui_fd->fd_grouplock.cg_gid;
+        } else {
+                descr->cld_mode  = mode;
+        }
         descr->cld_obj   = obj;
         descr->cld_start = start;
         descr->cld_end   = end;

diff --git a/lustre/liblustre/file.c b/lustre/liblustre/file.c
index b1e65f0..f30cdaf 100644 (file)
--- a/lustre/liblustre/file.c
+++ b/lustre/liblustre/file.c
@@ -382,12 +382,8 @@ int llu_md_close(struct obd_export *md_exp, struct inode *inode)
         ENTRY;
 
         /* clear group lock, if present */
-        if (fd->fd_flags & LL_FILE_GROUP_LOCKED) {
-                struct lov_stripe_md *lsm = llu_i2info(inode)->lli_smd;
-                fd->fd_flags &= ~(LL_FILE_GROUP_LOCKED|LL_FILE_IGNORE_LOCK);
-                rc = llu_extent_unlock(fd, inode, lsm, LCK_GROUP,
-                                       &fd->fd_cwlockh);
-        }
+        if (fd->fd_flags & LL_FILE_GROUP_LOCKED)
+                llu_put_grouplock(inode, fd->fd_grouplock.cg_gid);
 
         op_data.op_attr.ia_valid = ATTR_MODE | ATTR_ATIME_SET |
                                 ATTR_MTIME_SET | ATTR_CTIME_SET;

diff --git a/lustre/liblustre/llite_lib.h b/lustre/liblustre/llite_lib.h
index 3f67427..afbde6c 100644 (file)
--- a/lustre/liblustre/llite_lib.h
+++ b/lustre/liblustre/llite_lib.h
@@ -58,8 +58,7 @@
 struct ll_file_data {
         struct obd_client_handle fd_mds_och;
         __u32 fd_flags;
-        struct lustre_handle fd_cwlockh;
-        unsigned long fd_gid;
+        struct ccc_grouplock fd_grouplock;
 };
 
 struct llu_sb_info {
@@ -235,6 +234,7 @@ int llu_inode_getattr(struct inode *inode, struct obdo *obdo);
 int llu_md_setattr(struct inode *inode, struct md_op_data *op_data,
                    struct md_open_data **mod);
 int llu_setattr_raw(struct inode *inode, struct iattr *attr);
+int llu_put_grouplock(struct inode *inode, unsigned long arg);
 
 extern struct fssw_ops llu_fssw_ops;
 

diff --git a/lustre/liblustre/super.c b/lustre/liblustre/super.c
index de5f687..0b2088d 100644 (file)
--- a/lustre/liblustre/super.c
+++ b/lustre/liblustre/super.c
@@ -1599,56 +1599,51 @@ static int llu_get_grouplock(struct inode *inode, unsigned long arg)
 {
         struct llu_inode_info *lli = llu_i2info(inode);
         struct ll_file_data *fd = lli->lli_file_data;
-        ldlm_policy_data_t policy = { .l_extent = { .start = 0,
-                                                    .end = OBD_OBJECT_EOF}};
-        struct lustre_handle lockh = { 0 };
-        struct lov_stripe_md *lsm = lli->lli_smd;
-        ldlm_error_t err;
-        int flags = 0;
+        int rc;
+        struct ccc_grouplock grouplock;
         ENTRY;
 
+        if (fd->fd_flags & LL_FILE_IGNORE_LOCK) {
+                RETURN(-ENOTSUPP);
+        }
         if (fd->fd_flags & LL_FILE_GROUP_LOCKED) {
                 RETURN(-EINVAL);
         }
+        LASSERT(fd->fd_grouplock.cg_lock == NULL);
 
-        policy.l_extent.gid = arg;
-        if (lli->lli_open_flags & O_NONBLOCK)
-                flags = LDLM_FL_BLOCK_NOWAIT;
+        rc = cl_get_grouplock(cl_i2info(inode)->lli_clob,
+                              arg, (lli->lli_open_flags & O_NONBLOCK),
+                              &grouplock);
 
-        err = llu_extent_lock(fd, inode, lsm, LCK_GROUP, &policy, &lockh,
-                              flags);
-        if (err)
-                RETURN(err);
+        if (rc)
+                RETURN(rc);
 
-        fd->fd_flags |= LL_FILE_GROUP_LOCKED|LL_FILE_IGNORE_LOCK;
-        fd->fd_gid = arg;
-        memcpy(&fd->fd_cwlockh, &lockh, sizeof(lockh));
+        fd->fd_flags |= LL_FILE_GROUP_LOCKED;
+        fd->fd_grouplock = grouplock;
 
         RETURN(0);
 }
 
-static int llu_put_grouplock(struct inode *inode, unsigned long arg)
+int llu_put_grouplock(struct inode *inode, unsigned long arg)
 {
         struct llu_inode_info *lli = llu_i2info(inode);
         struct ll_file_data *fd = lli->lli_file_data;
-        struct lov_stripe_md *lsm = lli->lli_smd;
-        ldlm_error_t err;
+        struct ccc_grouplock grouplock;
         ENTRY;
 
         if (!(fd->fd_flags & LL_FILE_GROUP_LOCKED))
                 RETURN(-EINVAL);
 
-        if (fd->fd_gid != arg)
-                RETURN(-EINVAL);
+        LASSERT(fd->fd_grouplock.cg_lock != NULL);
 
-        fd->fd_flags &= ~(LL_FILE_GROUP_LOCKED|LL_FILE_IGNORE_LOCK);
+        if (fd->fd_grouplock.cg_gid != arg)
+                RETURN(-EINVAL);
 
-        err = llu_extent_unlock(fd, inode, lsm, LCK_GROUP, &fd->fd_cwlockh);
-        if (err)
-                RETURN(err);
+        grouplock = fd->fd_grouplock;
+        memset(&fd->fd_grouplock, 0, sizeof(fd->fd_grouplock));
+        fd->fd_flags &= ~LL_FILE_GROUP_LOCKED;
 
-        fd->fd_gid = 0;
-        memset(&fd->fd_cwlockh, 0, sizeof(fd->fd_cwlockh));
+        cl_put_grouplock(&grouplock);
 
         RETURN(0);
 }

diff --git a/lustre/llite/file.c b/lustre/llite/file.c
index 3570e55..708ad77 100644 (file)
--- a/lustre/llite/file.c
+++ b/lustre/llite/file.c
@@ -812,7 +812,10 @@ static ssize_t ll_file_io_generic(const struct lu_env *env,
         if (cl_io_rw_init(env, io, iot, *ppos, count) == 0) {
                 struct vvp_io *vio = vvp_env_io(env);
                 struct ccc_io *cio = ccc_env_io(env);
+                struct ll_inode_info *lli = ll_i2info(file->f_dentry->d_inode);
+                int write_sem_locked = 0;
 
+                cio->cui_fd  = LUSTRE_FPRIVATE(file);
                 vio->cui_io_subtype = args->via_io_subtype;
 
                 switch (vio->cui_io_subtype) {
@@ -822,6 +825,11 @@ static ssize_t ll_file_io_generic(const struct lu_env *env,
 #ifndef HAVE_FILE_WRITEV
                         cio->cui_iocb = args->u.normal.via_iocb;
 #endif
+                        if ((iot == CIT_WRITE) &&
+                            !(cio->cui_fd->fd_flags & LL_FILE_GROUP_LOCKED)) {
+                                down(&lli->lli_write_sem);
+                                write_sem_locked = 1;
+                        }
                         break;
                 case IO_SENDFILE:
                         vio->u.sendfile.cui_actor = args->u.sendfile.via_actor;
@@ -835,8 +843,9 @@ static ssize_t ll_file_io_generic(const struct lu_env *env,
                         CERROR("Unknow IO type - %u\n", vio->cui_io_subtype);
                         LBUG();
                 }
-                cio->cui_fd  = LUSTRE_FPRIVATE(file);
                 result = cl_io_loop(env, io);
+                if (write_sem_locked)
+                        up(&lli->lli_write_sem);
         } else {
                 /* cl_io_rw_init() handled IO */
                 result = io->ci_result;
@@ -1410,7 +1419,7 @@ int ll_get_grouplock(struct inode *inode, struct file *file, unsigned long arg)
 
         spin_lock(&lli->lli_lock);
         if (fd->fd_flags & LL_FILE_GROUP_LOCKED) {
-                CERROR("group lock already existed with gid %lu\n",
+                CWARN("group lock already existed with gid %lu\n",
                        fd->fd_grouplock.cg_gid);
                 spin_unlock(&lli->lli_lock);
                 RETURN(-EINVAL);
@@ -1449,22 +1458,20 @@ int ll_put_grouplock(struct inode *inode, struct file *file, unsigned long arg)
         spin_lock(&lli->lli_lock);
         if (!(fd->fd_flags & LL_FILE_GROUP_LOCKED)) {
                 spin_unlock(&lli->lli_lock);
-                CERROR("no group lock held\n");
+                CWARN("no group lock held\n");
                 RETURN(-EINVAL);
         }
         LASSERT(fd->fd_grouplock.cg_lock != NULL);
 
         if (fd->fd_grouplock.cg_gid != arg) {
-                CERROR("group lock %lu doesn't match current id %lu\n",
+                CWARN("group lock %lu doesn't match current id %lu\n",
                        arg, fd->fd_grouplock.cg_gid);
                 spin_unlock(&lli->lli_lock);
                 RETURN(-EINVAL);
         }
 
         grouplock = fd->fd_grouplock;
-        fd->fd_grouplock.cg_env = NULL;
-        fd->fd_grouplock.cg_lock = NULL;
-        fd->fd_grouplock.cg_gid = 0;
+        memset(&fd->fd_grouplock, 0, sizeof(fd->fd_grouplock));
         fd->fd_flags &= ~LL_FILE_GROUP_LOCKED;
         spin_unlock(&lli->lli_lock);
 

diff --git a/lustre/llite/vvp_io.c b/lustre/llite/vvp_io.c
index 68f9e8d..a602672 100644 (file)
--- a/lustre/llite/vvp_io.c
+++ b/lustre/llite/vvp_io.c
@@ -88,9 +88,7 @@ static void vvp_io_fini(const struct lu_env *env, const struct cl_io_slice *ios)
         struct cl_object *obj = io->ci_obj;
 
         CLOBINVRNT(env, obj, ccc_object_invariant(obj));
-        if (io->ci_type == CIT_WRITE)
-                up(&ll_i2info(ccc_object_inode(obj))->lli_write_sem);
-        else {
+        if (io->ci_type == CIT_READ) {
                 struct vvp_io     *vio  = cl2vvp_io(env, ios);
                 struct ccc_io     *cio  = cl2ccc_io(env, ios);
 
@@ -996,8 +994,6 @@ int vvp_io_init(const struct lu_env *env, struct cl_object *obj,
                 count = io->u.ci_rw.crw_count;
                 op    = io->ci_type == CIT_READ ?
                         LPROC_LL_READ_BYTES : LPROC_LL_WRITE_BYTES;
-                if (io->ci_type == CIT_WRITE)
-                        down(&ll_i2info(inode)->lli_write_sem);
                 /* "If nbyte is 0, read() will return 0 and have no other
                  *  results."  -- Single Unix Spec */
                 if (count == 0)

diff --git a/lustre/obdclass/cl_lock.c b/lustre/obdclass/cl_lock.c
index c35d7b7..8db4297 100644 (file)
--- a/lustre/obdclass/cl_lock.c
+++ b/lustre/obdclass/cl_lock.c
@@ -1789,9 +1789,12 @@ struct cl_lock *cl_lock_at_page(const struct lu_env *env, struct cl_object *obj,
         need->cld_start = need->cld_end = page->cp_index;
 
         spin_lock(&head->coh_lock_guard);
+        /* It is fine to match any group lock since there could be only one
+         * with a uniq gid and it conflicts with all other lock modes too */
         list_for_each_entry(scan, &head->coh_locks, cll_linkage) {
                 if (scan != except &&
-                    cl_lock_ext_match(&scan->cll_descr, need) &&
+                    (scan->cll_descr.cld_mode == CLM_GROUP ||
+                    cl_lock_ext_match(&scan->cll_descr, need)) &&
                     scan->cll_state >= CLS_HELD &&
                     scan->cll_state < CLS_FREEING &&
                     /*

diff --git a/lustre/tests/mpi/lp_utils.c b/lustre/tests/mpi/lp_utils.c
index 7f47b3d..c7a489f 100644 (file)
--- a/lustre/tests/mpi/lp_utils.c
+++ b/lustre/tests/mpi/lp_utils.c
@@ -88,8 +88,9 @@ inline void end(char *str) {
         MPI_Barrier(MPI_COMM_WORLD);
         if (verbose > 0 && rank == 0) {
                 gettimeofday(&t2, NULL);
-                elapsed = (t2.tv_sec + ((float)t2.tv_usec/1000000))
-                          - (t1.tv_sec + ((float)t1.tv_usec/1000000));
+
+                elapsed = t2.tv_sec - t1.tv_sec +
+                          (float)(t2.tv_usec-t1.tv_usec)/1000000;
                 if (elapsed >= 60) {
                         printf("%s:\tFinished %-15s(%.2f min)\n",
                                timestamp(), str, elapsed / 60);

diff --git a/lustre/tests/mpi/parallel_grouplock.c b/lustre/tests/mpi/parallel_grouplock.c
index e572e4e..0843deb 100644 (file)
--- a/lustre/tests/mpi/parallel_grouplock.c
+++ b/lustre/tests/mpi/parallel_grouplock.c
@@ -1013,7 +1013,8 @@ int main(int argc, char *argv[])
 
         if (size < MIN_GLHOST) {
                 fprintf(stderr, "Error: "
-                        "should be at least four tasks to run the test!\n");
+                        "%d tasks run, but should be at least %d tasks to run "
+                        "the test!\n", size, MIN_GLHOST);
                 MPI_Abort(MPI_COMM_WORLD, 2);
         }