From bc37f89a81ea0a2fae8668e21247552e8894bfd8 Mon Sep 17 00:00:00 2001 From: Vitaly Fertman Date: Fri, 10 Jun 2022 01:00:50 +0300 Subject: [PATCH] LU-16046 revert: "LU-9964 llite: prevent mulitple group locks" This reverts commit aba68250a67a10104c534bd726f67b31a7f35692 since it makes group unlock synchronous what leads to poor performance on shared file IO under group lock. Signed-off-by: Vitaly Fertman Change-Id: I4548986297c22e402acd051dbdf97fe58198d100 Reviewed-on: https://review.whamcloud.com/c/fs/lustre-release/+/48037 Reviewed-by: Alexander Reviewed-by: Zhenyu Xu Reviewed-by: Oleg Drokin Tested-by: jenkins Tested-by: Maloo --- lustre/ldlm/ldlm_request.c | 3 +- lustre/llite/file.c | 73 +++++++++++++++++-------------------------- lustre/llite/llite_internal.h | 4 --- lustre/llite/llite_lib.c | 3 -- lustre/osc/osc_lock.c | 2 -- lustre/tests/sanity.sh | 1 + 6 files changed, 30 insertions(+), 56 deletions(-) diff --git a/lustre/ldlm/ldlm_request.c b/lustre/ldlm/ldlm_request.c index 62595ac..8d609d1 100644 --- a/lustre/ldlm/ldlm_request.c +++ b/lustre/ldlm/ldlm_request.c @@ -1032,8 +1032,7 @@ int ldlm_cli_enqueue(struct obd_export *exp, struct ptlrpc_request **reqp, lock->l_conn_export = exp; lock->l_export = NULL; lock->l_blocking_ast = einfo->ei_cb_bl; - lock->l_flags |= (*flags & (LDLM_FL_NO_LRU | LDLM_FL_EXCL | - LDLM_FL_ATOMIC_CB)); + lock->l_flags |= (*flags & (LDLM_FL_NO_LRU | LDLM_FL_EXCL)); lock->l_activity = ktime_get_real_seconds(); /* lock not sent to server yet */ diff --git a/lustre/llite/file.c b/lustre/llite/file.c index ed628bd..5842e9f 100644 --- a/lustre/llite/file.c +++ b/lustre/llite/file.c @@ -2562,7 +2562,6 @@ out: RETURN(rc); } - static int ll_get_grouplock(struct inode *inode, struct file *file, unsigned long arg) { @@ -2578,28 +2577,18 @@ ll_get_grouplock(struct inode *inode, struct file *file, unsigned long arg) RETURN(-EINVAL); } - if (ll_file_nolock(file)) - RETURN(-EOPNOTSUPP); -retry: - if (file->f_flags & O_NONBLOCK) { - if (!mutex_trylock(&lli->lli_group_mutex)) - RETURN(-EAGAIN); - } else - mutex_lock(&lli->lli_group_mutex); + if (ll_file_nolock(file)) + RETURN(-EOPNOTSUPP); + read_lock(&lli->lli_lock); if (fd->fd_flags & LL_FILE_GROUP_LOCKED) { CWARN("group lock already existed with gid %lu\n", fd->fd_grouplock.lg_gid); - GOTO(out, rc = -EINVAL); - } - if (arg != lli->lli_group_gid && lli->lli_group_users != 0) { - if (file->f_flags & O_NONBLOCK) - GOTO(out, rc = -EAGAIN); - mutex_unlock(&lli->lli_group_mutex); - wait_var_event(&lli->lli_group_users, !lli->lli_group_users); - GOTO(retry, rc = 0); + read_unlock(&lli->lli_lock); + RETURN(-EINVAL); } LASSERT(fd->fd_grouplock.lg_lock == NULL); + read_unlock(&lli->lli_lock); /** * XXX: group lock needs to protect all OST objects while PFL @@ -2619,7 +2608,7 @@ retry: env = cl_env_get(&refcheck); if (IS_ERR(env)) - GOTO(out, rc = PTR_ERR(env)); + RETURN(PTR_ERR(env)); rc = cl_object_layout_get(env, obj, &cl); if (rc >= 0 && cl.cl_is_composite) @@ -2628,26 +2617,28 @@ retry: cl_env_put(env, &refcheck); if (rc < 0) - GOTO(out, rc); + RETURN(rc); } rc = cl_get_grouplock(ll_i2info(inode)->lli_clob, arg, (file->f_flags & O_NONBLOCK), &grouplock); - if (rc) - GOTO(out, rc); + RETURN(rc); + + write_lock(&lli->lli_lock); + if (fd->fd_flags & LL_FILE_GROUP_LOCKED) { + write_unlock(&lli->lli_lock); + CERROR("another thread just won the race\n"); + cl_put_grouplock(&grouplock); + RETURN(-EINVAL); + } fd->fd_flags |= LL_FILE_GROUP_LOCKED; fd->fd_grouplock = grouplock; - if (lli->lli_group_users == 0) - lli->lli_group_gid = grouplock.lg_gid; - lli->lli_group_users++; + write_unlock(&lli->lli_lock); CDEBUG(D_INFO, "group lock %lu obtained\n", arg); -out: - mutex_unlock(&lli->lli_group_mutex); - - RETURN(rc); + RETURN(0); } static int ll_put_grouplock(struct inode *inode, struct file *file, @@ -2656,40 +2647,32 @@ static int ll_put_grouplock(struct inode *inode, struct file *file, struct ll_inode_info *lli = ll_i2info(inode); struct ll_file_data *fd = file->private_data; struct ll_grouplock grouplock; - int rc; ENTRY; - mutex_lock(&lli->lli_group_mutex); + write_lock(&lli->lli_lock); if (!(fd->fd_flags & LL_FILE_GROUP_LOCKED)) { - CWARN("no group lock held\n"); - GOTO(out, rc = -EINVAL); - } + write_unlock(&lli->lli_lock); + CWARN("no group lock held\n"); + RETURN(-EINVAL); + } LASSERT(fd->fd_grouplock.lg_lock != NULL); if (fd->fd_grouplock.lg_gid != arg) { CWARN("group lock %lu doesn't match current id %lu\n", arg, fd->fd_grouplock.lg_gid); - GOTO(out, rc = -EINVAL); + write_unlock(&lli->lli_lock); + RETURN(-EINVAL); } grouplock = fd->fd_grouplock; memset(&fd->fd_grouplock, 0, sizeof(fd->fd_grouplock)); fd->fd_flags &= ~LL_FILE_GROUP_LOCKED; + write_unlock(&lli->lli_lock); cl_put_grouplock(&grouplock); - - lli->lli_group_users--; - if (lli->lli_group_users == 0) { - lli->lli_group_gid = 0; - wake_up_var(&lli->lli_group_users); - } CDEBUG(D_INFO, "group lock %lu released\n", arg); - GOTO(out, rc = 0); -out: - mutex_unlock(&lli->lli_group_mutex); - - RETURN(rc); + RETURN(0); } /** diff --git a/lustre/llite/llite_internal.h b/lustre/llite/llite_internal.h index 5608e8d..94d7025 100644 --- a/lustre/llite/llite_internal.h +++ b/lustre/llite/llite_internal.h @@ -241,10 +241,6 @@ struct ll_inode_info { enum pcc_dataset_flags lli_pcc_dsflags; struct pcc_inode *lli_pcc_inode; - struct mutex lli_group_mutex; - __u64 lli_group_users; - unsigned long lli_group_gid; - __u64 lli_attr_valid; __u64 lli_lazysize; __u64 lli_lazyblocks; diff --git a/lustre/llite/llite_lib.c b/lustre/llite/llite_lib.c index 0338b86..9db4ef5 100644 --- a/lustre/llite/llite_lib.c +++ b/lustre/llite/llite_lib.c @@ -1236,9 +1236,6 @@ void ll_lli_init(struct ll_inode_info *lli) lli->lli_pcc_inode = NULL; lli->lli_pcc_dsflags = PCC_DATASET_INVALID; lli->lli_pcc_generation = 0; - mutex_init(&lli->lli_group_mutex); - lli->lli_group_users = 0; - lli->lli_group_gid = 0; } mutex_init(&lli->lli_layout_mutex); memset(lli->lli_jobid, 0, sizeof(lli->lli_jobid)); diff --git a/lustre/osc/osc_lock.c b/lustre/osc/osc_lock.c index 1ebd516..6453b31 100644 --- a/lustre/osc/osc_lock.c +++ b/lustre/osc/osc_lock.c @@ -1233,8 +1233,6 @@ int osc_lock_init(const struct lu_env *env, oscl->ols_flags = osc_enq2ldlm_flags(enqflags); oscl->ols_speculative = !!(enqflags & CEF_SPECULATIVE); - if (lock->cll_descr.cld_mode == CLM_GROUP) - oscl->ols_flags |= LDLM_FL_ATOMIC_CB; if (oscl->ols_flags & LDLM_FL_HAS_INTENT) { oscl->ols_flags |= LDLM_FL_BLOCK_GRANTED; diff --git a/lustre/tests/sanity.sh b/lustre/tests/sanity.sh index f3d1618..ae0ded9 100755 --- a/lustre/tests/sanity.sh +++ b/lustre/tests/sanity.sh @@ -43,6 +43,7 @@ always_except LU-6493 42b always_except LU-14541 277 always_except LU-9054 312 always_except LU-8411 407 +always_except LU-16046 244b if $SHARED_KEY; then always_except LU-14181 64e 64f -- 1.8.3.1