From: Jinshan Xiong Date: Mon, 16 May 2016 20:27:17 +0000 (-0700) Subject: LU-4257 obdclass: Get rid of cl_env hash table X-Git-Tag: 2.8.54~4 X-Git-Url: https://git.whamcloud.com/?p=fs%2Flustre-release.git;a=commitdiff_plain;h=45332712783a4756bf5930d6bd5f697bbc27acdb LU-4257 obdclass: Get rid of cl_env hash table cl_env hash table is under heavy contention when there are lots of processes doing IO at the same time; reduce lock contention by replacing cl_env cache with percpu array; remove cl_env_nested_get() and cl_env_nested_put(); remove cl_env_reenter() and cl_env_reexit(); Signed-off-by: Jinshan Xiong Change-Id: I04137c93e2d48f21298fe954a1c2afe1006f44d5 Reviewed-on: http://review.whamcloud.com/20254 Tested-by: Jenkins Reviewed-by: Andreas Dilger Tested-by: Maloo Reviewed-by: Bobi Jam Reviewed-by: Oleg Drokin --- diff --git a/lustre/include/cl_object.h b/lustre/include/cl_object.h index 9102f2a..f50fee7 100644 --- a/lustre/include/cl_object.h +++ b/lustre/include/cl_object.h @@ -2444,35 +2444,12 @@ void cl_sync_io_end(const struct lu_env *env, struct cl_sync_io *anchor); * - allocation and destruction of environment is amortized by caching no * longer used environments instead of destroying them; * - * - there is a notion of "current" environment, attached to the kernel - * data structure representing current thread Top-level lustre code - * allocates an environment and makes it current, then calls into - * non-lustre code, that in turn calls lustre back. Low-level lustre - * code thus called can fetch environment created by the top-level code - * and reuse it, avoiding additional environment allocation. - * Right now, three interfaces can attach the cl_env to running thread: - * - cl_env_get - * - cl_env_implant - * - cl_env_reexit(cl_env_reenter had to be called priorly) - * * \see lu_env, lu_context, lu_context_key * @{ */ -struct cl_env_nest { - __u16 cen_refcheck; - void *cen_cookie; -}; - -struct lu_env *cl_env_peek(__u16 *refcheck); struct lu_env *cl_env_get(__u16 *refcheck); struct lu_env *cl_env_alloc(__u16 *refcheck, __u32 tags); -struct lu_env *cl_env_nested_get(struct cl_env_nest *nest); void cl_env_put(struct lu_env *env, __u16 *refcheck); -void cl_env_nested_put(struct cl_env_nest *nest, struct lu_env *env); -void *cl_env_reenter(void); -void cl_env_reexit(void *cookie); -void cl_env_implant(struct lu_env *env, __u16 *refcheck); -void cl_env_unplant(struct lu_env *env, __u16 *refcheck); unsigned cl_env_cache_purge(unsigned nr); struct lu_env *cl_env_percpu_get(void); void cl_env_percpu_put(struct lu_env *env); diff --git a/lustre/ldlm/ldlm_pool.c b/lustre/ldlm/ldlm_pool.c index 5ed717c..98a27ec 100644 --- a/lustre/ldlm/ldlm_pool.c +++ b/lustre/ldlm/ldlm_pool.c @@ -1050,7 +1050,6 @@ static unsigned long ldlm_pools_count(enum ldlm_side client, gfp_t gfp_mask) int nr_ns; struct ldlm_namespace *ns; struct ldlm_namespace *ns_old = NULL; /* loop detection */ - void *cookie; if (client == LDLM_NAMESPACE_CLIENT && !(gfp_mask & __GFP_FS)) return 0; @@ -1058,8 +1057,6 @@ static unsigned long ldlm_pools_count(enum ldlm_side client, gfp_t gfp_mask) CDEBUG(D_DLMTRACE, "Request to count %s locks from all pools\n", client == LDLM_NAMESPACE_CLIENT ? "client" : "server"); - cookie = cl_env_reenter(); - /* * Find out how many resources we may release. */ @@ -1068,7 +1065,6 @@ static unsigned long ldlm_pools_count(enum ldlm_side client, gfp_t gfp_mask) mutex_lock(ldlm_namespace_lock(client)); if (list_empty(ldlm_namespace_list(client))) { mutex_unlock(ldlm_namespace_lock(client)); - cl_env_reexit(cookie); return 0; } ns = ldlm_namespace_first_locked(client); @@ -1094,7 +1090,6 @@ static unsigned long ldlm_pools_count(enum ldlm_side client, gfp_t gfp_mask) ldlm_namespace_put(ns); } - cl_env_reexit(cookie); return total; } @@ -1104,13 +1099,10 @@ static unsigned long ldlm_pools_scan(enum ldlm_side client, int nr, unsigned long freed = 0; int tmp, nr_ns; struct ldlm_namespace *ns; - void *cookie; if (client == LDLM_NAMESPACE_CLIENT && !(gfp_mask & __GFP_FS)) return -1; - cookie = cl_env_reenter(); - /* * Shrink at least ldlm_namespace_nr_read(client) namespaces. */ @@ -1140,7 +1132,6 @@ static unsigned long ldlm_pools_scan(enum ldlm_side client, int nr, freed += ldlm_pool_shrink(&ns->ns_pool, cancel, gfp_mask); ldlm_namespace_put(ns); } - cl_env_reexit(cookie); /* * we only decrease the SLV in server pools shrinker, return * SHRINK_STOP to kernel to avoid needless loop. LU-1128 diff --git a/lustre/llite/file.c b/lustre/llite/file.c index 4a58632..258efb8 100644 --- a/lustre/llite/file.c +++ b/lustre/llite/file.c @@ -1246,72 +1246,45 @@ static ssize_t ll_file_aio_read(struct kiocb *iocb, const struct iovec *iov, unsigned long nr_segs, loff_t pos) { struct iovec *local_iov; - struct iov_iter *to; + struct iov_iter to; size_t iov_count; ssize_t result; - struct lu_env *env = NULL; - __u16 refcheck; ENTRY; result = ll_file_get_iov_count(iov, &nr_segs, &iov_count); if (result) RETURN(result); - if (nr_segs == 1) { - - env = cl_env_get(&refcheck); - if (IS_ERR(env)) - RETURN(PTR_ERR(env)); - - local_iov = &ll_env_info(env)->lti_local_iov; - *local_iov = *iov; - - } else { - OBD_ALLOC(local_iov, sizeof(*iov) * nr_segs); - if (local_iov == NULL) - RETURN(-ENOMEM); + OBD_ALLOC(local_iov, sizeof(*iov) * nr_segs); + if (local_iov == NULL) + RETURN(-ENOMEM); - memcpy(local_iov, iov, sizeof(*iov) * nr_segs); - } + memcpy(local_iov, iov, sizeof(*iov) * nr_segs); - OBD_ALLOC_PTR(to); - if (to == NULL) { - result = -ENOMEM; - goto out; - } # ifdef HAVE_IOV_ITER_INIT_DIRECTION - iov_iter_init(to, READ, local_iov, nr_segs, iov_count); + iov_iter_init(&to, READ, local_iov, nr_segs, iov_count); # else /* !HAVE_IOV_ITER_INIT_DIRECTION */ - iov_iter_init(to, local_iov, nr_segs, iov_count, 0); + iov_iter_init(&to, local_iov, nr_segs, iov_count, 0); # endif /* HAVE_IOV_ITER_INIT_DIRECTION */ - result = ll_file_read_iter(iocb, to); - - OBD_FREE_PTR(to); -out: - if (nr_segs == 1) - cl_env_put(env, &refcheck); - else - OBD_FREE(local_iov, sizeof(*iov) * nr_segs); + result = ll_file_read_iter(iocb, &to); + OBD_FREE(local_iov, sizeof(*iov) * nr_segs); RETURN(result); } static ssize_t ll_file_read(struct file *file, char __user *buf, size_t count, loff_t *ppos) { - struct lu_env *env; struct iovec iov = { .iov_base = buf, .iov_len = count }; struct kiocb *kiocb; ssize_t result; - __u16 refcheck; ENTRY; - env = cl_env_get(&refcheck); - if (IS_ERR(env)) - RETURN(PTR_ERR(env)); + OBD_ALLOC_PTR(kiocb); + if (kiocb == NULL) + RETURN(-ENOMEM); - kiocb = &ll_env_info(env)->lti_kiocb; init_sync_kiocb(kiocb, file); kiocb->ki_pos = *ppos; #ifdef HAVE_KIOCB_KI_LEFT @@ -1323,7 +1296,7 @@ static ssize_t ll_file_read(struct file *file, char __user *buf, size_t count, result = ll_file_aio_read(kiocb, &iov, 1, kiocb->ki_pos); *ppos = kiocb->ki_pos; - cl_env_put(env, &refcheck); + OBD_FREE_PTR(kiocb); RETURN(result); } @@ -1335,51 +1308,30 @@ static ssize_t ll_file_aio_write(struct kiocb *iocb, const struct iovec *iov, unsigned long nr_segs, loff_t pos) { struct iovec *local_iov; - struct iov_iter *from; + struct iov_iter from; size_t iov_count; ssize_t result; - struct lu_env *env = NULL; - __u16 refcheck; ENTRY; result = ll_file_get_iov_count(iov, &nr_segs, &iov_count); if (result) RETURN(result); - if (nr_segs == 1) { - env = cl_env_get(&refcheck); - if (IS_ERR(env)) - RETURN(PTR_ERR(env)); - - local_iov = &ll_env_info(env)->lti_local_iov; - *local_iov = *iov; - } else { - OBD_ALLOC(local_iov, sizeof(*iov) * nr_segs); - if (local_iov == NULL) - RETURN(-ENOMEM); + OBD_ALLOC(local_iov, sizeof(*iov) * nr_segs); + if (local_iov == NULL) + RETURN(-ENOMEM); - memcpy(local_iov, iov, sizeof(*iov) * nr_segs); - } + memcpy(local_iov, iov, sizeof(*iov) * nr_segs); - OBD_ALLOC_PTR(from); - if (from == NULL) { - result = -ENOMEM; - goto out; - } # ifdef HAVE_IOV_ITER_INIT_DIRECTION - iov_iter_init(from, WRITE, local_iov, nr_segs, iov_count); + iov_iter_init(&from, WRITE, local_iov, nr_segs, iov_count); # else /* !HAVE_IOV_ITER_INIT_DIRECTION */ - iov_iter_init(from, local_iov, nr_segs, iov_count, 0); + iov_iter_init(&from, local_iov, nr_segs, iov_count, 0); # endif /* HAVE_IOV_ITER_INIT_DIRECTION */ - result = ll_file_write_iter(iocb, from); + result = ll_file_write_iter(iocb, &from); - OBD_FREE_PTR(from); -out: - if (nr_segs == 1) - cl_env_put(env, &refcheck); - else - OBD_FREE(local_iov, sizeof(*iov) * nr_segs); + OBD_FREE(local_iov, sizeof(*iov) * nr_segs); RETURN(result); } @@ -1914,11 +1866,11 @@ restart: */ int ll_hsm_release(struct inode *inode) { - struct cl_env_nest nest; struct lu_env *env; struct obd_client_handle *och = NULL; __u64 data_version = 0; int rc; + __u16 refcheck; ENTRY; CDEBUG(D_INODE, "%s: Releasing file "DFID".\n", @@ -1934,12 +1886,12 @@ int ll_hsm_release(struct inode *inode) if (rc != 0) GOTO(out, rc); - env = cl_env_nested_get(&nest); + env = cl_env_get(&refcheck); if (IS_ERR(env)) GOTO(out, rc = PTR_ERR(env)); ll_merge_attr(env, inode); - cl_env_nested_put(&nest, env); + cl_env_put(env, &refcheck); /* Release the file. * NB: lease lock handle is released in mdc_hsm_release_pack() because @@ -2224,14 +2176,14 @@ static int ll_file_futimes_3(struct file *file, const struct ll_futimes_3 *lfu) static int ll_ladvise(struct inode *inode, struct file *file, __u64 flags, struct lu_ladvise *ladvise) { - struct cl_env_nest nest; struct lu_env *env; struct cl_io *io; struct cl_ladvise_io *lio; int rc; + __u16 refcheck; ENTRY; - env = cl_env_nested_get(&nest); + env = cl_env_get(&refcheck); if (IS_ERR(env)) RETURN(PTR_ERR(env)); @@ -2252,7 +2204,7 @@ static int ll_ladvise(struct inode *inode, struct file *file, __u64 flags, rc = io->ci_result; cl_io_fini(env, io); - cl_env_nested_put(&nest, env); + cl_env_put(env, &refcheck); RETURN(rc); } @@ -2787,18 +2739,18 @@ static int ll_flush(struct file *file, fl_owner_t id) int cl_sync_file_range(struct inode *inode, loff_t start, loff_t end, enum cl_fsync_mode mode, int ignore_layout) { - struct cl_env_nest nest; struct lu_env *env; struct cl_io *io; struct cl_fsync_io *fio; int result; + __u16 refcheck; ENTRY; if (mode != CL_FSYNC_NONE && mode != CL_FSYNC_LOCAL && mode != CL_FSYNC_DISCARD && mode != CL_FSYNC_ALL) RETURN(-EINVAL); - env = cl_env_nested_get(&nest); + env = cl_env_get(&refcheck); if (IS_ERR(env)) RETURN(PTR_ERR(env)); @@ -2821,7 +2773,7 @@ int cl_sync_file_range(struct inode *inode, loff_t start, loff_t end, if (result == 0) result = fio->fi_nr_written; cl_io_fini(env, io); - cl_env_nested_put(&nest, env); + cl_env_put(env, &refcheck); RETURN(result); } @@ -3873,15 +3825,15 @@ int ll_layout_conf(struct inode *inode, const struct cl_object_conf *conf) { struct ll_inode_info *lli = ll_i2info(inode); struct cl_object *obj = lli->lli_clob; - struct cl_env_nest nest; struct lu_env *env; int rc; + __u16 refcheck; ENTRY; if (obj == NULL) RETURN(0); - env = cl_env_nested_get(&nest); + env = cl_env_get(&refcheck); if (IS_ERR(env)) RETURN(PTR_ERR(env)); @@ -3916,7 +3868,7 @@ int ll_layout_conf(struct inode *inode, const struct cl_object_conf *conf) } out: - cl_env_nested_put(&nest, env); + cl_env_put(env, &refcheck); RETURN(rc); } diff --git a/lustre/llite/lcommon_cl.c b/lustre/llite/lcommon_cl.c index 673bfbd..94fda5a 100644 --- a/lustre/llite/lcommon_cl.c +++ b/lustre/llite/lcommon_cl.c @@ -227,20 +227,16 @@ void cl_inode_fini(struct inode *inode) { struct lu_env *env; struct ll_inode_info *lli = ll_i2info(inode); - struct cl_object *clob = lli->lli_clob; + struct cl_object *clob = lli->lli_clob; __u16 refcheck; - int emergency; + int emergency; - if (clob != NULL) { - void *cookie; - - cookie = cl_env_reenter(); - env = cl_env_get(&refcheck); - emergency = IS_ERR(env); + if (clob != NULL) { + env = cl_env_get(&refcheck); + emergency = IS_ERR(env); if (emergency) { mutex_lock(&cl_inode_fini_guard); LASSERT(cl_inode_fini_env != NULL); - cl_env_implant(cl_inode_fini_env, &refcheck); env = cl_inode_fini_env; } @@ -253,15 +249,11 @@ void cl_inode_fini(struct inode *inode) lu_object_ref_del(&clob->co_lu, "inode", inode); cl_object_put_last(env, clob); lli->lli_clob = NULL; - if (emergency) { - cl_env_unplant(cl_inode_fini_env, &refcheck); + if (emergency) mutex_unlock(&cl_inode_fini_guard); - } else { + else cl_env_put(env, &refcheck); - } - - cl_env_reexit(cookie); - } + } } /** diff --git a/lustre/llite/lcommon_misc.c b/lustre/llite/lcommon_misc.c index 80ce42e..47178ef 100644 --- a/lustre/llite/lcommon_misc.c +++ b/lustre/llite/lcommon_misc.c @@ -168,13 +168,11 @@ int cl_get_grouplock(struct cl_object *obj, unsigned long gid, int nonblock, return rc; } - lg->lg_env = cl_env_get(&refcheck); + lg->lg_env = env; lg->lg_io = io; lg->lg_lock = lock; lg->lg_gid = gid; - LASSERT(lg->lg_env == env); - cl_env_unplant(env, &refcheck); return 0; } @@ -183,14 +181,10 @@ void cl_put_grouplock(struct ll_grouplock *lg) struct lu_env *env = lg->lg_env; struct cl_io *io = lg->lg_io; struct cl_lock *lock = lg->lg_lock; - __u16 refcheck; LASSERT(lg->lg_env != NULL); LASSERT(lg->lg_gid != 0); - cl_env_implant(env, &refcheck); - cl_env_put(env, &refcheck); - cl_lock_release(env, lock); cl_io_fini(env, io); cl_env_put(env, NULL); diff --git a/lustre/llite/llite_internal.h b/lustre/llite/llite_internal.h index 54a2186..6c823be 100644 --- a/lustre/llite/llite_internal.h +++ b/lustre/llite/llite_internal.h @@ -1009,7 +1009,6 @@ struct ll_cl_context { }; struct ll_thread_info { - struct iovec lti_local_iov; struct vvp_io_args lti_args; struct ra_io_arg lti_ria; struct kiocb lti_kiocb; diff --git a/lustre/llite/llite_mmap.c b/lustre/llite/llite_mmap.c index aea44c7..72acd3d 100644 --- a/lustre/llite/llite_mmap.c +++ b/lustre/llite/llite_mmap.c @@ -87,46 +87,27 @@ struct vm_area_struct *our_vma(struct mm_struct *mm, unsigned long addr, /** * API independent part for page fault initialization. - * \param vma - virtual memory area addressed to page fault * \param env - corespondent lu_env to processing - * \param nest - nested level + * \param vma - virtual memory area addressed to page fault * \param index - page index corespondent to fault. * \parm ra_flags - vma readahead flags. * - * \return allocated and initialized env for fault operation. - * \retval EINVAL if env can't allocated - * \return other error codes from cl_io_init. + * \return error codes from cl_io_init. */ static struct cl_io * -ll_fault_io_init(struct vm_area_struct *vma, struct lu_env **env_ret, - struct cl_env_nest *nest, pgoff_t index, - unsigned long *ra_flags) +ll_fault_io_init(struct lu_env *env, struct vm_area_struct *vma, + pgoff_t index, unsigned long *ra_flags) { struct file *file = vma->vm_file; struct inode *inode = file->f_path.dentry->d_inode; struct cl_io *io; struct cl_fault_io *fio; - struct lu_env *env; int rc; ENTRY; - *env_ret = NULL; if (ll_file_nolock(file)) RETURN(ERR_PTR(-EOPNOTSUPP)); - /* - * page fault can be called when lustre IO is - * already active for the current thread, e.g., when doing read/write - * against user level buffer mapped from Lustre buffer. To avoid - * stomping on existing context, optionally force an allocation of a new - * one. - */ - env = cl_env_nested_get(nest); - if (IS_ERR(env)) - RETURN(ERR_PTR(-EINVAL)); - - *env_ret = env; - restart: io = vvp_env_thread_io(env); io->ci_obj = ll_i2info(inode)->lli_clob; @@ -166,7 +147,6 @@ restart: if (io->ci_need_restart) goto restart; - cl_env_nested_put(nest, env); io = ERR_PTR(rc); } @@ -180,16 +160,19 @@ static int ll_page_mkwrite0(struct vm_area_struct *vma, struct page *vmpage, struct lu_env *env; struct cl_io *io; struct vvp_io *vio; - struct cl_env_nest nest; int result; + __u16 refcheck; sigset_t set; struct inode *inode; struct ll_inode_info *lli; ENTRY; LASSERT(vmpage != NULL); + env = cl_env_get(&refcheck); + if (IS_ERR(env)) + RETURN(PTR_ERR(env)); - io = ll_fault_io_init(vma, &env, &nest, vmpage->index, NULL); + io = ll_fault_io_init(env, vma, vmpage->index, NULL); if (IS_ERR(io)) GOTO(out, result = PTR_ERR(io)); @@ -247,8 +230,8 @@ static int ll_page_mkwrite0(struct vm_area_struct *vma, struct page *vmpage, out_io: cl_io_fini(env, io); - cl_env_nested_put(&nest, env); out: + cl_env_put(env, &refcheck); CDEBUG(D_MMAP, "%s mkwrite with %d\n", current->comm, result); LASSERT(ergo(result == 0, PageLocked(vmpage))); @@ -292,14 +275,18 @@ static int ll_fault0(struct vm_area_struct *vma, struct vm_fault *vmf) struct vvp_io *vio = NULL; struct page *vmpage; unsigned long ra_flags; - struct cl_env_nest nest; - int result; + int result = 0; int fault_ret = 0; + __u16 refcheck; ENTRY; - io = ll_fault_io_init(vma, &env, &nest, vmf->pgoff, &ra_flags); + env = cl_env_get(&refcheck); + if (IS_ERR(env)) + RETURN(PTR_ERR(env)); + + io = ll_fault_io_init(env, vma, vmf->pgoff, &ra_flags); if (IS_ERR(io)) - RETURN(to_fault_error(PTR_ERR(io))); + GOTO(out, result = PTR_ERR(io)); result = io->ci_result; if (result == 0) { @@ -329,14 +316,15 @@ static int ll_fault0(struct vm_area_struct *vma, struct vm_fault *vmf) } } cl_io_fini(env, io); - cl_env_nested_put(&nest, env); vma->vm_flags |= ra_flags; + +out: + cl_env_put(env, &refcheck); if (result != 0 && !(fault_ret & VM_FAULT_RETRY)) fault_ret |= to_fault_error(result); - CDEBUG(D_MMAP, "%s fault %d/%d\n", - current->comm, fault_ret, result); + CDEBUG(D_MMAP, "%s fault %d/%d\n", current->comm, fault_ret, result); RETURN(fault_ret); } diff --git a/lustre/llite/rw.c b/lustre/llite/rw.c index 5297b42..1626d89 100644 --- a/lustre/llite/rw.c +++ b/lustre/llite/rw.c @@ -892,10 +892,10 @@ int ll_writepage(struct page *vmpage, struct writeback_control *wbc) struct cl_io *io; struct cl_page *page; struct cl_object *clob; - struct cl_env_nest nest; bool redirtied = false; bool unlocked = false; int result; + __u16 refcheck; ENTRY; LASSERT(PageLocked(vmpage)); @@ -903,7 +903,7 @@ int ll_writepage(struct page *vmpage, struct writeback_control *wbc) LASSERT(ll_i2dtexp(inode) != NULL); - env = cl_env_nested_get(&nest); + env = cl_env_get(&refcheck); if (IS_ERR(env)) GOTO(out, result = PTR_ERR(env)); @@ -964,7 +964,7 @@ int ll_writepage(struct page *vmpage, struct writeback_control *wbc) } } - cl_env_nested_put(&nest, env); + cl_env_put(env, &refcheck); GOTO(out, result); out: diff --git a/lustre/llite/rw26.c b/lustre/llite/rw26.c index b362532..d9482b2 100644 --- a/lustre/llite/rw26.c +++ b/lustre/llite/rw26.c @@ -128,7 +128,6 @@ static void ll_invalidatepage(struct page *vmpage, static int ll_releasepage(struct page *vmpage, RELEASEPAGE_ARG_TYPE gfp_mask) { struct lu_env *env; - void *cookie; struct cl_object *obj; struct cl_page *page; struct address_space *mapping; @@ -154,7 +153,6 @@ static int ll_releasepage(struct page *vmpage, RELEASEPAGE_ARG_TYPE gfp_mask) if (page == NULL) return 1; - cookie = cl_env_reenter(); env = cl_env_percpu_get(); LASSERT(!IS_ERR(env)); @@ -180,7 +178,6 @@ static int ll_releasepage(struct page *vmpage, RELEASEPAGE_ARG_TYPE gfp_mask) cl_page_put(env, page); cl_env_percpu_put(env); - cl_env_reexit(cookie); return result; } @@ -355,14 +352,14 @@ ll_direct_IO( struct kiocb *iocb, struct iov_iter *iter, loff_t file_offset) { - struct lu_env *env; + struct ll_cl_context *lcc; + const struct lu_env *env; struct cl_io *io; struct file *file = iocb->ki_filp; struct inode *inode = file->f_mapping->host; ssize_t count = iov_iter_count(iter); ssize_t tot_bytes = 0, result = 0; size_t size = MAX_DIO_SIZE; - __u16 refcheck; /* FIXME: io smaller than PAGE_SIZE is broken on ia64 ??? */ if ((file_offset & ~PAGE_MASK) || (count & ~PAGE_MASK)) @@ -378,9 +375,13 @@ ll_direct_IO( if (iov_iter_alignment(iter) & ~PAGE_MASK) return -EINVAL; - env = cl_env_get(&refcheck); + lcc = ll_cl_find(file); + if (lcc == NULL) + RETURN(-EIO); + + env = lcc->lcc_env; LASSERT(!IS_ERR(env)); - io = vvp_env_io(env)->vui_cl.cis_io; + io = lcc->lcc_io; LASSERT(io != NULL); /* 0. Need locking between buffered and direct access. and race with @@ -448,7 +449,6 @@ out: vio->u.write.vui_written += tot_bytes; } - cl_env_put(env, &refcheck); return tot_bytes ? : result; } #else /* !HAVE_DIRECTIO_ITER && !HAVE_IOV_ITER_RW */ @@ -487,7 +487,8 @@ static ssize_t ll_direct_IO(int rw, struct kiocb *iocb, const struct iovec *iov, loff_t file_offset, unsigned long nr_segs) { - struct lu_env *env; + struct ll_cl_context *lcc; + const struct lu_env *env; struct cl_io *io; struct file *file = iocb->ki_filp; struct inode *inode = file->f_mapping->host; @@ -495,7 +496,6 @@ ll_direct_IO(int rw, struct kiocb *iocb, const struct iovec *iov, ssize_t tot_bytes = 0, result = 0; unsigned long seg = 0; size_t size = MAX_DIO_SIZE; - __u16 refcheck; ENTRY; /* FIXME: io smaller than PAGE_SIZE is broken on ia64 ??? */ @@ -515,10 +515,14 @@ ll_direct_IO(int rw, struct kiocb *iocb, const struct iovec *iov, RETURN(-EINVAL); } - env = cl_env_get(&refcheck); - LASSERT(!IS_ERR(env)); - io = vvp_env_io(env)->vui_cl.cis_io; - LASSERT(io != NULL); + lcc = ll_cl_find(file); + if (lcc == NULL) + RETURN(-EIO); + + env = lcc->lcc_env; + LASSERT(!IS_ERR(env)); + io = lcc->lcc_io; + LASSERT(io != NULL); for (seg = 0; seg < nr_segs; seg++) { size_t iov_left = iov[seg].iov_len; @@ -585,7 +589,6 @@ out: vio->u.write.vui_written += tot_bytes; } - cl_env_put(env, &refcheck); RETURN(tot_bytes ? tot_bytes : result); } #endif /* HAVE_DIRECTIO_ITER || HAVE_IOV_ITER_RW */ diff --git a/lustre/lov/lov_io.c b/lustre/lov/lov_io.c index ebf45ba..f945dc45 100644 --- a/lustre/lov/lov_io.c +++ b/lustre/lov/lov_io.c @@ -181,12 +181,7 @@ static int lov_io_sub_init(const struct lu_env *env, struct lov_io *lio, sub->sub_env = ld->ld_emrg[stripe]->emrg_env; sub->sub_borrowed = 1; } else { - void *cookie; - - /* obtain new environment */ - cookie = cl_env_reenter(); sub->sub_env = cl_env_get(&sub->sub_refcheck); - cl_env_reexit(cookie); if (IS_ERR(sub->sub_env)) result = PTR_ERR(sub->sub_env); diff --git a/lustre/lov/lov_object.c b/lustre/lov/lov_object.c index bd10384..4971163 100644 --- a/lustre/lov/lov_object.c +++ b/lustre/lov/lov_object.c @@ -734,7 +734,6 @@ static int lov_layout_change(const struct lu_env *unused, union lov_layout_state *state = &lov->u; const struct lov_layout_operations *old_ops; const struct lov_layout_operations *new_ops; - void *cookie; struct lu_env *env; __u16 refcheck; int rc; @@ -742,12 +741,9 @@ static int lov_layout_change(const struct lu_env *unused, LASSERT(0 <= lov->lo_type && lov->lo_type < ARRAY_SIZE(lov_dispatch)); - cookie = cl_env_reenter(); env = cl_env_get(&refcheck); - if (IS_ERR(env)) { - cl_env_reexit(cookie); + if (IS_ERR(env)) RETURN(PTR_ERR(env)); - } LASSERT(0 <= llt && llt < ARRAY_SIZE(lov_dispatch)); @@ -789,8 +785,6 @@ static int lov_layout_change(const struct lu_env *unused, out: cl_env_put(env, &refcheck); - cl_env_reexit(cookie); - RETURN(rc); } diff --git a/lustre/obdclass/cl_internal.h b/lustre/obdclass/cl_internal.h index 103262b..c398a83 100644 --- a/lustre/obdclass/cl_internal.h +++ b/lustre/obdclass/cl_internal.h @@ -54,25 +54,6 @@ enum clt_nesting_level { }; /** - * Counters used to check correctness of cl_lock interface usage. - */ -struct cl_thread_counters { - /** - * Number of outstanding calls to cl_lock_mutex_get() made by the - * current thread. For debugging. - */ - int ctc_nr_locks_locked; - /** List of locked locks. */ - struct lu_ref ctc_locks_locked; - /** Number of outstanding holds on locks. */ - int ctc_nr_held; - /** Number of outstanding uses on locks. */ - int ctc_nr_used; - /** Number of held extent locks. */ - int ctc_nr_locks_acquired; -}; - -/** * Thread local state internal for generic cl-code. */ struct cl_thread_info { @@ -87,10 +68,6 @@ struct cl_thread_info { */ struct cl_lock_descr clt_descr; struct cl_page_list clt_list; - /** - * Counters for every level of lock nesting. - */ - struct cl_thread_counters clt_counters[CNL_NR]; /** @} debugging */ /* diff --git a/lustre/obdclass/cl_io.c b/lustre/obdclass/cl_io.c index 2da5856..5728ac0 100644 --- a/lustre/obdclass/cl_io.c +++ b/lustre/obdclass/cl_io.c @@ -424,7 +424,6 @@ void cl_io_unlock(const struct lu_env *env, struct cl_io *io) scan->cis_iop->op[io->ci_type].cio_unlock(env, scan); } io->ci_state = CIS_UNLOCKED; - LASSERT(!cl_env_info(env)->clt_counters[CNL_TOP].ctc_nr_locks_acquired); EXIT; } EXPORT_SYMBOL(cl_io_unlock); diff --git a/lustre/obdclass/cl_object.c b/lustre/obdclass/cl_object.c index f006dd0..8f6705e 100644 --- a/lustre/obdclass/cl_object.c +++ b/lustre/obdclass/cl_object.c @@ -569,39 +569,19 @@ EXPORT_SYMBOL(cl_site_stats_print); * bz20044, bz22683. */ -static struct list_head cl_envs; -static unsigned cl_envs_cached_nr = 0; -static unsigned cl_envs_cached_max = 128; /* XXX: prototype: arbitrary limit - * for now. */ -static DEFINE_SPINLOCK(cl_envs_guard); +static unsigned cl_envs_cached_max = 32; /* XXX: prototype: arbitrary limit + * for now. */ +static struct cl_env_cache { + rwlock_t cec_guard; + unsigned cec_count; + struct list_head cec_envs; +} *cl_envs = NULL; struct cl_env { void *ce_magic; struct lu_env ce_lu; struct lu_context ce_ses; -#ifdef LL_TASK_CL_ENV - void *ce_prev; -#else - /** - * This allows cl_env to be entered into cl_env_hash which implements - * the current thread -> client environment lookup. - */ - struct hlist_node ce_node; -#endif - /** - * Owner for the current cl_env. - * - * If LL_TASK_CL_ENV is defined, this point to the owning current, - * only for debugging purpose ; - * Otherwise hash is used, and this is the key for cfs_hash. - * Now current thread pid is stored. Note using thread pointer would - * lead to unbalanced hash because of its specific allocation locality - * and could be varied for different platforms and OSes, even different - * OS versions. - */ - void *ce_owner; - /* * Linkage into global list of all client environments. Used for * garbage collection. @@ -632,159 +612,13 @@ struct cl_env { static void cl_env_init0(struct cl_env *cle, void *debug) { - LASSERT(cle->ce_ref == 0); - LASSERT(cle->ce_magic == &cl_env_init0); - LASSERT(cle->ce_debug == NULL && cle->ce_owner == NULL); - - cle->ce_ref = 1; - cle->ce_debug = debug; - CL_ENV_INC(busy); -} - - -#ifndef LL_TASK_CL_ENV -/* - * The implementation of using hash table to connect cl_env and thread - */ - -static struct cfs_hash *cl_env_hash; - -static unsigned cl_env_hops_hash(struct cfs_hash *lh, - const void *key, unsigned mask) -{ -#if BITS_PER_LONG == 64 - return cfs_hash_u64_hash((__u64)key, mask); -#else - return cfs_hash_u32_hash((__u32)key, mask); -#endif -} - -static void *cl_env_hops_obj(struct hlist_node *hn) -{ - struct cl_env *cle = hlist_entry(hn, struct cl_env, ce_node); - + LASSERT(cle->ce_ref == 0); LASSERT(cle->ce_magic == &cl_env_init0); - return (void *)cle; -} - -static int cl_env_hops_keycmp(const void *key, struct hlist_node *hn) -{ - struct cl_env *cle = cl_env_hops_obj(hn); + LASSERT(cle->ce_debug == NULL); - LASSERT(cle->ce_owner != NULL); - return (key == cle->ce_owner); -} - -static void cl_env_hops_noop(struct cfs_hash *hs, struct hlist_node *hn) -{ - struct cl_env *cle = hlist_entry(hn, struct cl_env, ce_node); - LASSERT(cle->ce_magic == &cl_env_init0); -} - -static struct cfs_hash_ops cl_env_hops = { - .hs_hash = cl_env_hops_hash, - .hs_key = cl_env_hops_obj, - .hs_keycmp = cl_env_hops_keycmp, - .hs_object = cl_env_hops_obj, - .hs_get = cl_env_hops_noop, - .hs_put_locked = cl_env_hops_noop, -}; - -static inline struct cl_env *cl_env_fetch(void) -{ - struct cl_env *cle; - - cle = cfs_hash_lookup(cl_env_hash, (void *) (long) current->pid); - LASSERT(ergo(cle, cle->ce_magic == &cl_env_init0)); - return cle; -} - -static inline void cl_env_attach(struct cl_env *cle) -{ - if (cle) { - int rc; - - LASSERT(cle->ce_owner == NULL); - cle->ce_owner = (void *) (long) current->pid; - rc = cfs_hash_add_unique(cl_env_hash, cle->ce_owner, - &cle->ce_node); - LASSERT(rc == 0); - } -} - -static inline void cl_env_do_detach(struct cl_env *cle) -{ - void *cookie; - - LASSERT(cle->ce_owner == (void *) (long) current->pid); - cookie = cfs_hash_del(cl_env_hash, cle->ce_owner, - &cle->ce_node); - LASSERT(cookie == cle); - cle->ce_owner = NULL; -} - -static int cl_env_store_init(void) { - cl_env_hash = cfs_hash_create("cl_env", - HASH_CL_ENV_BITS, HASH_CL_ENV_BITS, - HASH_CL_ENV_BKT_BITS, 0, - CFS_HASH_MIN_THETA, - CFS_HASH_MAX_THETA, - &cl_env_hops, - CFS_HASH_RW_BKTLOCK); - return cl_env_hash != NULL ? 0 :-ENOMEM; -} - -static void cl_env_store_fini(void) { - cfs_hash_putref(cl_env_hash); -} - -#else /* LL_TASK_CL_ENV */ -/* - * The implementation of store cl_env directly in thread structure. - */ - -static inline struct cl_env *cl_env_fetch(void) -{ - struct cl_env *cle; - - cle = current->LL_TASK_CL_ENV; - if (cle && cle->ce_magic != &cl_env_init0) - cle = NULL; - return cle; -} - -static inline void cl_env_attach(struct cl_env *cle) -{ - if (cle) { - LASSERT(cle->ce_owner == NULL); - cle->ce_owner = current; - cle->ce_prev = current->LL_TASK_CL_ENV; - current->LL_TASK_CL_ENV = cle; - } -} - -static inline void cl_env_do_detach(struct cl_env *cle) -{ - LASSERT(cle->ce_owner == current); - LASSERT(current->LL_TASK_CL_ENV == cle); - current->LL_TASK_CL_ENV = cle->ce_prev; - cle->ce_owner = NULL; -} - -static int cl_env_store_init(void) { return 0; } -static void cl_env_store_fini(void) { } - -#endif /* LL_TASK_CL_ENV */ - -static inline struct cl_env *cl_env_detach(struct cl_env *cle) -{ - if (cle == NULL) - cle = cl_env_fetch(); - - if (cle && cle->ce_owner) - cl_env_do_detach(cle); - - return cle; + cle->ce_ref = 1; + cle->ce_debug = debug; + CL_ENV_INC(busy); } static struct lu_env *cl_env_new(__u32 ctx_tags, __u32 ses_tags, void *debug) @@ -834,17 +668,22 @@ static struct lu_env *cl_env_obtain(void *debug) { struct cl_env *cle; struct lu_env *env; + int cpu = get_cpu(); ENTRY; - spin_lock(&cl_envs_guard); - LASSERT(equi(cl_envs_cached_nr == 0, list_empty(&cl_envs))); - if (cl_envs_cached_nr > 0) { + + read_lock(&cl_envs[cpu].cec_guard); + LASSERT(equi(cl_envs[cpu].cec_count == 0, + list_empty(&cl_envs[cpu].cec_envs))); + if (cl_envs[cpu].cec_count > 0) { int rc; - cle = container_of(cl_envs.next, struct cl_env, ce_linkage); + cle = container_of(cl_envs[cpu].cec_envs.next, struct cl_env, + ce_linkage); list_del_init(&cle->ce_linkage); - cl_envs_cached_nr--; - spin_unlock(&cl_envs_guard); + cl_envs[cpu].cec_count--; + read_unlock(&cl_envs[cpu].cec_guard); + put_cpu(); env = &cle->ce_lu; rc = lu_env_refill(env); @@ -857,7 +696,8 @@ static struct lu_env *cl_env_obtain(void *debug) env = ERR_PTR(rc); } } else { - spin_unlock(&cl_envs_guard); + read_unlock(&cl_envs[cpu].cec_guard); + put_cpu(); env = cl_env_new(lu_context_tags_default, lu_session_tags_default, debug); } @@ -869,27 +709,6 @@ static inline struct cl_env *cl_env_container(struct lu_env *env) return container_of(env, struct cl_env, ce_lu); } -struct lu_env *cl_env_peek(__u16 *refcheck) -{ - struct lu_env *env; - struct cl_env *cle; - - CL_ENV_INC(lookup); - - /* check that we don't go far from untrusted pointer */ - CLASSERT(offsetof(struct cl_env, ce_magic) == 0); - - env = NULL; - cle = cl_env_fetch(); - if (cle != NULL) { - CL_ENV_INC(hit); - env = &cle->ce_lu; - *refcheck = ++cle->ce_ref; - } - CDEBUG(D_OTHER, "%d@%p\n", cle ? cle->ce_ref : 0, cle); - return env; -} - /** * Returns lu_env: if there already is an environment associated with the * current thread, it is returned, otherwise, new environment is allocated. @@ -907,17 +726,13 @@ struct lu_env *cl_env_get(__u16 *refcheck) { struct lu_env *env; - env = cl_env_peek(refcheck); - if (env == NULL) { - env = cl_env_obtain(__builtin_return_address(0)); - if (!IS_ERR(env)) { - struct cl_env *cle; + env = cl_env_obtain(__builtin_return_address(0)); + if (!IS_ERR(env)) { + struct cl_env *cle; - cle = cl_env_container(env); - cl_env_attach(cle); - *refcheck = cle->ce_ref; - CDEBUG(D_OTHER, "%d@%p\n", cle->ce_ref, cle); - } + cle = cl_env_container(env); + *refcheck = cle->ce_ref; + CDEBUG(D_OTHER, "%d@%p\n", cle->ce_ref, cle); } return env; } @@ -932,7 +747,6 @@ struct lu_env *cl_env_alloc(__u16 *refcheck, __u32 tags) { struct lu_env *env; - LASSERT(cl_env_peek(refcheck) == NULL); env = cl_env_new(tags, tags, __builtin_return_address(0)); if (!IS_ERR(env)) { struct cl_env *cle; @@ -947,7 +761,6 @@ EXPORT_SYMBOL(cl_env_alloc); static void cl_env_exit(struct cl_env *cle) { - LASSERT(cle->ce_owner == NULL); lu_context_exit(&cle->ce_lu.le_ctx); lu_context_exit(&cle->ce_ses); } @@ -960,21 +773,26 @@ static void cl_env_exit(struct cl_env *cle) unsigned cl_env_cache_purge(unsigned nr) { struct cl_env *cle; + unsigned i; ENTRY; - spin_lock(&cl_envs_guard); - for (; !list_empty(&cl_envs) && nr > 0; --nr) { - cle = container_of(cl_envs.next, struct cl_env, ce_linkage); - list_del_init(&cle->ce_linkage); - LASSERT(cl_envs_cached_nr > 0); - cl_envs_cached_nr--; - spin_unlock(&cl_envs_guard); + for_each_possible_cpu(i) { + write_lock(&cl_envs[i].cec_guard); + for (; !list_empty(&cl_envs[i].cec_envs) && nr > 0; --nr) { + cle = container_of(cl_envs[i].cec_envs.next, + struct cl_env, ce_linkage); + list_del_init(&cle->ce_linkage); + LASSERT(cl_envs[i].cec_count > 0); + cl_envs[i].cec_count--; + write_unlock(&cl_envs[i].cec_guard); - cl_env_fini(cle); - spin_lock(&cl_envs_guard); + cl_env_fini(cle); + write_lock(&cl_envs[i].cec_guard); + } + LASSERT(equi(cl_envs[i].cec_count == 0, + list_empty(&cl_envs[i].cec_envs))); + write_unlock(&cl_envs[i].cec_guard); } - LASSERT(equi(cl_envs_cached_nr == 0, list_empty(&cl_envs))); - spin_unlock(&cl_envs_guard); RETURN(nr); } EXPORT_SYMBOL(cl_env_cache_purge); @@ -997,8 +815,9 @@ void cl_env_put(struct lu_env *env, __u16 *refcheck) CDEBUG(D_OTHER, "%d@%p\n", cle->ce_ref, cle); if (--cle->ce_ref == 0) { + int cpu = get_cpu(); + CL_ENV_DEC(busy); - cl_env_detach(cle); cle->ce_debug = NULL; cl_env_exit(cle); /* @@ -1007,108 +826,21 @@ void cl_env_put(struct lu_env *env, __u16 *refcheck) * Return environment to the cache only when it was allocated * with the standard tags. */ - if (cl_envs_cached_nr < cl_envs_cached_max && - (env->le_ctx.lc_tags & ~LCT_HAS_EXIT) == LCT_CL_THREAD && - (env->le_ses->lc_tags & ~LCT_HAS_EXIT) == LCT_SESSION) { - spin_lock(&cl_envs_guard); - list_add(&cle->ce_linkage, &cl_envs); - cl_envs_cached_nr++; - spin_unlock(&cl_envs_guard); + if (cl_envs[cpu].cec_count < cl_envs_cached_max && + (env->le_ctx.lc_tags & ~LCT_HAS_EXIT) == LCT_CL_THREAD && + (env->le_ses->lc_tags & ~LCT_HAS_EXIT) == LCT_SESSION) { + read_lock(&cl_envs[cpu].cec_guard); + list_add(&cle->ce_linkage, &cl_envs[cpu].cec_envs); + cl_envs[cpu].cec_count++; + read_unlock(&cl_envs[cpu].cec_guard); } else cl_env_fini(cle); + put_cpu(); } } EXPORT_SYMBOL(cl_env_put); /** - * Declares a point of re-entrancy. - * - * \see cl_env_reexit() - */ -void *cl_env_reenter(void) -{ - return cl_env_detach(NULL); -} -EXPORT_SYMBOL(cl_env_reenter); - -/** - * Exits re-entrancy. - */ -void cl_env_reexit(void *cookie) -{ - cl_env_detach(NULL); - cl_env_attach(cookie); -} -EXPORT_SYMBOL(cl_env_reexit); - -/** - * Setup user-supplied \a env as a current environment. This is to be used to - * guaranteed that environment exists even when cl_env_get() fails. It is up - * to user to ensure proper concurrency control. - * - * \see cl_env_unplant() - */ -void cl_env_implant(struct lu_env *env, __u16 *refcheck) -{ - struct cl_env *cle = cl_env_container(env); - - LASSERT(cle->ce_ref > 0); - - cl_env_attach(cle); - cl_env_get(refcheck); - CDEBUG(D_OTHER, "%d@%p\n", cle->ce_ref, cle); -} -EXPORT_SYMBOL(cl_env_implant); - -/** - * Detach environment installed earlier by cl_env_implant(). - */ -void cl_env_unplant(struct lu_env *env, __u16 *refcheck) -{ - struct cl_env *cle = cl_env_container(env); - - LASSERT(cle->ce_ref > 1); - - CDEBUG(D_OTHER, "%d@%p\n", cle->ce_ref, cle); - - cl_env_detach(cle); - cl_env_put(env, refcheck); -} -EXPORT_SYMBOL(cl_env_unplant); - -struct lu_env *cl_env_nested_get(struct cl_env_nest *nest) -{ - struct lu_env *env; - - nest->cen_cookie = NULL; - env = cl_env_peek(&nest->cen_refcheck); - if (env != NULL) { - if (!cl_io_is_going(env)) - return env; - else { - cl_env_put(env, &nest->cen_refcheck); - nest->cen_cookie = cl_env_reenter(); - } - } - env = cl_env_get(&nest->cen_refcheck); - if (IS_ERR(env)) { - cl_env_reexit(nest->cen_cookie); - return env; - } - - LASSERT(!cl_io_is_going(env)); - return env; -} -EXPORT_SYMBOL(cl_env_nested_get); - -void cl_env_nested_put(struct cl_env_nest *nest, struct lu_env *env) -{ - cl_env_put(env, &nest->cen_refcheck); - cl_env_reexit(nest->cen_cookie); -} -EXPORT_SYMBOL(cl_env_nested_put); - -/** * Converts struct cl_attr to struct ost_lvb. * * \see cl_lvb2attr @@ -1153,6 +885,10 @@ static int cl_env_percpu_init(void) for_each_possible_cpu(i) { struct lu_env *env; + rwlock_init(&cl_envs[i].cec_guard); + INIT_LIST_HEAD(&cl_envs[i].cec_envs); + cl_envs[i].cec_count = 0; + cle = &cl_env_percpu[i]; env = &cle->ce_lu; @@ -1219,7 +955,6 @@ void cl_env_percpu_put(struct lu_env *env) LASSERT(cle->ce_ref == 0); CL_ENV_DEC(busy); - cl_env_detach(cle); cle->ce_debug = NULL; put_cpu(); @@ -1233,7 +968,6 @@ struct lu_env *cl_env_percpu_get() cle = &cl_env_percpu[get_cpu()]; cl_env_init0(cle, __builtin_return_address(0)); - cl_env_attach(cle); return &cle->ce_lu; } EXPORT_SYMBOL(cl_env_percpu_get); @@ -1298,51 +1032,19 @@ LU_KEY_INIT_FINI(cl0, struct cl_thread_info); static void *cl_key_init(const struct lu_context *ctx, struct lu_context_key *key) { - struct cl_thread_info *info; - - info = cl0_key_init(ctx, key); - if (!IS_ERR(info)) { - size_t i; - - for (i = 0; i < ARRAY_SIZE(info->clt_counters); ++i) - lu_ref_init(&info->clt_counters[i].ctc_locks_locked); - } - return info; + return cl0_key_init(ctx, key); } static void cl_key_fini(const struct lu_context *ctx, struct lu_context_key *key, void *data) { - struct cl_thread_info *info; - size_t i; - - info = data; - for (i = 0; i < ARRAY_SIZE(info->clt_counters); ++i) - lu_ref_fini(&info->clt_counters[i].ctc_locks_locked); - cl0_key_fini(ctx, key, data); -} - -static void cl_key_exit(const struct lu_context *ctx, - struct lu_context_key *key, void *data) -{ - struct cl_thread_info *info = data; - size_t i; - - for (i = 0; i < ARRAY_SIZE(info->clt_counters); ++i) { - LASSERT(info->clt_counters[i].ctc_nr_held == 0); - LASSERT(info->clt_counters[i].ctc_nr_used == 0); - LASSERT(info->clt_counters[i].ctc_nr_locks_acquired == 0); - LASSERT(info->clt_counters[i].ctc_nr_locks_locked == 0); - lu_ref_fini(&info->clt_counters[i].ctc_locks_locked); - lu_ref_init(&info->clt_counters[i].ctc_locks_locked); - } + cl0_key_fini(ctx, key, data); } static struct lu_context_key cl_key = { .lct_tags = LCT_CL_THREAD, .lct_init = cl_key_init, .lct_fini = cl_key_fini, - .lct_exit = cl_key_exit }; static struct lu_kmem_descr cl_object_caches[] = { @@ -1366,35 +1068,33 @@ int cl_global_init(void) { int result; - INIT_LIST_HEAD(&cl_envs); + OBD_ALLOC(cl_envs, sizeof(*cl_envs) * num_possible_cpus()); + if (cl_envs == NULL) + GOTO(out, result = -ENOMEM); - result = cl_env_store_init(); + result = lu_kmem_init(cl_object_caches); if (result) - return result; - - result = lu_kmem_init(cl_object_caches); - if (result) - goto out_store; + GOTO(out_envs, result); - LU_CONTEXT_KEY_INIT(&cl_key); - result = lu_context_key_register(&cl_key); - if (result) - goto out_kmem; + LU_CONTEXT_KEY_INIT(&cl_key); + result = lu_context_key_register(&cl_key); + if (result) + GOTO(out_kmem, result); result = cl_env_percpu_init(); - if (result) - /* no cl_env_percpu_fini on error */ - goto out_context; + if (result) /* no cl_env_percpu_fini on error */ + GOTO(out_keys, result); - return 0; + return 0; -out_context: - lu_context_key_degister(&cl_key); +out_keys: + lu_context_key_degister(&cl_key); out_kmem: - lu_kmem_fini(cl_object_caches); -out_store: - cl_env_store_fini(); - return result; + lu_kmem_fini(cl_object_caches); +out_envs: + OBD_FREE(cl_envs, sizeof(*cl_envs) * num_possible_cpus()); +out: + return result; } /** @@ -1403,7 +1103,7 @@ out_store: void cl_global_fini(void) { cl_env_percpu_fini(); - lu_context_key_degister(&cl_key); - lu_kmem_fini(cl_object_caches); - cl_env_store_fini(); + lu_context_key_degister(&cl_key); + lu_kmem_fini(cl_object_caches); + OBD_FREE(cl_envs, sizeof(*cl_envs) * num_possible_cpus()); } diff --git a/lustre/obdclass/lu_object.c b/lustre/obdclass/lu_object.c index f6f5bb3..c5ca975 100644 --- a/lustre/obdclass/lu_object.c +++ b/lustre/obdclass/lu_object.c @@ -1593,14 +1593,9 @@ static struct list_head lu_context_remembered; */ void lu_context_key_quiesce(struct lu_context_key *key) { - struct lu_context *ctx; - extern unsigned cl_env_cache_purge(unsigned nr); + struct lu_context *ctx; - if (!(key->lct_tags & LCT_QUIESCENT)) { - /* - * XXX layering violation. - */ - cl_env_cache_purge(~0); + if (!(key->lct_tags & LCT_QUIESCENT)) { /* * XXX memory barrier has to go here. */ diff --git a/lustre/obdecho/echo_client.c b/lustre/obdecho/echo_client.c index c29bb65..ea82f26 100644 --- a/lustre/obdecho/echo_client.c +++ b/lustre/obdecho/echo_client.c @@ -639,16 +639,10 @@ static void echo_thread_key_fini(const struct lu_context *ctx, OBD_SLAB_FREE_PTR(info, echo_thread_kmem); } -static void echo_thread_key_exit(const struct lu_context *ctx, - struct lu_context_key *key, void *data) -{ -} - static struct lu_context_key echo_thread_key = { .lct_tags = LCT_CL_THREAD, .lct_init = echo_thread_key_init, .lct_fini = echo_thread_key_fini, - .lct_exit = echo_thread_key_exit }; static void *echo_session_key_init(const struct lu_context *ctx, @@ -669,16 +663,10 @@ static void echo_session_key_fini(const struct lu_context *ctx, OBD_SLAB_FREE_PTR(session, echo_session_kmem); } -static void echo_session_key_exit(const struct lu_context *ctx, - struct lu_context_key *key, void *data) -{ -} - static struct lu_context_key echo_session_key = { .lct_tags = LCT_SESSION, .lct_init = echo_session_key_init, .lct_fini = echo_session_key_fini, - .lct_exit = echo_session_key_exit }; LU_TYPE_INIT_FINI(echo, &echo_thread_key, &echo_session_key); @@ -1068,12 +1056,14 @@ static struct lu_device *echo_device_free(const struct lu_env *env, while (next && !ed->ed_next_ismd) next = next->ld_type->ldt_ops->ldto_device_free(env, next); - LASSERT(ed->ed_site == d->ld_site); - echo_site_fini(env, ed); - cl_device_fini(&ed->ed_cl); - OBD_FREE_PTR(ed); + LASSERT(ed->ed_site == d->ld_site); + echo_site_fini(env, ed); + cl_device_fini(&ed->ed_cl); + OBD_FREE_PTR(ed); - return NULL; + cl_env_cache_purge(~0); + + return NULL; } static const struct lu_device_type_operations echo_device_type_ops = { diff --git a/lustre/osc/osc_cache.c b/lustre/osc/osc_cache.c index 664d4ea..8c03d73 100644 --- a/lustre/osc/osc_cache.c +++ b/lustre/osc/osc_cache.c @@ -961,7 +961,6 @@ static int osc_extent_wait(const struct lu_env *env, struct osc_extent *ext, static int osc_extent_truncate(struct osc_extent *ext, pgoff_t trunc_index, bool partial) { - struct cl_env_nest nest; struct lu_env *env; struct cl_io *io; struct osc_object *obj = ext->oe_obj; @@ -975,6 +974,7 @@ static int osc_extent_truncate(struct osc_extent *ext, pgoff_t trunc_index, int grants = 0; int nr_pages = 0; int rc = 0; + __u16 refcheck; ENTRY; LASSERT(sanity_check(ext) == 0); @@ -984,7 +984,7 @@ static int osc_extent_truncate(struct osc_extent *ext, pgoff_t trunc_index, /* Request new lu_env. * We can't use that env from osc_cache_truncate_start() because * it's from lov_io_sub and not fully initialized. */ - env = cl_env_nested_get(&nest); + env = cl_env_get(&refcheck); io = &osc_env_info(env)->oti_io; io->ci_obj = cl_object_top(osc2cl(obj)); rc = cl_io_init(env, io, CIT_MISC, io->ci_obj); @@ -1069,7 +1069,7 @@ static int osc_extent_truncate(struct osc_extent *ext, pgoff_t trunc_index, out: cl_io_fini(env, io); - cl_env_nested_put(&nest, env); + cl_env_put(env, &refcheck); RETURN(rc); } diff --git a/lustre/osc/osc_lock.c b/lustre/osc/osc_lock.c index 7574887..b5ce0b3 100644 --- a/lustre/osc/osc_lock.c +++ b/lustre/osc/osc_lock.c @@ -305,12 +305,12 @@ static int osc_lock_upcall(void *cookie, struct lustre_handle *lockh, struct osc_lock *oscl = cookie; struct cl_lock_slice *slice = &oscl->ols_cl; struct lu_env *env; - struct cl_env_nest nest; int rc; + __u16 refcheck; ENTRY; - env = cl_env_nested_get(&nest); + env = cl_env_get(&refcheck); /* should never happen, similar to osc_ldlm_blocking_ast(). */ LASSERT(!IS_ERR(env)); @@ -349,7 +349,7 @@ static int osc_lock_upcall(void *cookie, struct lustre_handle *lockh, if (oscl->ols_owner != NULL) cl_sync_io_note(env, oscl->ols_owner, rc); - cl_env_nested_put(&nest, env); + cl_env_put(env, &refcheck); RETURN(rc); } @@ -360,10 +360,10 @@ static int osc_lock_upcall_agl(void *cookie, struct lustre_handle *lockh, struct osc_object *osc = cookie; struct ldlm_lock *dlmlock; struct lu_env *env; - struct cl_env_nest nest; + __u16 refcheck; ENTRY; - env = cl_env_nested_get(&nest); + env = cl_env_get(&refcheck); LASSERT(!IS_ERR(env)); if (errcode == ELDLM_LOCK_MATCHED) @@ -386,7 +386,7 @@ static int osc_lock_upcall_agl(void *cookie, struct lustre_handle *lockh, out: cl_object_put(env, osc2cl(osc)); - cl_env_nested_put(&nest, env); + cl_env_put(env, &refcheck); RETURN(ldlm_error2errno(errcode)); } @@ -394,13 +394,13 @@ static int osc_lock_flush(struct osc_object *obj, pgoff_t start, pgoff_t end, enum cl_lock_mode mode, int discard) { struct lu_env *env; - struct cl_env_nest nest; + __u16 refcheck; int rc = 0; int rc2 = 0; ENTRY; - env = cl_env_nested_get(&nest); + env = cl_env_get(&refcheck); if (IS_ERR(env)) RETURN(PTR_ERR(env)); @@ -418,7 +418,7 @@ static int osc_lock_flush(struct osc_object *obj, pgoff_t start, pgoff_t end, if (rc == 0 && rc2 < 0) rc = rc2; - cl_env_nested_put(&nest, env); + cl_env_put(env, &refcheck); RETURN(rc); } @@ -550,7 +550,7 @@ static int osc_ldlm_blocking_ast(struct ldlm_lock *dlmlock, } case LDLM_CB_CANCELING: { struct lu_env *env; - struct cl_env_nest nest; + __u16 refcheck; /* * This can be called in the context of outer IO, e.g., @@ -563,14 +563,14 @@ static int osc_ldlm_blocking_ast(struct ldlm_lock *dlmlock, * new environment has to be created to not corrupt outer * context. */ - env = cl_env_nested_get(&nest); + env = cl_env_get(&refcheck); if (IS_ERR(env)) { result = PTR_ERR(env); break; } result = osc_dlm_blocking_ast0(env, dlmlock, data, flag); - cl_env_nested_put(&nest, env); + cl_env_put(env, &refcheck); break; } default: @@ -582,61 +582,62 @@ static int osc_ldlm_blocking_ast(struct ldlm_lock *dlmlock, static int osc_ldlm_glimpse_ast(struct ldlm_lock *dlmlock, void *data) { struct ptlrpc_request *req = data; - struct cl_env_nest nest; struct lu_env *env; struct ost_lvb *lvb; struct req_capsule *cap; + struct cl_object *obj = NULL; int result; + __u16 refcheck; ENTRY; LASSERT(lustre_msg_get_opc(req->rq_reqmsg) == LDLM_GL_CALLBACK); - env = cl_env_nested_get(&nest); - if (!IS_ERR(env)) { - struct cl_object *obj = NULL; + env = cl_env_get(&refcheck); + if (IS_ERR(env)) + GOTO(out, result = PTR_ERR(env)); - lock_res_and_lock(dlmlock); - if (dlmlock->l_ast_data != NULL) { - obj = osc2cl(dlmlock->l_ast_data); - cl_object_get(obj); + + lock_res_and_lock(dlmlock); + if (dlmlock->l_ast_data != NULL) { + obj = osc2cl(dlmlock->l_ast_data); + cl_object_get(obj); + } + unlock_res_and_lock(dlmlock); + + if (obj != NULL) { + /* Do not grab the mutex of cl_lock for glimpse. + * See LU-1274 for details. + * BTW, it's okay for cl_lock to be cancelled during + * this period because server can handle this race. + * See ldlm_server_glimpse_ast() for details. + * cl_lock_mutex_get(env, lock); */ + cap = &req->rq_pill; + req_capsule_extend(cap, &RQF_LDLM_GL_CALLBACK); + req_capsule_set_size(cap, &RMF_DLM_LVB, RCL_SERVER, + sizeof *lvb); + result = req_capsule_server_pack(cap); + if (result == 0) { + lvb = req_capsule_server_get(cap, &RMF_DLM_LVB); + result = cl_object_glimpse(env, obj, lvb); } - unlock_res_and_lock(dlmlock); + if (!exp_connect_lvb_type(req->rq_export)) + req_capsule_shrink(&req->rq_pill, &RMF_DLM_LVB, + sizeof(struct ost_lvb_v1), RCL_SERVER); + cl_object_put(env, obj); + } else { + /* + * These errors are normal races, so we don't want to + * fill the console with messages by calling + * ptlrpc_error() + */ + lustre_pack_reply(req, 1, NULL, NULL); + result = -ELDLM_NO_LOCK_DATA; + } + cl_env_put(env, &refcheck); + EXIT; - if (obj != NULL) { - /* Do not grab the mutex of cl_lock for glimpse. - * See LU-1274 for details. - * BTW, it's okay for cl_lock to be cancelled during - * this period because server can handle this race. - * See ldlm_server_glimpse_ast() for details. - * cl_lock_mutex_get(env, lock); */ - cap = &req->rq_pill; - req_capsule_extend(cap, &RQF_LDLM_GL_CALLBACK); - req_capsule_set_size(cap, &RMF_DLM_LVB, RCL_SERVER, - sizeof *lvb); - result = req_capsule_server_pack(cap); - if (result == 0) { - lvb = req_capsule_server_get(cap, &RMF_DLM_LVB); - result = cl_object_glimpse(env, obj, lvb); - } - if (!exp_connect_lvb_type(req->rq_export)) - req_capsule_shrink(&req->rq_pill, - &RMF_DLM_LVB, - sizeof(struct ost_lvb_v1), - RCL_SERVER); - cl_object_put(env, obj); - } else { - /* - * These errors are normal races, so we don't want to - * fill the console with messages by calling - * ptlrpc_error() - */ - lustre_pack_reply(req, 1, NULL, NULL); - result = -ELDLM_NO_LOCK_DATA; - } - cl_env_nested_put(&nest, env); - } else - result = PTR_ERR(env); +out: req->rq_status = result; RETURN(result); } @@ -692,12 +693,12 @@ static unsigned long osc_lock_weight(const struct lu_env *env, */ unsigned long osc_ldlm_weigh_ast(struct ldlm_lock *dlmlock) { - struct cl_env_nest nest; struct lu_env *env; struct osc_object *obj; struct osc_lock *oscl; unsigned long weight; bool found = false; + __u16 refcheck; ENTRY; might_sleep(); @@ -708,7 +709,7 @@ unsigned long osc_ldlm_weigh_ast(struct ldlm_lock *dlmlock) * the upper context because cl_lock_put don't modify environment * variables. But just in case .. */ - env = cl_env_nested_get(&nest); + env = cl_env_get(&refcheck); if (IS_ERR(env)) /* Mostly because lack of memory, do not eliminate this lock */ RETURN(1); @@ -736,7 +737,7 @@ unsigned long osc_ldlm_weigh_ast(struct ldlm_lock *dlmlock) EXIT; out: - cl_env_nested_put(&nest, env); + cl_env_put(env, &refcheck); return weight; } diff --git a/lustre/osc/osc_page.c b/lustre/osc/osc_page.c index 2d97d8a..4f2e01b 100644 --- a/lustre/osc/osc_page.c +++ b/lustre/osc/osc_page.c @@ -807,16 +807,16 @@ long osc_lru_shrink(const struct lu_env *env, struct client_obd *cli, */ static long osc_lru_reclaim(struct client_obd *cli, unsigned long npages) { - struct cl_env_nest nest; struct lu_env *env; struct cl_client_cache *cache = cli->cl_cache; int max_scans; + __u16 refcheck; long rc = 0; ENTRY; LASSERT(cache != NULL); - env = cl_env_nested_get(&nest); + env = cl_env_get(&refcheck); if (IS_ERR(env)) RETURN(rc); @@ -871,7 +871,7 @@ static long osc_lru_reclaim(struct client_obd *cli, unsigned long npages) spin_unlock(&cache->ccc_lru_lock); out: - cl_env_nested_put(&nest, env); + cl_env_put(env, &refcheck); CDEBUG(D_CACHE, "%s: cli %p freed %ld pages.\n", cli_name(cli), cli, rc); return rc; @@ -1163,10 +1163,10 @@ unsigned long osc_cache_shrink_scan(struct shrinker *sk, { struct client_obd *cli; struct client_obd *stop_anchor = NULL; - struct cl_env_nest nest; struct lu_env *env; long shrank = 0; int rc; + __u16 refcheck; if (sc->nr_to_scan == 0) return 0; @@ -1174,7 +1174,7 @@ unsigned long osc_cache_shrink_scan(struct shrinker *sk, if (!(sc->gfp_mask & __GFP_FS)) return SHRINK_STOP; - env = cl_env_nested_get(&nest); + env = cl_env_get(&refcheck); if (IS_ERR(env)) return SHRINK_STOP; @@ -1207,7 +1207,7 @@ unsigned long osc_cache_shrink_scan(struct shrinker *sk, spin_unlock(&osc_shrink_lock); out: - cl_env_nested_put(&nest, env); + cl_env_put(env, &refcheck); return shrank; }