void lu_object_put(const struct lu_env *env, struct lu_object *o)
{
struct lu_site_bkt_data *bkt;
- struct lu_object_header *top;
- struct lu_site *site;
- struct lu_object *orig;
+ struct lu_object_header *top = o->lo_header;
+ struct lu_site *site = o->lo_dev->ld_site;
+ struct lu_object *orig = o;
struct cfs_hash_bd bd;
- const struct lu_fid *fid;
-
- top = o->lo_header;
- site = o->lo_dev->ld_site;
- orig = o;
+ const struct lu_fid *fid = lu_object_fid(o);
+ bool is_dying;
/*
* till we have full fids-on-OST implemented anonymous objects
* are possible in OSP. such an object isn't listed in the site
* so we should not remove it from the site.
*/
- fid = lu_object_fid(o);
if (fid_is_zero(fid)) {
LASSERT(top->loh_hash.next == NULL
&& top->loh_hash.pprev == NULL);
cfs_hash_bd_get(site->ls_obj_hash, &top->loh_fid, &bd);
bkt = cfs_hash_bd_extra_get(site->ls_obj_hash, &bd);
+ is_dying = lu_object_is_dying(top);
if (!cfs_hash_bd_dec_and_lock(site->ls_obj_hash, &bd, &top->loh_ref)) {
- if (lu_object_is_dying(top)) {
+ /* at this point the object reference is dropped and lock is
+ * not taken, so lu_object should not be touched because it
+ * can be freed by concurrent thread. Use local variable for
+ * check.
+ */
+ if (is_dying) {
/*
* somebody may be waiting for this, currently only
* used for cl_object, see cl_object_put_last().
o->lo_ops->loo_object_release(env, o);
}
+ /* don't use local 'is_dying' here because if was taken without lock
+ * but here we need the latest actual value of it so check lu_object
+ * directly here.
+ */
if (!lu_object_is_dying(top) &&
(lu_object_exists(orig) || lu_object_is_cl(orig))) {
LASSERT(list_empty(&top->loh_lru));
static struct lu_context_key *lu_keys[LU_CONTEXT_KEY_NR] = { NULL, };
-DEFINE_RWLOCK(lu_keys_guard);
static DECLARE_RWSEM(lu_key_initing);
/**
LASSERT(key->lct_owner != NULL);
result = -ENFILE;
- write_lock(&lu_keys_guard);
+ atomic_set(&key->lct_used, 1);
+ lu_ref_init(&key->lct_reference);
for (i = 0; i < ARRAY_SIZE(lu_keys); ++i) {
- if (lu_keys[i] == NULL) {
- key->lct_index = i;
- atomic_set(&key->lct_used, 1);
- lu_keys[i] = key;
- lu_ref_init(&key->lct_reference);
- result = 0;
- atomic_inc(&key_set_version);
- break;
- }
+ if (lu_keys[i])
+ continue;
+ key->lct_index = i;
+ if (cmpxchg(&lu_keys[i], NULL, key) != NULL)
+ continue;
+
+ result = 0;
+ atomic_inc(&key_set_version);
+ break;
}
- write_unlock(&lu_keys_guard);
+ if (result) {
+ lu_ref_fini(&key->lct_reference);
+ atomic_set(&key->lct_used, 0);
+ }
return result;
}
EXPORT_SYMBOL(lu_context_key_register);
key = lu_keys[index];
LASSERT(key != NULL);
LASSERT(key->lct_fini != NULL);
- LASSERT(atomic_read(&key->lct_used) > 1);
+ LASSERT(atomic_read(&key->lct_used) > 0);
key->lct_fini(ctx, key, ctx->lc_value[index]);
lu_ref_del(&key->lct_reference, "ctx", ctx);
- atomic_dec(&key->lct_used);
+ if (atomic_dec_and_test(&key->lct_used))
+ wake_up_var(&key->lct_used);
LASSERT(key->lct_owner != NULL);
if ((ctx->lc_tags & LCT_NOREF) == 0) {
lu_context_key_quiesce(key);
- write_lock(&lu_keys_guard);
key_fini(&lu_shrink_env.le_ctx, key->lct_index);
/**
* Wait until all transient contexts referencing this key have
* run lu_context_key::lct_fini() method.
*/
- while (atomic_read(&key->lct_used) > 1) {
- write_unlock(&lu_keys_guard);
- CDEBUG(D_INFO, "lu_context_key_degister: \"%s\" %p, %d\n",
- key->lct_owner ? key->lct_owner->name : "", key,
- atomic_read(&key->lct_used));
- schedule();
- write_lock(&lu_keys_guard);
- }
- if (lu_keys[key->lct_index]) {
- lu_keys[key->lct_index] = NULL;
+ atomic_dec(&key->lct_used);
+ wait_var_event(&key->lct_used, atomic_read(&key->lct_used) == 0);
+
+ if (!WARN_ON(lu_keys[key->lct_index] == NULL))
lu_ref_fini(&key->lct_reference);
- }
- write_unlock(&lu_keys_guard);
- LASSERTF(atomic_read(&key->lct_used) == 1,
- "key has instances: %d\n",
- atomic_read(&key->lct_used));
+ smp_store_release(&lu_keys[key->lct_index], NULL);
}
EXPORT_SYMBOL(lu_context_key_degister);
* List of remembered contexts. XXX document me.
*/
static LIST_HEAD(lu_context_remembered);
+static DEFINE_SPINLOCK(lu_context_remembered_guard);
/**
* Destroy \a key in all remembered contexts. This is used to destroy key
key->lct_tags |= LCT_QUIESCENT;
up_write(&lu_key_initing);
- write_lock(&lu_keys_guard);
+ spin_lock(&lu_context_remembered_guard);
list_for_each_entry(ctx, &lu_context_remembered, lc_remember) {
spin_until_cond(READ_ONCE(ctx->lc_state) != LCS_LEAVING);
key_fini(ctx, key->lct_index);
}
- write_unlock(&lu_keys_guard);
+ spin_unlock(&lu_context_remembered_guard);
}
}
ctx->lc_state = LCS_INITIALIZED;
ctx->lc_tags = tags;
if (tags & LCT_REMEMBER) {
- write_lock(&lu_keys_guard);
+ spin_lock(&lu_context_remembered_guard);
list_add(&ctx->lc_remember, &lu_context_remembered);
- write_unlock(&lu_keys_guard);
+ spin_unlock(&lu_context_remembered_guard);
} else {
INIT_LIST_HEAD(&ctx->lc_remember);
}
if ((ctx->lc_tags & LCT_REMEMBER) == 0) {
LASSERT(list_empty(&ctx->lc_remember));
- keys_fini(ctx);
-
- } else { /* could race with key degister */
- write_lock(&lu_keys_guard);
- keys_fini(ctx);
+ } else {
+ /* could race with key degister */
+ spin_lock(&lu_context_remembered_guard);
list_del_init(&ctx->lc_remember);
- write_unlock(&lu_keys_guard);
+ spin_unlock(&lu_context_remembered_guard);
}
+ keys_fini(ctx);
}
EXPORT_SYMBOL(lu_context_fini);
#ifdef HAVE_SERVER_SUPPORT
void lu_context_tags_update(__u32 tags)
{
- write_lock(&lu_keys_guard);
+ spin_lock(&lu_context_remembered_guard);
lu_context_tags_default |= tags;
atomic_inc(&key_set_version);
- write_unlock(&lu_keys_guard);
+ spin_unlock(&lu_context_remembered_guard);
}
EXPORT_SYMBOL(lu_context_tags_update);
void lu_context_tags_clear(__u32 tags)
{
- write_lock(&lu_keys_guard);
+ spin_lock(&lu_context_remembered_guard);
lu_context_tags_default &= ~tags;
atomic_inc(&key_set_version);
- write_unlock(&lu_keys_guard);
+ spin_unlock(&lu_context_remembered_guard);
}
EXPORT_SYMBOL(lu_context_tags_clear);
void lu_session_tags_update(__u32 tags)
{
- write_lock(&lu_keys_guard);
+ spin_lock(&lu_context_remembered_guard);
lu_session_tags_default |= tags;
atomic_inc(&key_set_version);
- write_unlock(&lu_keys_guard);
+ spin_unlock(&lu_context_remembered_guard);
}
EXPORT_SYMBOL(lu_session_tags_update);
void lu_session_tags_clear(__u32 tags)
{
- write_lock(&lu_keys_guard);
+ spin_lock(&lu_context_remembered_guard);
lu_session_tags_default &= ~tags;
atomic_inc(&key_set_version);
- write_unlock(&lu_keys_guard);
+ spin_unlock(&lu_context_remembered_guard);
}
EXPORT_SYMBOL(lu_session_tags_clear);
#endif /* HAVE_SERVER_SUPPORT */
}
EXPORT_SYMBOL(lu_env_refill_by_tags);
+
+struct lu_env_item {
+ struct task_struct *lei_task; /* rhashtable key */
+ struct rhash_head lei_linkage;
+ struct lu_env *lei_env;
+};
+
+static const struct rhashtable_params lu_env_rhash_params = {
+ .key_len = sizeof(struct task_struct *),
+ .key_offset = offsetof(struct lu_env_item, lei_task),
+ .head_offset = offsetof(struct lu_env_item, lei_linkage),
+ };
+
+struct rhashtable lu_env_rhash;
+
+struct lu_env_percpu {
+ struct task_struct *lep_task;
+ struct lu_env *lep_env ____cacheline_aligned_in_smp;
+};
+
+static struct lu_env_percpu lu_env_percpu[NR_CPUS];
+
+int lu_env_add(struct lu_env *env)
+{
+ struct lu_env_item *lei, *old;
+
+ LASSERT(env);
+
+ OBD_ALLOC_PTR(lei);
+ if (!lei)
+ return -ENOMEM;
+
+ lei->lei_task = current;
+ lei->lei_env = env;
+
+ old = rhashtable_lookup_get_insert_fast(&lu_env_rhash,
+ &lei->lei_linkage,
+ lu_env_rhash_params);
+ LASSERT(!old);
+
+ return 0;
+}
+EXPORT_SYMBOL(lu_env_add);
+
+void lu_env_remove(struct lu_env *env)
+{
+ struct lu_env_item *lei;
+ const void *task = current;
+ int i;
+
+ for_each_possible_cpu(i) {
+ if (lu_env_percpu[i].lep_env == env) {
+ LASSERT(lu_env_percpu[i].lep_task == task);
+ lu_env_percpu[i].lep_task = NULL;
+ lu_env_percpu[i].lep_env = NULL;
+ }
+ }
+
+ rcu_read_lock();
+ lei = rhashtable_lookup_fast(&lu_env_rhash, &task,
+ lu_env_rhash_params);
+ if (lei && rhashtable_remove_fast(&lu_env_rhash, &lei->lei_linkage,
+ lu_env_rhash_params) == 0)
+ OBD_FREE_PTR(lei);
+ rcu_read_unlock();
+}
+EXPORT_SYMBOL(lu_env_remove);
+
+struct lu_env *lu_env_find(void)
+{
+ struct lu_env *env = NULL;
+ struct lu_env_item *lei;
+ const void *task = current;
+ int i = get_cpu();
+
+ if (lu_env_percpu[i].lep_task == current) {
+ env = lu_env_percpu[i].lep_env;
+ put_cpu();
+ LASSERT(env);
+ return env;
+ }
+
+ lei = rhashtable_lookup_fast(&lu_env_rhash, &task,
+ lu_env_rhash_params);
+ if (lei) {
+ env = lei->lei_env;
+ lu_env_percpu[i].lep_task = current;
+ lu_env_percpu[i].lep_env = env;
+ }
+ put_cpu();
+
+ return env;
+}
+EXPORT_SYMBOL(lu_env_find);
+
static struct shrinker *lu_site_shrinker;
typedef struct lu_site_stats{
*/
int lu_global_init(void)
{
- int result;
+ int result;
DEF_SHRINKER_VAR(shvar, lu_cache_shrink,
lu_cache_shrink_count, lu_cache_shrink_scan);
if (lu_site_shrinker == NULL)
return -ENOMEM;
+ result = rhashtable_init(&lu_env_rhash, &lu_env_rhash_params);
+
return result;
}
lu_env_fini(&lu_shrink_env);
up_write(&lu_sites_guard);
+ rhashtable_destroy(&lu_env_rhash);
+
lu_ref_global_fini();
}
git://git.whamcloud.com / fs / lustre-release.git / blobdiff