X-Git-Url: https://git.whamcloud.com/?a=blobdiff_plain;f=lustre%2Fobdclass%2Fcl_page.c;h=67ca346cb4b6c3cec62bf162aa415ba0d3b4b62d;hb=5cae09a2409dcd396a1ee7be1eca7d3bbf77365e;hp=1266c79acdd9b7a4fa76709d0fa5448d29565c01;hpb=b14160d67ef3fc5b0d294c8d76c20317a05cda8c;p=fs%2Flustre-release.git diff --git a/lustre/obdclass/cl_page.c b/lustre/obdclass/cl_page.c index 1266c79..67ca346 100644 --- a/lustre/obdclass/cl_page.c +++ b/lustre/obdclass/cl_page.c @@ -90,9 +90,26 @@ static struct lu_kmem_descr cl_page_caches[] = { ((void)sizeof(env), (void)sizeof(page), (void)sizeof !!(exp)) #endif /* !INVARIANT_CHECK */ +/* Disable page statistic by default due to huge performance penalty. */ +#ifdef CONFIG_DEBUG_PAGESTATE_TRACKING +#define CS_PAGE_INC(o, item) \ + cfs_atomic_inc(&cl_object_site(o)->cs_pages.cs_stats[CS_##item]) +#define CS_PAGE_DEC(o, item) \ + cfs_atomic_dec(&cl_object_site(o)->cs_pages.cs_stats[CS_##item]) +#define CS_PAGESTATE_INC(o, state) \ + cfs_atomic_inc(&cl_object_site(o)->cs_pages_state[state]) +#define CS_PAGESTATE_DEC(o, state) \ + cfs_atomic_dec(&cl_object_site(o)->cs_pages_state[state]) +#else +#define CS_PAGE_INC(o, item) +#define CS_PAGE_DEC(o, item) +#define CS_PAGESTATE_INC(o, state) +#define CS_PAGESTATE_DEC(o, state) +#endif + /** * Internal version of cl_page_top, it should be called with page referenced, - * or coh_page_guard held. + * or cp_lock held. */ static struct cl_page *cl_page_top_trusted(struct cl_page *page) { @@ -118,7 +135,7 @@ static void cl_page_get_trust(struct cl_page *page) * Checkless version for trusted users. */ if (cfs_atomic_inc_return(&page->cp_ref) == 1) - cfs_atomic_inc(&cl_object_site(page->cp_obj)->cs_pages.cs_busy); + CS_PAGE_INC(page->cp_obj, busy); } /** @@ -134,10 +151,8 @@ cl_page_at_trusted(const struct cl_page *page, const struct cl_page_slice *slice; #ifdef INVARIANT_CHECK - struct cl_object_header *ch = cl_object_header(page->cp_obj); - if (!cfs_atomic_read(&page->cp_ref)) - LASSERT_SPIN_LOCKED(&ch->coh_page_guard); + LASSERT_SPIN_LOCKED(&page->cp_lock); #endif ENTRY; @@ -205,7 +220,7 @@ int cl_page_gang_lookup(const struct lu_env *env, struct cl_object *obj, hdr = cl_object_header(obj); pvec = cl_env_info(env)->clt_pvec; dtype = cl_object_top(obj)->co_lu.lo_dev->ld_type; - cfs_spin_lock(&hdr->coh_page_guard); + spin_lock(&hdr->coh_page_guard); while ((nr = radix_tree_gang_lookup(&hdr->coh_tree, (void **)pvec, idx, CLT_PVEC_SIZE)) > 0) { int end_of_region = 0; @@ -251,7 +266,7 @@ int cl_page_gang_lookup(const struct lu_env *env, struct cl_object *obj, * check that pages weren't truncated (cl_page_own() returns * error in the latter case). */ - cfs_spin_unlock(&hdr->coh_page_guard); + spin_unlock(&hdr->coh_page_guard); tree_lock = 0; for (i = 0; i < j; ++i) { @@ -270,19 +285,18 @@ int cl_page_gang_lookup(const struct lu_env *env, struct cl_object *obj, if (res != CLP_GANG_OKAY) break; - cfs_spin_lock(&hdr->coh_page_guard); - tree_lock = 1; - } - if (tree_lock) - cfs_spin_unlock(&hdr->coh_page_guard); - RETURN(res); + spin_lock(&hdr->coh_page_guard); + tree_lock = 1; + } + if (tree_lock) + spin_unlock(&hdr->coh_page_guard); + RETURN(res); } EXPORT_SYMBOL(cl_page_gang_lookup); static void cl_page_free(const struct lu_env *env, struct cl_page *page) { struct cl_object *obj = page->cp_obj; - struct cl_site *site = cl_object_site(obj); PASSERT(env, page, cfs_list_empty(&page->cp_batch)); PASSERT(env, page, page->cp_owner == NULL); @@ -300,11 +314,8 @@ static void cl_page_free(const struct lu_env *env, struct cl_page *page) cfs_list_del_init(page->cp_layers.next); slice->cpl_ops->cpo_fini(env, slice); } - cfs_atomic_dec(&site->cs_pages.cs_total); - -#ifdef LUSTRE_PAGESTATE_TRACKING - cfs_atomic_dec(&site->cs_pages_state[page->cp_state]); -#endif + CS_PAGE_DEC(obj, total); + CS_PAGESTATE_DEC(obj, page->cp_state); lu_object_ref_del_at(&obj->co_lu, page->cp_obj_ref, "cl_page", page); cl_object_put(env, obj); lu_ref_fini(&page->cp_reference); @@ -330,7 +341,6 @@ static int cl_page_alloc(const struct lu_env *env, struct cl_object *o, struct cl_page *page; struct cl_page *err = NULL; struct lu_object_header *head; - struct cl_site *site = cl_object_site(o); int result; ENTRY; @@ -344,11 +354,12 @@ static int cl_page_alloc(const struct lu_env *env, struct cl_object *o, "cl_page", page); page->cp_index = ind; cl_page_state_set_trust(page, CPS_CACHED); - page->cp_type = type; - CFS_INIT_LIST_HEAD(&page->cp_layers); - CFS_INIT_LIST_HEAD(&page->cp_batch); - CFS_INIT_LIST_HEAD(&page->cp_flight); - cfs_mutex_init(&page->cp_mutex); + spin_lock_init(&page->cp_lock); + page->cp_type = type; + CFS_INIT_LIST_HEAD(&page->cp_layers); + CFS_INIT_LIST_HEAD(&page->cp_batch); + CFS_INIT_LIST_HEAD(&page->cp_flight); + mutex_init(&page->cp_mutex); lu_ref_init(&page->cp_reference); head = o->co_lu.lo_header; cfs_list_for_each_entry(o, &head->loh_layers, @@ -365,13 +376,10 @@ static int cl_page_alloc(const struct lu_env *env, struct cl_object *o, } } if (err == NULL) { - cfs_atomic_inc(&site->cs_pages.cs_busy); - cfs_atomic_inc(&site->cs_pages.cs_total); - -#ifdef LUSTRE_PAGESTATE_TRACKING - cfs_atomic_inc(&site->cs_pages_state[CPS_CACHED]); -#endif - cfs_atomic_inc(&site->cs_pages.cs_created); + CS_PAGE_INC(o, busy); + CS_PAGE_INC(o, total); + CS_PAGE_INC(o, create); + CS_PAGESTATE_DEC(o, CPS_CACHED); result = 0; } } else @@ -400,7 +408,6 @@ static struct cl_page *cl_page_find0(const struct lu_env *env, struct cl_page *page = NULL; struct cl_page *ghost = NULL; struct cl_object_header *hdr; - struct cl_site *site = cl_object_site(o); int err; LASSERT(type == CPT_CACHEABLE || type == CPT_TRANSIENT); @@ -409,12 +416,17 @@ static struct cl_page *cl_page_find0(const struct lu_env *env, ENTRY; hdr = cl_object_header(o); - cfs_atomic_inc(&site->cs_pages.cs_lookup); + CS_PAGE_INC(o, lookup); CDEBUG(D_PAGE, "%lu@"DFID" %p %lx %d\n", idx, PFID(&hdr->coh_lu.loh_fid), vmpage, vmpage->private, type); /* fast path. */ if (type == CPT_CACHEABLE) { + /* cl_page::cp_lock is used to protect the page state and + * refcount, but need an external lock to protect the + * child/parent relationship, so vmpage lock must be held for + * this purpose. */ + KLASSERT(PageLocked(vmpage)); /* * cl_vmpage_page() can be called here without any locks as * @@ -433,7 +445,7 @@ static struct cl_page *cl_page_find0(const struct lu_env *env, } if (page != NULL) { - cfs_atomic_inc(&site->cs_pages.cs_hit); + CS_PAGE_INC(o, hit); RETURN(page); } @@ -455,7 +467,7 @@ static struct cl_page *cl_page_find0(const struct lu_env *env, * XXX optimization: use radix_tree_preload() here, and change tree * gfp mask to GFP_KERNEL in cl_object_header_init(). */ - cfs_spin_lock(&hdr->coh_page_guard); + spin_lock(&hdr->coh_page_guard); err = radix_tree_insert(&hdr->coh_tree, idx, page); if (err != 0) { ghost = page; @@ -483,10 +495,10 @@ static struct cl_page *cl_page_find0(const struct lu_env *env, } hdr->coh_pages++; } - cfs_spin_unlock(&hdr->coh_page_guard); + spin_unlock(&hdr->coh_page_guard); if (unlikely(ghost != NULL)) { - cfs_atomic_dec(&site->cs_pages.cs_busy); + CS_PAGE_DEC(o, busy); cl_page_delete0(env, ghost, 0); cl_page_free(env, ghost); } @@ -550,9 +562,6 @@ static void cl_page_state_set0(const struct lu_env *env, struct cl_page *page, enum cl_page_state state) { enum cl_page_state old; -#ifdef LUSTRE_PAGESTATE_TRACKING - struct cl_site *site = cl_object_site(page->cp_obj); -#endif /* * Matrix of allowed state transitions [old][new], for sanity @@ -605,10 +614,8 @@ static void cl_page_state_set0(const struct lu_env *env, PASSERT(env, page, equi(state == CPS_OWNED, page->cp_owner != NULL)); -#ifdef LUSTRE_PAGESTATE_TRACKING - cfs_atomic_dec(&site->cs_pages_state[page->cp_state]); - cfs_atomic_inc(&site->cs_pages_state[state]); -#endif + CS_PAGESTATE_DEC(page->cp_obj, page->cp_state); + CS_PAGESTATE_INC(page->cp_obj, state); cl_page_state_set_trust(page, state); } EXIT; @@ -630,10 +637,9 @@ static void cl_page_state_set(const struct lu_env *env, */ void cl_page_get(struct cl_page *page) { - ENTRY; - LASSERT(page->cp_state != CPS_FREEING); - cl_page_get_trust(page); - EXIT; + ENTRY; + cl_page_get_trust(page); + EXIT; } EXPORT_SYMBOL(cl_page_get); @@ -648,28 +654,24 @@ EXPORT_SYMBOL(cl_page_get); */ void cl_page_put(const struct lu_env *env, struct cl_page *page) { - struct cl_object_header *hdr; - struct cl_site *site = cl_object_site(page->cp_obj); - PASSERT(env, page, cfs_atomic_read(&page->cp_ref) > !!page->cp_parent); ENTRY; CL_PAGE_HEADER(D_TRACE, env, page, "%d\n", cfs_atomic_read(&page->cp_ref)); - hdr = cl_object_header(cl_object_top(page->cp_obj)); - if (cfs_atomic_dec_and_lock(&page->cp_ref, &hdr->coh_page_guard)) { - cfs_atomic_dec(&site->cs_pages.cs_busy); + if (cfs_atomic_dec_and_lock(&page->cp_ref, &page->cp_lock)) { + CS_PAGE_DEC(page->cp_obj, busy); /* We're going to access the page w/o a reference, but it's - * ok because we have grabbed the lock coh_page_guard, which + * ok because we have grabbed the lock cp_lock, which * means nobody is able to free this page behind us. */ if (page->cp_state == CPS_FREEING) { /* We drop the page reference and check the page state - * inside the coh_page_guard. So that if it gets here, + * inside the cp_lock. So that if it gets here, * it is the REALLY last reference to this page. */ - cfs_spin_unlock(&hdr->coh_page_guard); + spin_unlock(&page->cp_lock); LASSERT(cfs_atomic_read(&page->cp_ref) == 0); PASSERT(env, page, page->cp_owner == NULL); @@ -683,10 +685,10 @@ void cl_page_put(const struct lu_env *env, struct cl_page *page) EXIT; return; } - cfs_spin_unlock(&hdr->coh_page_guard); - } + spin_unlock(&page->cp_lock); + } - EXIT; + EXIT; } EXPORT_SYMBOL(cl_page_put); @@ -718,8 +720,8 @@ EXPORT_SYMBOL(cl_page_vmpage); */ struct cl_page *cl_vmpage_page(cfs_page_t *vmpage, struct cl_object *obj) { - struct cl_page *page; - struct cl_object_header *hdr; + struct cl_page *top; + struct cl_page *page; ENTRY; KLASSERT(PageLocked(vmpage)); @@ -734,18 +736,20 @@ struct cl_page *cl_vmpage_page(cfs_page_t *vmpage, struct cl_object *obj) * This loop assumes that ->private points to the top-most page. This * can be rectified easily. */ - hdr = cl_object_header(cl_object_top(obj)); - cfs_spin_lock(&hdr->coh_page_guard); - for (page = (void *)vmpage->private; - page != NULL; page = page->cp_child) { - if (cl_object_same(page->cp_obj, obj)) { - cl_page_get_trust(page); - break; - } - } - cfs_spin_unlock(&hdr->coh_page_guard); - LASSERT(ergo(page, page->cp_type == CPT_CACHEABLE)); - RETURN(page); + top = (struct cl_page *)vmpage->private; + if (top == NULL) + RETURN(NULL); + + spin_lock(&top->cp_lock); + for (page = top; page != NULL; page = page->cp_child) { + if (cl_object_same(page->cp_obj, obj)) { + cl_page_get_trust(page); + break; + } + } + spin_unlock(&top->cp_lock); + LASSERT(ergo(page, page->cp_type == CPT_CACHEABLE)); + RETURN(page); } EXPORT_SYMBOL(cl_vmpage_page); @@ -1154,13 +1158,13 @@ static void cl_page_delete0(const struct lu_env *env, struct cl_page *pg, struct cl_object_header *hdr; hdr = cl_object_header(tmp->cp_obj); - cfs_spin_lock(&hdr->coh_page_guard); - value = radix_tree_delete(&hdr->coh_tree, - tmp->cp_index); - PASSERT(env, tmp, value == tmp); - PASSERT(env, tmp, hdr->coh_pages > 0); - hdr->coh_pages--; - cfs_spin_unlock(&hdr->coh_page_guard); + spin_lock(&hdr->coh_page_guard); + value = radix_tree_delete(&hdr->coh_tree, + tmp->cp_index); + PASSERT(env, tmp, value == tmp); + PASSERT(env, tmp, hdr->coh_pages > 0); + hdr->coh_pages--; + spin_unlock(&hdr->coh_page_guard); } } @@ -1358,13 +1362,6 @@ void cl_page_completion(const struct lu_env *env, pg->cp_sync_io = NULL; cl_sync_io_note(anchor, ioret); } - - /* Don't assert the page writeback bit here because the lustre file - * may be as a backend of swap space. in this case, the page writeback - * is set by VM, and obvious we shouldn't clear it at all. Fortunately - * this type of pages are all TRANSIENT pages. */ - KLASSERT(ergo(pg->cp_type == CPT_CACHEABLE, - !PageWriteback(cl_page_vmpage(env, pg)))); EXIT; } EXPORT_SYMBOL(cl_page_completion);