From c9efb1e343216d857b27538de9c7eb0e1249af19 Mon Sep 17 00:00:00 2001 From: wangdi Date: Mon, 14 May 2012 16:45:07 -0700 Subject: [PATCH] LU-1370 obdecho: few fixes about md_echo clients 1. Remove LU_NOREF flag from echo client ctxt, because it conflicts with cl ctxt(which has no NOREF flag). So if echo_client and normal lustre client share the same context cache, this flag will cause echo module refcount not being dropped. 2. Ma_lmm_size in ma will be changed during mdd_xxx, so we can not use that to free the ma_lmm. 3. Remove lu_device from linkage on lu_site during cleanup phase. Signed-off-by: Di Wang Change-Id: I08fec689ef9c8b64284f7400ab0fc4dc362ebf87 Reviewed-on: http://review.whamcloud.com/2784 Reviewed-by: Jinshan Xiong Tested-by: Hudson Tested-by: Maloo Reviewed-by: Fan Yong Reviewed-by: Oleg Drokin --- lustre/cmm/cmm_device.c | 1 + lustre/include/lu_object.h | 3 + lustre/mdd/mdd_device.c | 1 + lustre/mdt/mdt_handler.c | 8 +-- lustre/obdclass/lu_object.c | 21 ++++++- lustre/obdecho/echo_client.c | 116 ++++++++++++++++++++------------------- lustre/osd-ldiskfs/osd_handler.c | 1 + 7 files changed, 88 insertions(+), 63 deletions(-) diff --git a/lustre/cmm/cmm_device.c b/lustre/cmm/cmm_device.c index f5422b0..1aabc1a 100644 --- a/lustre/cmm/cmm_device.c +++ b/lustre/cmm/cmm_device.c @@ -629,6 +629,7 @@ static int cmm_process_config(const struct lu_env *env, } case LCFG_CLEANUP: { + lu_dev_del_linkage(d->ld_site, d); cmm_device_shutdown(env, m, cfg); } default: diff --git a/lustre/include/lu_object.h b/lustre/include/lu_object.h index 3f626f7..2ae02b3 100644 --- a/lustre/include/lu_object.h +++ b/lustre/include/lu_object.h @@ -676,6 +676,9 @@ void lu_object_fini (struct lu_object *o); void lu_object_add_top (struct lu_object_header *h, struct lu_object *o); void lu_object_add (struct lu_object *before, struct lu_object *o); +void lu_dev_add_linkage(struct lu_site *s, struct lu_device *d); +void lu_dev_del_linkage(struct lu_site *s, struct lu_device *d); + /** * Helpers to initialize and finalize device types. */ diff --git a/lustre/mdd/mdd_device.c b/lustre/mdd/mdd_device.c index 029f6e1..47eab65 100644 --- a/lustre/mdd/mdd_device.c +++ b/lustre/mdd/mdd_device.c @@ -1015,6 +1015,7 @@ static int mdd_process_config(const struct lu_env *env, mdd_changelog_init(env, m); break; case LCFG_CLEANUP: + lu_dev_del_linkage(d->ld_site, d); mdd_device_shutdown(env, m, cfg); default: rc = next->ld_ops->ldo_process_config(env, next, cfg); diff --git a/lustre/mdt/mdt_handler.c b/lustre/mdt/mdt_handler.c index 801ef7e..64fcecf 100644 --- a/lustre/mdt/mdt_handler.c +++ b/lustre/mdt/mdt_handler.c @@ -4301,6 +4301,8 @@ static void mdt_stack_fini(const struct lu_env *env, info = lu_context_key_get(&env->le_ctx, &mdt_thread_key); LASSERT(info != NULL); + lu_dev_del_linkage(top->ld_site, top); + bufs = &info->mti_u.bufs; /* process cleanup, pass mdt obd name to get obd umount flags */ lustre_cfg_bufs_reset(bufs, obd->obd_name); @@ -4314,7 +4316,6 @@ static void mdt_stack_fini(const struct lu_env *env, CERROR("Cannot alloc lcfg!\n"); return; } - LASSERT(top); top->ld_ops->ldo_process_config(env, top, lcfg); lustre_cfg_free(lcfg); @@ -4374,10 +4375,7 @@ static struct lu_device *mdt_layer_setup(struct lu_env *env, lu_device_get(d); lu_ref_add(&d->ld_reference, "lu-stack", &lu_site_init); - cfs_spin_lock(&d->ld_site->ls_ld_lock); - cfs_list_add_tail(&d->ld_linkage, &d->ld_site->ls_ld_linkage); - cfs_spin_unlock(&d->ld_site->ls_ld_lock); - + lu_dev_add_linkage(d->ld_site, d); RETURN(d); out_alloc: ldt->ldt_ops->ldto_device_free(env, d); diff --git a/lustre/obdclass/lu_object.c b/lustre/obdclass/lu_object.c index 9d6746d..507b4f2 100644 --- a/lustre/obdclass/lu_object.c +++ b/lustre/obdclass/lu_object.c @@ -886,6 +886,23 @@ cfs_hash_ops_t lu_site_hash_ops = { .hs_put_locked = lu_obj_hop_put_locked, }; +void lu_dev_add_linkage(struct lu_site *s, struct lu_device *d) +{ + cfs_spin_lock(&s->ls_ld_lock); + if (cfs_list_empty(&d->ld_linkage)) + cfs_list_add(&d->ld_linkage, &s->ls_ld_linkage); + cfs_spin_unlock(&s->ls_ld_lock); +} +EXPORT_SYMBOL(lu_dev_add_linkage); + +void lu_dev_del_linkage(struct lu_site *s, struct lu_device *d) +{ + cfs_spin_lock(&s->ls_ld_lock); + cfs_list_del_init(&d->ld_linkage); + cfs_spin_unlock(&s->ls_ld_lock); +} +EXPORT_SYMBOL(lu_dev_del_linkage); + /** * Initialize site \a s, with \a d as the top level device. */ @@ -964,9 +981,7 @@ int lu_site_init(struct lu_site *s, struct lu_device *top) CFS_INIT_LIST_HEAD(&s->ls_ld_linkage); cfs_spin_lock_init(&s->ls_ld_lock); - cfs_spin_lock(&s->ls_ld_lock); - cfs_list_add(&top->ld_linkage, &s->ls_ld_linkage); - cfs_spin_unlock(&s->ls_ld_lock); + lu_dev_add_linkage(s, top); RETURN(0); } diff --git a/lustre/obdecho/echo_client.c b/lustre/obdecho/echo_client.c index d52f9ea..2c96cf5 100644 --- a/lustre/obdecho/echo_client.c +++ b/lustre/obdecho/echo_client.c @@ -1472,38 +1472,39 @@ out_put: } static int echo_set_lmm_size(const struct lu_env *env, - struct lu_device *ld, - struct md_attr *ma) -{ - struct md_device *md = lu2md_dev(ld); - int lmm_size, cookie_size, rc; - ENTRY; - - md = lu2md_dev(ld); - rc = md->md_ops->mdo_maxsize_get(env, md, - &lmm_size, &cookie_size); - if (rc) - RETURN(rc); - - ma->ma_lmm_size = lmm_size; - if (lmm_size > 0) { - OBD_ALLOC(ma->ma_lmm, lmm_size); - if (ma->ma_lmm == NULL) { - ma->ma_lmm_size = 0; - RETURN(-ENOMEM); - } - } - - ma->ma_cookie_size = cookie_size; - if (cookie_size > 0) { - OBD_ALLOC(ma->ma_cookie, cookie_size); - if (ma->ma_cookie == NULL) { - ma->ma_cookie_size = 0; - RETURN(-ENOMEM); - } - } - - RETURN(0); + struct lu_device *ld, + struct md_attr *ma, int *lmm_size, + int *cookie_size) +{ + struct md_device *md = lu2md_dev(ld); + int rc; + ENTRY; + + md = lu2md_dev(ld); + rc = md->md_ops->mdo_maxsize_get(env, md, + lmm_size, cookie_size); + if (rc) + RETURN(rc); + + ma->ma_lmm_size = *lmm_size; + if (*lmm_size > 0) { + OBD_ALLOC(ma->ma_lmm, *lmm_size); + if (ma->ma_lmm == NULL) { + ma->ma_lmm_size = 0; + RETURN(-ENOMEM); + } + } + + ma->ma_cookie_size = *cookie_size; + if (*cookie_size > 0) { + OBD_ALLOC(ma->ma_cookie, *cookie_size); + if (ma->ma_cookie == NULL) { + ma->ma_cookie_size = 0; + RETURN(-ENOMEM); + } + } + + RETURN(0); } static int echo_create_md_object(const struct lu_env *env, @@ -1521,6 +1522,8 @@ static int echo_create_md_object(const struct lu_env *env, struct md_attr *ma = &info->eti_ma; struct lu_device *ld = ed->ed_next; int rc = 0; + int lmm_size = 0; + int cookie_size = 0; int i; parent = lu_object_locate(ec_parent->lo_header, ld->ld_type); @@ -1533,7 +1536,7 @@ static int echo_create_md_object(const struct lu_env *env, memset(spec, 0, sizeof(*spec)); if (stripe_count != 0) { spec->sp_cr_flags |= FMODE_WRITE; - rc = echo_set_lmm_size(env, ld, ma); + rc = echo_set_lmm_size(env, ld, ma, &lmm_size, &cookie_size); if (rc) GOTO(out_free, rc); if (stripe_count != -1) { @@ -1578,12 +1581,12 @@ static int echo_create_md_object(const struct lu_env *env, } out_free: - if (ma->ma_lmm_size > 0 && ma->ma_lmm != NULL) - OBD_FREE(ma->ma_lmm, ma->ma_lmm_size); - if (ma->ma_cookie_size > 0 && ma->ma_cookie != NULL) - OBD_FREE(ma->ma_cookie, ma->ma_cookie_size); + if (lmm_size > 0 && ma->ma_lmm != NULL) + OBD_FREE(ma->ma_lmm, lmm_size); + if (cookie_size > 0 && ma->ma_cookie != NULL) + OBD_FREE(ma->ma_cookie, cookie_size); - return rc; + return rc; } static struct lu_object *echo_md_lookup(const struct lu_env *env, @@ -1684,6 +1687,8 @@ static int echo_getattr_object(const struct lu_env *env, struct md_attr *ma = &info->eti_ma; struct lu_device *ld = ed->ed_next; int rc = 0; + int lmm_size = 0; + int cookie_size = 0; int i; parent = lu_object_locate(ec_parent->lo_header, ld->ld_type); @@ -1693,9 +1698,9 @@ static int echo_getattr_object(const struct lu_env *env, } memset(ma, 0, sizeof(*ma)); - rc = echo_set_lmm_size(env, ld, ma); - if (rc) - GOTO(out_free, rc); + rc = echo_set_lmm_size(env, ld, ma, &lmm_size, &cookie_size); + if (rc) + GOTO(out_free, rc); ma->ma_need |= MA_INODE | MA_LOV | MA_PFID | MA_HSM | MA_ACL_DEF; ma->ma_acl = info->eti_xattr_buf; @@ -1737,11 +1742,11 @@ static int echo_getattr_object(const struct lu_env *env, } out_free: - if (ma->ma_lmm_size > 0 && ma->ma_lmm != NULL) - OBD_FREE(ma->ma_lmm, ma->ma_lmm_size); - if (ma->ma_cookie_size > 0 && ma->ma_cookie != NULL) - OBD_FREE(ma->ma_cookie, ma->ma_cookie_size); - return rc; + if (lmm_size > 0 && ma->ma_lmm != NULL) + OBD_FREE(ma->ma_lmm, lmm_size); + if (cookie_size > 0 && ma->ma_cookie != NULL) + OBD_FREE(ma->ma_cookie, cookie_size); + return rc; } static int echo_lookup_object(const struct lu_env *env, @@ -1845,6 +1850,8 @@ static int echo_destroy_object(const struct lu_env *env, struct lu_device *ld = ed->ed_next; struct lu_object *parent; int rc = 0; + int lmm_size = 0; + int cookie_size = 0; int i; ENTRY; @@ -1859,7 +1866,7 @@ static int echo_destroy_object(const struct lu_env *env, ma->ma_need = MA_INODE; ma->ma_valid = 0; - rc = echo_set_lmm_size(env, ld, ma); + rc = echo_set_lmm_size(env, ld, ma, &lmm_size, &cookie_size); if (rc) GOTO(out_free, rc); if (name != NULL) { @@ -1888,11 +1895,11 @@ static int echo_destroy_object(const struct lu_env *env, } out_free: - if (ma->ma_lmm_size > 0 && ma->ma_lmm != NULL) - OBD_FREE(ma->ma_lmm, ma->ma_lmm_size); - if (ma->ma_cookie_size > 0 && ma->ma_cookie != NULL) - OBD_FREE(ma->ma_cookie, ma->ma_cookie_size); - RETURN(rc); + if (lmm_size > 0 && ma->ma_lmm != NULL) + OBD_FREE(ma->ma_lmm, lmm_size); + if (cookie_size > 0 && ma->ma_cookie != NULL) + OBD_FREE(ma->ma_cookie, cookie_size); + RETURN(rc); } static struct lu_object *echo_resolve_path(const struct lu_env *env, @@ -1964,9 +1971,8 @@ static struct lu_object *echo_resolve_path(const struct lu_env *env, RETURN(parent); } -#define ECHO_MD_CTX_TAG (LCT_REMEMBER | LCT_NOREF | LCT_MD_THREAD) -#define ECHO_MD_SES_TAG (LCT_SESSION | LCT_REMEMBER | LCT_NOREF) - +#define ECHO_MD_CTX_TAG (LCT_REMEMBER | LCT_MD_THREAD) +#define ECHO_MD_SES_TAG (LCT_REMEMBER | LCT_SESSION) static int echo_md_handler(struct echo_device *ed, int command, char *path, int path_len, int id, int count, struct obd_ioctl_data *data) diff --git a/lustre/osd-ldiskfs/osd_handler.c b/lustre/osd-ldiskfs/osd_handler.c index 0dcc349..991dc11 100644 --- a/lustre/osd-ldiskfs/osd_handler.c +++ b/lustre/osd-ldiskfs/osd_handler.c @@ -4457,6 +4457,7 @@ static int osd_process_config(const struct lu_env *env, err = osd_mount(env, o, cfg); break; case LCFG_CLEANUP: + lu_dev_del_linkage(d->ld_site, d); err = osd_shutdown(env, o); break; default: -- 1.8.3.1