From 1d66195fc11d3ed8117562520c09974965b40f8f Mon Sep 17 00:00:00 2001 From: zhanghc Date: Thu, 20 Aug 2009 15:53:51 +0000 Subject: [PATCH] b=19069 pass (re)connect flag to lprocfs_exp_setup to optimize the implmentation of nid stats in export (obd_export->exp_nid_stats) i=andrew.perepechko@sun.com i=hongchao.zhang@sun.com --- lustre/include/lprocfs_status.h | 6 +++--- lustre/mds/handler.c | 2 +- lustre/mds/mds_fs.c | 5 +++-- lustre/mds/mds_internal.h | 1 + lustre/mgs/mgs_fs.c | 7 ++++--- lustre/mgs/mgs_handler.c | 5 ++--- lustre/mgs/mgs_internal.h | 5 ++--- lustre/obdclass/lprocfs_status.c | 29 ++++++++++++----------------- lustre/obdfilter/filter.c | 10 ++++++---- 9 files changed, 34 insertions(+), 36 deletions(-) diff --git a/lustre/include/lprocfs_status.h b/lustre/include/lprocfs_status.h index cb1dab5..0c957e1 100644 --- a/lustre/include/lprocfs_status.h +++ b/lustre/include/lprocfs_status.h @@ -396,7 +396,7 @@ struct obd_export; extern int lprocfs_add_clear_entry(struct obd_device * obd, cfs_proc_dir_entry_t *entry); extern int lprocfs_exp_setup(struct obd_export *exp, - lnet_nid_t *peer_nid, int *newnid); + lnet_nid_t *peer_nid, int reconnect, int *newnid); extern int lprocfs_exp_cleanup(struct obd_export *exp); extern cfs_proc_dir_entry_t *lprocfs_add_simple(struct proc_dir_entry *root, char *name, @@ -774,8 +774,8 @@ static inline void lprocfs_free_obd_stats(struct obd_device *obddev) struct obd_export; static inline int lprocfs_add_clear_entry(struct obd_export *exp) { return 0; } -static inline int lprocfs_exp_setup(struct obd_export *exp, - lnet_nid_t *peer_nid, int *newnid) +static inline int lprocfs_exp_setup(struct obd_export *exp, lnet_nid_t *peer_nid, + int reconnect, int *newnid) { return 0; } static inline int lprocfs_exp_cleanup(struct obd_export *exp) { return 0; } diff --git a/lustre/mds/handler.c b/lustre/mds/handler.c index b024261..268f275 100644 --- a/lustre/mds/handler.c +++ b/lustre/mds/handler.c @@ -319,7 +319,7 @@ static int mds_reconnect(struct obd_export *exp, struct obd_device *obd, rc = mds_connect_internal(exp, data); if (rc == 0) - mds_export_stats_init(obd, exp, localdata); + mds_export_stats_init(obd, exp, 1, localdata); RETURN(rc); } diff --git a/lustre/mds/mds_fs.c b/lustre/mds/mds_fs.c index 97e6c38..d8ce0d0 100644 --- a/lustre/mds/mds_fs.c +++ b/lustre/mds/mds_fs.c @@ -64,12 +64,13 @@ int mds_export_stats_init(struct obd_device *obd, struct obd_export *exp, + int reconnect, void *localdata) { lnet_nid_t *client_nid = localdata; int rc, num_stats, newnid = 0; - rc = lprocfs_exp_setup(exp, client_nid, &newnid); + rc = lprocfs_exp_setup(exp, client_nid, reconnect, &newnid); if (rc) { /* Mask error for already created * /proc entries */ @@ -228,7 +229,7 @@ int mds_client_add(struct obd_device *obd, struct obd_export *exp, med->med_lr_off = le32_to_cpu(mds->mds_server_data->lsd_client_start) + (cl_idx * le16_to_cpu(mds->mds_server_data->lsd_client_size)); LASSERTF(med->med_lr_off > 0, "med_lr_off = %llu\n", med->med_lr_off); - mds_export_stats_init(obd, exp, localdata); + mds_export_stats_init(obd, exp, 0, localdata); if (new_client) { struct lvfs_run_ctxt *saved = NULL; diff --git a/lustre/mds/mds_internal.h b/lustre/mds/mds_internal.h index 2a60940..15da505 100644 --- a/lustre/mds/mds_internal.h +++ b/lustre/mds/mds_internal.h @@ -267,6 +267,7 @@ int mds_update_client_epoch(struct obd_export *exp); void mds_update_last_epoch(struct obd_device *obd); int mds_export_stats_init(struct obd_device *obd, struct obd_export *exp, + int reconnect, void *client_nid); int mds_client_add(struct obd_device *obd, struct obd_export *exp, int cl_off, void *localdata); diff --git a/lustre/mgs/mgs_fs.c b/lustre/mgs/mgs_fs.c index cc6b3f0..5a8aea4 100644 --- a/lustre/mgs/mgs_fs.c +++ b/lustre/mgs/mgs_fs.c @@ -59,14 +59,15 @@ #include "mgs_internal.h" -static int mgs_export_stats_init(struct obd_device *obd, +int mgs_export_stats_init(struct obd_device *obd, struct obd_export *exp, + int reconnect, void *localdata) { lnet_nid_t *client_nid = localdata; int rc, num_stats, newnid = 0; - rc = lprocfs_exp_setup(exp, client_nid, &newnid); + rc = lprocfs_exp_setup(exp, client_nid, reconnect, &newnid); if (rc) { /* Mask error for already created * /proc entries */ @@ -112,7 +113,7 @@ int mgs_client_add(struct obd_device *obd, struct obd_export *exp, void *localdata) { - return mgs_export_stats_init(obd, exp, localdata); + return 0; } /* Remove client export data from the MGS */ diff --git a/lustre/mgs/mgs_handler.c b/lustre/mgs/mgs_handler.c index fe47cac..9d41f09 100644 --- a/lustre/mgs/mgs_handler.c +++ b/lustre/mgs/mgs_handler.c @@ -87,8 +87,7 @@ static int mgs_connect(struct lustre_handle *conn, struct obd_device *obd, data->ocd_version = LUSTRE_VERSION_CODE; } - rc = mgs_client_add(obd, exp, localdata); - + rc = mgs_export_stats_init(obd, exp, 0, localdata); if (rc) { class_disconnect(exp); lprocfs_exp_cleanup(exp); @@ -116,7 +115,7 @@ static int mgs_reconnect(struct obd_export *exp, struct obd_device *obd, data->ocd_version = LUSTRE_VERSION_CODE; } - RETURN(0); + RETURN(mgs_export_stats_init(obd, exp, 1, localdata)); } static int mgs_disconnect(struct obd_export *exp) diff --git a/lustre/mgs/mgs_internal.h b/lustre/mgs/mgs_internal.h index 7594bf9..2bb77fd 100644 --- a/lustre/mgs/mgs_internal.h +++ b/lustre/mgs/mgs_internal.h @@ -86,9 +86,8 @@ int mgs_pool_cmd(struct obd_device *obd, enum lcfg_command_type cmd, char *poolname, char *fsname, char *ostname); /* mgs_fs.c */ -int mgs_client_add(struct obd_device *obd, - struct obd_export *exp, - void *localdata); +int mgs_export_stats_init(struct obd_device *obd, struct obd_export *exp, + int reconnect, void *localdata); int mgs_client_free(struct obd_export *exp); int mgs_fs_setup(struct obd_device *obd, struct vfsmount *mnt); int mgs_fs_cleanup(struct obd_device *obddev); diff --git a/lustre/obdclass/lprocfs_status.c b/lustre/obdclass/lprocfs_status.c index bb9ebb7..20d010d 100644 --- a/lustre/obdclass/lprocfs_status.c +++ b/lustre/obdclass/lprocfs_status.c @@ -1656,7 +1656,8 @@ int lprocfs_nid_stats_clear_write(struct file *file, const char *buffer, } EXPORT_SYMBOL(lprocfs_nid_stats_clear_write); -int lprocfs_exp_setup(struct obd_export *exp, lnet_nid_t *nid, int *newnid) +int lprocfs_exp_setup(struct obd_export *exp, lnet_nid_t *nid, int reconnect, + int *newnid) { int rc = 0; struct nid_stat *new_stat, *old_stat; @@ -1686,7 +1687,8 @@ int lprocfs_exp_setup(struct obd_export *exp, lnet_nid_t *nid, int *newnid) new_stat->nid = *nid; new_stat->nid_obd = exp->exp_obd; - atomic_set(&new_stat->nid_exp_ref_count, 0); + /* we need set default refcount to 1 to balance obd_disconnect() */ + atomic_set(&new_stat->nid_exp_ref_count, 1); old_stat = lustre_hash_findadd_unique(obd->obd_nid_stats_hash, nid, &new_stat->nid_hash); @@ -1697,21 +1699,16 @@ int lprocfs_exp_setup(struct obd_export *exp, lnet_nid_t *nid, int *newnid) /* Return -EALREADY here so that we know that the /proc * entry already has been created */ if (old_stat != new_stat) { - spin_lock(&obd->obd_nid_lock); - if (exp->exp_nid_stats != old_stat) { - if (exp->exp_nid_stats) - nidstat_putref(exp->exp_nid_stats); - exp->exp_nid_stats = old_stat; - } else { - /* lustre_hash_findadd_unique() has added - * old_stat's refcount */ - nidstat_putref(old_stat); - } - - spin_unlock(&obd->obd_nid_lock); + /* if this reconnect to live export from diffrent nid, we need + * to release old stats because disconnect will be never called. + * */ + if (reconnect && exp->exp_nid_stats) + nidstat_putref(exp->exp_nid_stats); + exp->exp_nid_stats = old_stat; GOTO(destroy_new, rc = -EALREADY); } + /* not found - create */ new_stat->nid_proc = proc_mkdir(libcfs_nid2str(*nid), obd->obd_proc_exports_entry); @@ -1737,9 +1734,6 @@ int lprocfs_exp_setup(struct obd_export *exp, lnet_nid_t *nid, int *newnid) GOTO(destroy_new_ns, rc); } - if (exp->exp_nid_stats) - nidstat_putref(exp->exp_nid_stats); - nidstat_getref(new_stat); exp->exp_nid_stats = new_stat; *newnid = 1; /* protect competitive add to list, not need locking on destroy */ @@ -1755,6 +1749,7 @@ destroy_new_ns: lustre_hash_del(obd->obd_nid_stats_hash, nid, &new_stat->nid_hash); destroy_new: + nidstat_putref(new_stat); OBD_FREE_PTR(new_stat); RETURN(rc); } diff --git a/lustre/obdfilter/filter.c b/lustre/obdfilter/filter.c index eb6f1fb..9cdcb40 100644 --- a/lustre/obdfilter/filter.c +++ b/lustre/obdfilter/filter.c @@ -233,6 +233,7 @@ static int lprocfs_init_rw_stats(struct obd_device *obd, plus the procfs overhead :( */ static int filter_export_stats_init(struct obd_device *obd, struct obd_export *exp, + int reconnect, void *client_nid) { struct proc_dir_entry *brw_entry; @@ -242,7 +243,8 @@ static int filter_export_stats_init(struct obd_device *obd, if (obd_uuid_equals(&exp->exp_client_uuid, &obd->obd_uuid)) /* Self-export gets no proc entry */ RETURN(0); - rc = lprocfs_exp_setup(exp, (lnet_nid_t *)client_nid, &newnid); + rc = lprocfs_exp_setup(exp, (lnet_nid_t *)client_nid, + reconnect, &newnid); if (rc) { /* Mask error for already created * /proc entries */ @@ -947,7 +949,7 @@ static int filter_init_server_data(struct obd_device *obd, struct file * filp) } else { fed = &exp->exp_filter_data; fed->fed_lcd = lcd; - filter_export_stats_init(obd, exp, NULL); + filter_export_stats_init(obd, exp, 0, NULL); rc = filter_client_add(obd, exp, cl_idx); /* can't fail for existing client */ LASSERTF(rc == 0, "rc = %d\n", rc); @@ -2399,7 +2401,7 @@ static int filter_reconnect(struct obd_export *exp, struct obd_device *obd, rc = filter_connect_internal(exp, data); if (rc == 0) - filter_export_stats_init(obd, exp, localdata); + filter_export_stats_init(obd, exp, 1, localdata); RETURN(rc); } @@ -2431,7 +2433,7 @@ static int filter_connect(struct lustre_handle *conn, struct obd_device *obd, if (rc) GOTO(cleanup, rc); - filter_export_stats_init(obd, exp, localdata); + filter_export_stats_init(obd, exp, 0, localdata); if (!obd->obd_replayable) GOTO(cleanup, rc = 0); -- 1.8.3.1