From e6f8938e434c2c7d3e398766e5f1d153a23031eb Mon Sep 17 00:00:00 2001 From: wangdi Date: Tue, 27 Nov 2007 23:01:06 +0000 Subject: [PATCH] Branch: b1_6 b=14016 Fix BUGON when do per-client proc cleanup. Patch from shadow i=Andreas,Nathan --- lustre/include/class_hash.h | 20 +++- lustre/include/lprocfs_status.h | 17 +++- lustre/include/lustre_export.h | 12 +-- lustre/include/obd.h | 8 +- lustre/mds/handler.c | 42 +------- lustre/obdclass/class_hash.c | 195 +++++++++++++++++++++++++++--------- lustre/obdclass/lprocfs_status.c | 210 +++++++++++++++++++++++++-------------- lustre/obdclass/obd_config.c | 15 ++- lustre/obdfilter/filter.c | 47 ++------- 9 files changed, 350 insertions(+), 216 deletions(-) diff --git a/lustre/include/class_hash.h b/lustre/include/class_hash.h index efc8144..1be8c52 100644 --- a/lustre/include/class_hash.h +++ b/lustre/include/class_hash.h @@ -7,6 +7,8 @@ #include +/* #define LUSTRE_HASH_DEBUG 1 */ + /* define the hash bucket*/ struct lustre_hash_bucket { struct hlist_head lhb_head; @@ -16,7 +18,7 @@ struct lustre_hash_bucket { * it will help us to analyse the hash distribute */ int lhb_item_count; -#endif +#endif }; struct lustre_hash_operations; @@ -81,12 +83,16 @@ lustre_hash_delitem_nolock(struct lustre_class_hash_body *hash_body, RETURN(0); } +typedef void (*hash_item_iterate_cb) (void *obj, void *data); + int lustre_hash_init(struct lustre_class_hash_body **hash_body, char *hashname, __u32 hashsize, struct lustre_hash_operations *hash_operations); void lustre_hash_exit(struct lustre_class_hash_body **hash_body); int lustre_hash_additem_unique(struct lustre_class_hash_body *hash_body, void *key, struct hlist_node *actual_hnode); +void *lustre_hash_findadd_unique(struct lustre_class_hash_body *hash_body, + void *key, struct hlist_node *actual_hnode); int lustre_hash_additem(struct lustre_class_hash_body *hash_body, void *key, struct hlist_node *actual_hnode); int lustre_hash_delitem_by_key(struct lustre_class_hash_body *hash_body, @@ -96,9 +102,15 @@ int lustre_hash_delitem(struct lustre_class_hash_body *hash_body, void *key, void lustre_hash_bucket_iterate(struct lustre_class_hash_body *hash_body, void *key, hash_item_iterate_cb, void *data); +void lustre_hash_iterate_all(struct lustre_class_hash_body *hash_body, + hash_item_iterate_cb, void *data); + void * lustre_hash_get_object_by_key(struct lustre_class_hash_body *hash_body, void *key); +__u32 djb2_hashfn(struct lustre_class_hash_body *hash_body, void* key, + size_t size); + /* ( uuid <-> export ) hash operations define */ __u32 uuid_hashfn(struct lustre_class_hash_body *hash_body, void * key); int uuid_hash_key_compare(void *key, struct hlist_node * compared_hnode); @@ -117,4 +129,10 @@ int conn_hash_key_compare(void *key, struct hlist_node * compared_hnode); void * conn_refcount_get(struct hlist_node * actual_hnode); void conn_refcount_put(struct hlist_node * actual_hnode); +/* ( nid <-> nidstats ) hash operations define. uses nid_hashfn */ +int nidstats_hash_key_compare(void *key, struct hlist_node * compared_hnode); +void* nidstats_refcount_get(struct hlist_node * actual_hnode); +void nidstats_refcount_put(struct hlist_node * actual_hnode); +extern struct lustre_hash_operations nid_stat_hash_operations; + #endif /* __CLASS_HASH_H */ diff --git a/lustre/include/lprocfs_status.h b/lustre/include/lprocfs_status.h index 31b7380..9bfe431 100644 --- a/lustre/include/lprocfs_status.h +++ b/lustre/include/lprocfs_status.h @@ -354,8 +354,13 @@ extern int lprocfs_add_simple(struct proc_dir_entry *root, char *name, read_proc_t *read_proc, write_proc_t *write_proc, void *data); struct nid_stat; -extern void lprocfs_free_client_stats(struct nid_stat *stat); extern void lprocfs_free_per_client_stats(struct obd_device *obd); +extern int lprocfs_nid_stats_clear_write(struct file *file, const char *buffer, + unsigned long count, void *data); +extern int lprocfs_nid_stats_clear_read(char *page, char **start, off_t off, + int count, int *eof, void *data); + + extern struct file_operations lprocfs_evict_client_fops; extern int lprocfs_seq_create(cfs_proc_dir_entry_t *parent, char *name, @@ -562,9 +567,17 @@ static inline int lprocfs_add_simple(struct proc_dir_entry *root, void *data) {return 0; } struct nid_stat; -static inline void lprocfs_free_client_stats(struct nid_stat *stat){} static inline void lprocfs_free_per_client_stats(struct obd_device *obd) {} +static inline +int lprocfs_nid_stats_clear_write(struct file *file, const char *buffer, + unsigned long count, void *data) +{return count;} +static inline +int lprocfs_nid_stats_clear_read(char *page, char **start, off_t off, + int count, int *eof, void *data) +{return count;} + static inline cfs_proc_dir_entry_t * lprocfs_register(const char *name, cfs_proc_dir_entry_t *parent, diff --git a/lustre/include/lustre_export.h b/lustre/include/lustre_export.h index df68b2d..dab7bad 100644 --- a/lustre/include/lustre_export.h +++ b/lustre/include/lustre_export.h @@ -59,7 +59,8 @@ struct filter_export_data { typedef struct nid_stat { lnet_nid_t nid; - struct list_head nid_chain; + struct hlist_node nid_hash; + struct list_head nid_list; struct obd_device *nid_obd; struct proc_dir_entry *nid_proc; struct lprocfs_stats *nid_stats; @@ -128,16 +129,7 @@ static inline int imp_connect_lru_resize(struct obd_import *imp) return !!(ocd->ocd_connect_flags & OBD_CONNECT_LRU_RESIZE); } -struct exp_uuid_cb_data { - char *page; - int count; - int *eof; - int *len; - struct obd_export *exp; -}; - extern struct obd_export *class_conn2export(struct lustre_handle *conn); extern struct obd_device *class_conn2obd(struct lustre_handle *conn); -typedef void (*hash_item_iterate_cb) (void *); #endif /* __EXPORT_H */ diff --git a/lustre/include/obd.h b/lustre/include/obd.h index d58905e..b9cc606 100644 --- a/lustre/include/obd.h +++ b/lustre/include/obd.h @@ -731,13 +731,15 @@ struct obd_device { /* uuid-export hash body */ struct lustre_class_hash_body *obd_uuid_hash_body; /* nid-export hash body */ - struct lustre_class_hash_body *obd_nid_hash_body; - atomic_t obd_refcount; + struct lustre_class_hash_body *obd_nid_hash_body; + /* nid stats body */ + struct lustre_class_hash_body *obd_nid_stats_hash_body; + struct list_head obd_nid_stats; + atomic_t obd_refcount; cfs_waitq_t obd_refcount_waitq; cfs_waitq_t obd_llog_waitq; struct list_head obd_exports; int obd_num_exports; - struct list_head obd_proc_nid_list; spinlock_t nid_lock; struct ldlm_namespace *obd_namespace; struct ptlrpc_client obd_ldlm_client; /* XXX OST/MDS only */ diff --git a/lustre/mds/handler.c b/lustre/mds/handler.c index fba1806..e863c39 100644 --- a/lustre/mds/handler.c +++ b/lustre/mds/handler.c @@ -1896,38 +1896,6 @@ static void fsoptions_to_mds_flags(struct mds_obd *mds, char *options) } } -static int mds_nid_stats_clear_read(char *page, char **start, off_t off, - int count, int *eof, void *data) -{ - *eof = 1; - return snprintf(page, count, "%s\n", - "Write into this file to clear all nid stats and " - "stale nid entries"); -} - -static int mds_nid_stats_clear_write(struct file *file, const char *buffer, - unsigned long count, void *data) -{ - struct obd_device *obd = (struct obd_device *)data; - struct list_head *nids= &obd->obd_proc_nid_list; - nid_stat_t *client_stat = NULL, *nxt; - - spin_lock(&obd->nid_lock); - - list_for_each_entry_safe (client_stat, nxt, nids, nid_chain) { - if (!client_stat->nid_exp_ref_count) - lprocfs_free_client_stats(client_stat); - else if (client_stat->nid_stats) { - lprocfs_clear_stats(client_stat->nid_stats); - } - } - - spin_unlock(&obd->nid_lock); - - return count; -} - - /* mount the file system (secretly). lustre_cfg parameters are: * 1 = device * 2 = fstype @@ -2014,8 +1982,8 @@ static int mds_setup(struct obd_device *obd, obd_count len, void *buf) if (obd->obd_proc_exports_entry) lprocfs_add_simple(obd->obd_proc_exports_entry, - "clear", mds_nid_stats_clear_read, - mds_nid_stats_clear_write, obd); + "clear", lprocfs_nid_stats_clear_read, + lprocfs_nid_stats_clear_write, obd); if (lcfg->lcfg_bufcount >= 4 && LUSTRE_CFG_BUFLEN(lcfg, 3) > 0) { class_uuid_t uuid; @@ -2099,8 +2067,8 @@ err_fs: upcall_cache_cleanup(mds->mds_group_hash); mds->mds_group_hash = NULL; err_ns: - lprocfs_obd_cleanup(obd); lprocfs_free_obd_stats(obd); + lprocfs_obd_cleanup(obd); ldlm_namespace_free(obd->obd_namespace, 0); obd->obd_namespace = NULL; err_ops: @@ -2282,10 +2250,10 @@ static int mds_cleanup(struct obd_device *obd) we just need to drop our ref */ class_export_put(mds->mds_osc_exp); - lprocfs_free_per_client_stats(obd); remove_proc_entry("clear", obd->obd_proc_exports_entry); - lprocfs_obd_cleanup(obd); + lprocfs_free_per_client_stats(obd); lprocfs_free_obd_stats(obd); + lprocfs_obd_cleanup(obd); lquota_cleanup(mds_quota_interface_ref, obd); diff --git a/lustre/obdclass/class_hash.c b/lustre/obdclass/class_hash.c index e7aeedd..fd725aa 100644 --- a/lustre/obdclass/class_hash.c +++ b/lustre/obdclass/class_hash.c @@ -39,7 +39,7 @@ int lustre_hash_init(struct lustre_class_hash_body **hash_body_new, n++; i >>= 1; } - + LASSERTF(n == 1, "hashsize %u isn't 2^n\n", hashsize); /* alloc space for hash_body */ @@ -59,7 +59,7 @@ int lustre_hash_init(struct lustre_class_hash_body **hash_body_new, /* alloc space for the hash tables */ OBD_ALLOC(hash_body->lchb_hash_tables, - sizeof(*hash_body->lchb_hash_tables) * hash_body->lchb_hash_max_size); + sizeof(*hash_body->lchb_hash_tables) * hash_body->lchb_hash_max_size); if (hash_body->lchb_hash_tables == NULL) { OBD_FREE(hash_body, sizeof(*hash_body)); @@ -178,6 +178,53 @@ int lustre_hash_additem_unique(struct lustre_class_hash_body *hash_body, EXPORT_SYMBOL(lustre_hash_additem_unique); /* + * only allow unique @key in hashtables, if the same @key has existed + * in hashtables, it will return with fails. + */ +void* lustre_hash_findadd_unique(struct lustre_class_hash_body *hash_body, + void *key, struct hlist_node *actual_hnode) +{ + int hashent; + struct lustre_hash_bucket *bucket = NULL; + struct lustre_hash_operations *hop = hash_body->lchb_hash_operations; + struct hlist_node * hash_item_hnode = NULL; + void *obj; + ENTRY; + + LASSERT(hlist_unhashed(actual_hnode)); + hashent = hop->lustre_hashfn(hash_body, key); + + /* get the hash-bucket and lock it */ + bucket = &hash_body->lchb_hash_tables[hashent]; + spin_lock(&bucket->lhb_lock); + + hash_item_hnode = lustre_hash_getitem_in_bucket_nolock(hash_body, + hashent, key); + if ( hash_item_hnode != NULL) { + /* the added-item exist in hashtables, so cannot add it again */ + obj = hop->lustre_hash_object_refcount_get(hash_item_hnode); + spin_unlock(&bucket->lhb_lock); + RETURN(obj); + } + + hlist_add_head(actual_hnode, &(bucket->lhb_head)); + +#ifdef LUSTRE_HASH_DEBUG + /* hash distribute debug */ + hash_body->lchb_hash_tables[hashent].lhb_item_count++; + CDEBUG(D_INFO, "hashname[%s] bucket[%d] has [%d] hashitem\n", + hash_body->hashname, hashent, + hash_body->lchb_hash_tables[hashent].lhb_item_count); +#endif + obj = hop->lustre_hash_object_refcount_get(actual_hnode); + + spin_unlock(&bucket->lhb_lock); + + RETURN(obj); +} +EXPORT_SYMBOL(lustre_hash_findadd_unique); + +/* * this version of additem, it allow multi same @key in hashtables. * in this additem version, we don't need to check if exist same @key in hash * tables, we only add it to related hashbucket. @@ -304,10 +351,8 @@ void lustre_hash_bucket_iterate(struct lustre_class_hash_body *hash_body, hlist_for_each(hash_item_node, &(bucket->lhb_head)) { find = hop->lustre_hash_key_compare(key, hash_item_node); if (find) { - tmp = (struct obd_export *)hop->lustre_hash_object_refcount_get( - hash_item_node); - ((struct exp_uuid_cb_data *)data)->exp = tmp; - func(data); + tmp = hop->lustre_hash_object_refcount_get(hash_item_node); + func(tmp, data); hop->lustre_hash_object_refcount_put(hash_item_node); } } @@ -315,6 +360,35 @@ void lustre_hash_bucket_iterate(struct lustre_class_hash_body *hash_body, } EXPORT_SYMBOL(lustre_hash_bucket_iterate); +void lustre_hash_iterate_all(struct lustre_class_hash_body *hash_body, + hash_item_iterate_cb func, void *data) +{ + int i; + struct lustre_hash_operations *hop = hash_body->lchb_hash_operations; + ENTRY; + + for( i = 0; i < hash_body->lchb_hash_max_size; i++ ) { + struct lustre_hash_bucket * bucket; + struct hlist_node * actual_hnode, *pos; + void *obj; + + bucket = &hash_body->lchb_hash_tables[i]; +#ifdef LUSTRE_HASH_DEBUG + CDEBUG(D_INFO, "idx %d - bucket %p\n", i, bucket); +#endif + spin_lock(&bucket->lhb_lock); /* lock the bucket */ + hlist_for_each_safe(actual_hnode, pos, &(bucket->lhb_head)) { + obj = hop->lustre_hash_object_refcount_get(actual_hnode); + func(obj, data); + hop->lustre_hash_object_refcount_put(actual_hnode); + } + spin_unlock(&bucket->lhb_lock); + } + EXIT; +} +EXPORT_SYMBOL(lustre_hash_iterate_all); + + void * lustre_hash_get_object_by_key(struct lustre_class_hash_body *hash_body, void *key) { @@ -346,6 +420,24 @@ void * lustre_hash_get_object_by_key(struct lustre_class_hash_body *hash_body, } EXPORT_SYMBOL(lustre_hash_get_object_by_key); +/* string hashing using djb2 hash algorithm */ +__u32 djb2_hashfn(struct lustre_class_hash_body *hash_body, void* key, + size_t size) +{ + __u32 hash = 5381; + int i; + char *ptr = key; + + LASSERT(key != NULL); + + for (i=0; ilchb_hash_max_size - 1); + + RETURN(hash); +} + /* * define (uuid <-> export) hash operations and function define */ @@ -358,26 +450,11 @@ struct lustre_hash_operations uuid_hash_operations = { .lustre_hash_object_refcount_put = uuid_export_refcount_put, }; -/* string hashing using djb2 hash algorithm */ __u32 uuid_hashfn(struct lustre_class_hash_body *hash_body, void * key) { - __u32 hash = 5381; - struct obd_uuid * uuid_key = NULL; - int c; - char *ptr = NULL; - - LASSERT(key != NULL); - - uuid_key = (struct obd_uuid*)key; - ptr = uuid_key->uuid; + struct obd_uuid * uuid_key = key; - while ((c = *ptr++)) { - hash = hash * 33 + c; - } - - hash &= (hash_body->lchb_hash_max_size - 1); - - RETURN(hash); + return djb2_hashfn(hash_body, uuid_key->uuid, sizeof(uuid_key->uuid)); } /* Note, it is impossible to find an export that is in failed state with @@ -439,21 +516,9 @@ struct lustre_hash_operations nid_hash_operations = { .lustre_hash_object_refcount_put = nid_export_refcount_put, }; -/* string hashing using djb2 hash algorithm */ __u32 nid_hashfn(struct lustre_class_hash_body *hash_body, void * key) { - __u32 hash = 5381; - int i; - char *ptr = key; - - LASSERT(key != NULL); - - for(i = 0 ; i < sizeof(lnet_nid_t) ; i ++) - hash = hash * 33 + ptr[i]; - - hash &= (hash_body->lchb_hash_max_size - 1); - - RETURN(hash); + return djb2_hashfn(hash_body, key, sizeof(lnet_nid_t)); } /* Note, it is impossible to find an export that is in failed state with @@ -514,21 +579,9 @@ struct lustre_hash_operations conn_hash_operations = { }; EXPORT_SYMBOL(conn_hash_operations); -/* string hashing using djb2 hash algorithm */ __u32 conn_hashfn(struct lustre_class_hash_body *hash_body, void * key) { - __u32 hash = 5381; - char *ptr = key; - int i; - - LASSERT(key != NULL); - - for(i = 0 ; i < sizeof(lnet_process_id_t) ; i ++) - hash = hash * 33 + ptr[i]; - - hash &= (hash_body->lchb_hash_max_size - 1); - - RETURN(hash); + return djb2_hashfn(hash_body, key, sizeof(lnet_process_id_t)); } int conn_hash_key_compare(void *key, struct hlist_node *compared_hnode) @@ -574,3 +627,47 @@ void conn_refcount_put(struct hlist_node *actual_hnode) atomic_dec(&c->c_refcount); } +/*******************************************************************************/ +/* ( nid<>nidstats ) hash operations define */ + +struct lustre_hash_operations nid_stat_hash_operations = { + .lustre_hashfn = nid_hashfn, + .lustre_hash_key_compare = nidstats_hash_key_compare, + .lustre_hash_object_refcount_get = nidstats_refcount_get, + .lustre_hash_object_refcount_put = nidstats_refcount_put, +}; +EXPORT_SYMBOL(nid_stat_hash_operations); + +int nidstats_hash_key_compare(void *key, struct hlist_node * compared_hnode) +{ + struct nid_stat *data; + lnet_nid_t *nid_key; + + LASSERT( key != NULL); + + nid_key = (lnet_nid_t*)key; + data = hlist_entry(compared_hnode, struct nid_stat, nid_hash); + + return (data->nid == *nid_key); +} + +void* nidstats_refcount_get(struct hlist_node * actual_hnode) +{ + struct nid_stat *data; + + data = hlist_entry(actual_hnode, struct nid_stat, nid_hash); + data->nid_exp_ref_count++; + + RETURN(data); +} + +void nidstats_refcount_put(struct hlist_node * actual_hnode) +{ + struct nid_stat *data; + + data = hlist_entry(actual_hnode, struct nid_stat, nid_hash); + data->nid_exp_ref_count--; + +} + +/*******************************************************************************/ diff --git a/lustre/obdclass/lprocfs_status.c b/lustre/obdclass/lprocfs_status.c index 2e529da..7cb6e99 100644 --- a/lustre/obdclass/lprocfs_status.c +++ b/lustre/obdclass/lprocfs_status.c @@ -288,13 +288,13 @@ void lprocfs_remove(struct proc_dir_entry **rooth) struct proc_dir_entry *rm_entry; struct proc_dir_entry *parent; - if (!root) + if (!root) return; *rooth = NULL; parent = root->parent; LASSERT(parent != NULL); - + while (1) { while (temp->subdir != NULL) temp = temp->subdir; @@ -312,7 +312,7 @@ void lprocfs_remove(struct proc_dir_entry **rooth) * by _lprocfs_lock. */ down_write(&_lprocfs_lock); rm_entry->data = NULL; - remove_proc_entry(rm_entry->name, rm_entry->parent); + remove_proc_entry(rm_entry->name, temp); up_write(&_lprocfs_lock); if (temp == parent) break; @@ -739,11 +739,19 @@ int lprocfs_obd_cleanup(struct obd_device *obd) return 0; } -void lprocfs_free_client_stats(nid_stat_t *client_stat) +void lprocfs_free_client_stats(void *obj, void *data) { - LASSERT(client_stat->nid_exp_ref_count == 0); + struct nid_stat *client_stat = obj; + + CDEBUG(D_CONFIG, "stat %p - data %p/%p/%p\n", client_stat, + client_stat->nid_proc, client_stat->nid_stats, + client_stat->nid_brw_stats); - list_del(&client_stat->nid_chain); + LASSERTF(client_stat->nid_exp_ref_count == 0, "count %d\n", + client_stat->nid_exp_ref_count); + + hlist_del_init(&client_stat->nid_hash); + list_del(&client_stat->nid_list); if (client_stat->nid_proc) lprocfs_remove(&client_stat->nid_proc); @@ -754,26 +762,24 @@ void lprocfs_free_client_stats(nid_stat_t *client_stat) if (client_stat->nid_brw_stats) OBD_FREE(client_stat->nid_brw_stats, sizeof(struct brw_stats)); - OBD_FREE(client_stat, sizeof(struct nid_stat)); + OBD_FREE(client_stat, sizeof(*client_stat)); return; } void lprocfs_free_per_client_stats(struct obd_device *obd) { - - struct list_head *nids= &obd->obd_proc_nid_list; - nid_stat_t *client_stat = NULL, *nxt; + struct nid_stat *stat; ENTRY; - spin_lock(&obd->nid_lock); - - list_for_each_entry_safe (client_stat, nxt, nids, nid_chain) - lprocfs_free_client_stats(client_stat); - - spin_unlock(&obd->nid_lock); - return; + /* we need extra list - because hash_exit called to early */ + while(!list_empty(&obd->obd_nid_stats)) { + stat = list_entry(obd->obd_nid_stats.next, + struct nid_stat, nid_list); + lprocfs_free_client_stats(stat, NULL); + } + EXIT; } struct lprocfs_stats *lprocfs_alloc_stats(unsigned int num, @@ -851,7 +857,7 @@ void lprocfs_clear_stats(struct lprocfs_stats *stats) num_cpu = lprocfs_stats_lock(stats, LPROCFS_GET_NUM_CPU); for (i = 0; i < num_cpu; i++) { - for (j = 0; j < stats->ls_num; j++) { + for (j = 0; j < stats->ls_num; j++) { percpu_cntr = &(stats->ls_percpu[i])->lp_cntr[j]; atomic_inc(&percpu_cntr->lc_cntl.la_entry); percpu_cntr->lc_count = 0; @@ -1169,14 +1175,22 @@ int lprocfs_exp_rd_nid(char *page, char **start, off_t off, int count, return snprintf(page, count, "%s\n", obd_export_nid2str(exp)); } -void lprocfs_exp_print_uuid(void *cb_data) +struct exp_uuid_cb_data { + char *page; + int count; + int *eof; + int *len; +}; + +void lprocfs_exp_print_uuid(void *obj, void *cb_data) { + struct obd_export *exp = (struct obd_export *)obj; struct exp_uuid_cb_data *data = (struct exp_uuid_cb_data *)cb_data; - if (data->exp->exp_nid_stats) + if (exp->exp_nid_stats) *data->len += snprintf((data->page + *data->len), data->count, "%s\n", - obd_uuid2str(&data->exp->exp_client_uuid)); + obd_uuid2str(&exp->exp_client_uuid)); } int lprocfs_exp_rd_uuid(char *page, char **start, off_t off, int count, @@ -1195,84 +1209,137 @@ int lprocfs_exp_rd_uuid(char *page, char **start, off_t off, int count, cb_data.count = count; cb_data.eof = eof; cb_data.len = &len; - cb_data.exp = NULL; lustre_hash_bucket_iterate(obd->obd_nid_hash_body, &stats->nid, lprocfs_exp_print_uuid, &cb_data); return (*cb_data.len); } +int lprocfs_nid_stats_clear_read(char *page, char **start, off_t off, + int count, int *eof, void *data) +{ + *eof = 1; + return snprintf(page, count, "%s\n", + "Write into this file to clear all nid stats and " + "stale nid entries"); +} +EXPORT_SYMBOL(lprocfs_nid_stats_clear_read); + +void lprocfs_nid_stats_clear_write_cb(void *obj, void *data) +{ + struct nid_stat *client_stat = obj; + int i; + + if(client_stat->nid_exp_ref_count == 1) { + hlist_del_init(&client_stat->nid_hash); + lprocfs_free_client_stats(client_stat, data); + OBD_FREE(client_stat, sizeof(struct nid_stat)); + EXIT; + return; + } + /* we has reference to object - only clear data*/ + if (client_stat->nid_stats) { + lprocfs_clear_stats(client_stat->nid_stats); + } + + if (client_stat->nid_brw_stats) { + for (i = 0; i < BRW_LAST; i++) + lprocfs_oh_clear(&client_stat->nid_brw_stats->hist[i]); + } + EXIT; + return; +} + + +int lprocfs_nid_stats_clear_write(struct file *file, const char *buffer, + unsigned long count, void *data) +{ + struct obd_device *obd = (struct obd_device *)data; + + + lustre_hash_iterate_all(obd->obd_nid_stats_hash_body, + lprocfs_free_client_stats, NULL); + + return count; +} +EXPORT_SYMBOL(lprocfs_nid_stats_clear_write); + int lprocfs_exp_setup(struct obd_export *exp, lnet_nid_t nid, int *newnid) { int rc = 0; - struct nid_stat *tmp = NULL; + struct nid_stat *tmp = NULL, *tmp1; struct obd_device *obd = NULL; - struct obd_export *export = NULL; ENTRY; - if (exp->exp_obd) - obd = exp->exp_obd; + *newnid = 0; if (!exp || !exp->exp_obd || !exp->exp_obd->obd_proc_exports_entry || - !obd->obd_nid_hash_body) + !exp->exp_obd->obd_nid_stats_hash_body) RETURN(-EINVAL); - *newnid = 0; - if (!nid) - RETURN(rc); - - export = lustre_hash_get_object_by_key(obd->obd_nid_hash_body, - &nid); - if (export) { - exp->exp_nid_stats = export->exp_nid_stats; - *newnid = 0; - class_export_put(export); - } else { - OBD_ALLOC(tmp, sizeof(struct nid_stat)); - if (tmp == NULL) - GOTO(out, rc = -ENOMEM); - - tmp->nid = nid; - tmp->nid_obd = exp->exp_obd; - tmp->nid_exp_ref_count = 0; - tmp->nid_proc = proc_mkdir(libcfs_nid2str(nid), - exp->exp_obd->obd_proc_exports_entry); - if (!tmp->nid_proc) { - CERROR("Error making export directory for" - " nid %s\n", libcfs_nid2str(nid)); - OBD_FREE(tmp, sizeof(struct nid_stat)); - GOTO(out, rc = -ENOMEM); - } + RETURN(0); - rc = lprocfs_add_simple(tmp->nid_proc, "uuid", - lprocfs_exp_rd_uuid, NULL, tmp); - if (rc) - CERROR("Error adding the uuid file\n"); + obd = exp->exp_obd; - exp->exp_nid_stats = tmp; - *newnid = 1; + CDEBUG(D_CONFIG, "using hash %p\n", obd->obd_nid_stats_hash_body); - spin_lock(&obd->nid_lock); + OBD_ALLOC(tmp, sizeof(struct nid_stat)); + if (tmp == NULL) + RETURN(-ENOMEM); - list_add_tail(&tmp->nid_chain, - &exp->exp_obd->obd_proc_nid_list); + tmp->nid = nid; + tmp->nid_obd = exp->exp_obd; + tmp->nid_exp_ref_count = 1; /* need live in hash after destroy export */ - spin_unlock(&obd->nid_lock); + tmp1= lustre_hash_findadd_unique(obd->obd_nid_stats_hash_body, &nid, + &tmp->nid_hash); + CDEBUG(D_INFO, "Found stats %p for nid %s - ref %d\n", + tmp1, libcfs_nid2str(nid), tmp->nid_exp_ref_count); + if (tmp1 != tmp) { + exp->exp_nid_stats = tmp1; + GOTO(destroy_new, rc = 0); } - if (exp->exp_nid_stats) - exp->exp_nid_stats->nid_exp_ref_count++; -out: + /* not found - create */ + tmp->nid_proc = proc_mkdir(libcfs_nid2str(nid), + obd->obd_proc_exports_entry); + if (!tmp->nid_proc) { + CERROR("Error making export directory for" + " nid %s\n", libcfs_nid2str(nid)); + lustre_hash_delitem(obd->obd_nid_stats_hash_body, &nid, + &tmp->nid_hash); + GOTO(destroy_new, rc = -ENOMEM); + } + + rc = lprocfs_add_simple(tmp->nid_proc, "uuid", + lprocfs_exp_rd_uuid, NULL, tmp); + if (rc) + CWARN("Error adding the uuid file\n"); + + /* this used only for destroy and because not need locking */ + list_add(&tmp->nid_list, &obd->obd_nid_stats); + + exp->exp_nid_stats = tmp; + *newnid = 1; RETURN(rc); -} +destroy_new: + OBD_FREE(tmp, sizeof(struct nid_stat)); + RETURN(rc); +} int lprocfs_exp_cleanup(struct obd_export *exp) { - if (exp->exp_nid_stats) { - exp->exp_nid_stats->nid_exp_ref_count--; - exp->exp_nid_stats = NULL; - } + struct nid_stat *stat = exp->exp_nid_stats; + + if(!stat) + RETURN(0); + + stat->nid_exp_ref_count--; + CDEBUG(D_INFO, "Put stat %p - %d\n", stat, stat->nid_exp_ref_count); + + exp->exp_nid_stats = NULL; + return 0; } @@ -1613,7 +1680,6 @@ EXPORT_SYMBOL(lprocfs_add_vars); EXPORT_SYMBOL(lprocfs_obd_setup); EXPORT_SYMBOL(lprocfs_obd_cleanup); EXPORT_SYMBOL(lprocfs_add_simple); -EXPORT_SYMBOL(lprocfs_free_client_stats); EXPORT_SYMBOL(lprocfs_free_per_client_stats); EXPORT_SYMBOL(lprocfs_alloc_stats); EXPORT_SYMBOL(lprocfs_free_stats); diff --git a/lustre/obdclass/obd_config.c b/lustre/obdclass/obd_config.c index cfa973f..f5c7be5 100644 --- a/lustre/obdclass/obd_config.c +++ b/lustre/obdclass/obd_config.c @@ -190,7 +190,7 @@ int class_attach(struct lustre_cfg *lcfg) CFS_INIT_LIST_HEAD(&obd->obd_exports); CFS_INIT_LIST_HEAD(&obd->obd_exports_timed); - INIT_LIST_HEAD(&obd->obd_proc_nid_list); + CFS_INIT_LIST_HEAD(&obd->obd_nid_stats); spin_lock_init(&obd->nid_lock); spin_lock_init(&obd->obd_dev_lock); sema_init(&obd->obd_dev_sem, 1); @@ -290,10 +290,19 @@ int class_setup(struct obd_device *obd, struct lustre_cfg *lcfg) /* create a nid-export hash body */ err = lustre_hash_init(&obd->obd_nid_hash_body, "NID_HASH", 128, &nid_hash_operations); + if (err) { + spin_unlock(&obd->obd_dev_lock); + GOTO(err_hash, err); + } + + /* create a nid-stats hash body */ + err = lustre_hash_init(&obd->obd_nid_stats_hash_body, "NID_STATS", + 128, &nid_stat_hash_operations); spin_unlock(&obd->obd_dev_lock); if (err) GOTO(err_hash, err); + exp = class_new_export(obd, &obd->obd_uuid); if (IS_ERR(exp)) RETURN(PTR_ERR(exp)); @@ -322,6 +331,7 @@ err_exp: err_hash: lustre_hash_exit(&obd->obd_uuid_hash_body); lustre_hash_exit(&obd->obd_nid_hash_body); + lustre_hash_exit(&obd->obd_nid_stats_hash_body); obd->obd_starting = 0; CERROR("setup %s failed (%d)\n", obd->obd_name, err); RETURN(err); @@ -460,6 +470,9 @@ int class_cleanup(struct obd_device *obd, struct lustre_cfg *lcfg) /* destroy a nid-export hash body */ lustre_hash_exit(&obd->obd_nid_hash_body); + /* destroy a nid-stats hash body */ + lustre_hash_exit(&obd->obd_nid_stats_hash_body); + /* Precleanup stage 1, we must make sure all exports (other than the self-export) get destroyed. */ err = obd_precleanup(obd, OBD_CLEANUP_EXPORTS); diff --git a/lustre/obdfilter/filter.c b/lustre/obdfilter/filter.c index 0ee1b37..927241cb 100644 --- a/lustre/obdfilter/filter.c +++ b/lustre/obdfilter/filter.c @@ -1792,42 +1792,6 @@ err_mntput: obd->u.obt.obt_sb = 0; return rc; } -static int filter_nid_stats_clear_read(char *page, char **start, off_t off, - int count, int *eof, void *data) -{ - *eof = 1; - return snprintf(page, count, "%s\n", - "Write into this file to clear all nid stats and " - "stale nid entries"); -} - -static int filter_nid_stats_clear_write(struct file *file, const char *buffer, - unsigned long count, void *data) -{ - struct obd_device *obd = (struct obd_device *)data; - struct list_head *nids = &obd->obd_proc_nid_list; - nid_stat_t *client_stat = NULL, *nxt; - int i; - - spin_lock(&obd->nid_lock); - - list_for_each_entry_safe (client_stat, nxt, nids, nid_chain) { - if (!client_stat->nid_exp_ref_count) { - lprocfs_free_client_stats(client_stat); - } else { - if (client_stat->nid_stats) - lprocfs_clear_stats(client_stat->nid_stats); - if (client_stat->nid_brw_stats) - for (i = 0; i < BRW_LAST; i++) - lprocfs_oh_clear( - &client_stat->nid_brw_stats->hist[i]); - } - } - - spin_unlock(&obd->nid_lock); - - return count; -} static int filter_setup(struct obd_device *obd, obd_count len, void *buf) { @@ -1868,8 +1832,8 @@ static int filter_setup(struct obd_device *obd, obd_count len, void *buf) } if (obd->obd_proc_exports_entry) lprocfs_add_simple(obd->obd_proc_exports_entry, "clear", - filter_nid_stats_clear_read, - filter_nid_stats_clear_write, obd); + lprocfs_nid_stats_clear_read, + lprocfs_nid_stats_clear_write, obd); memcpy((void *)addr, lustre_cfg_buf(lcfg, 4), LUSTRE_CFG_BUFLEN(lcfg, 4)); @@ -1877,8 +1841,9 @@ static int filter_setup(struct obd_device *obd, obd_count len, void *buf) OBD_PAGE_FREE(page); if (rc) { - lprocfs_obd_cleanup(obd); + lprocfs_free_per_client_stats(obd); lprocfs_free_obd_stats(obd); + lprocfs_obd_cleanup(obd); } return rc; @@ -2006,10 +1971,10 @@ static int filter_cleanup(struct obd_device *obd) } } - lprocfs_free_per_client_stats(obd); remove_proc_entry("clear", obd->obd_proc_exports_entry); - lprocfs_obd_cleanup(obd); + lprocfs_free_per_client_stats(obd); lprocfs_free_obd_stats(obd); + lprocfs_obd_cleanup(obd); lquota_cleanup(filter_quota_interface_ref, obd); -- 1.8.3.1