X-Git-Url: https://git.whamcloud.com/?p=fs%2Flustre-release.git;a=blobdiff_plain;f=libcfs%2Flibcfs%2Fhash.c;h=228cf0b022a58ee38198caa74b14b05578983ae0;hp=db30f8c9cbd0d5253ea652928e800000af89b775;hb=c4e98d7f89c2c35550d82b10cddbfe93fef75dab;hpb=673f374637d090ba8c4a2cec48eb250b42d2dfa8 diff --git a/libcfs/libcfs/hash.c b/libcfs/libcfs/hash.c index db30f8c..228cf0b 100644 --- a/libcfs/libcfs/hash.c +++ b/libcfs/libcfs/hash.c @@ -1,6 +1,4 @@ -/* -*- mode: c; c-basic-offset: 8; indent-tabs-mode: nil; -*- - * vim:expandtab:shiftwidth=8:tabstop=8: - * +/* * GPL HEADER START * * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. @@ -17,17 +15,15 @@ * * You should have received a copy of the GNU General Public License * version 2 along with this program; If not, see - * http://www.sun.com/software/products/lustre/docs/GPLv2.pdf - * - * Please contact Sun Microsystems, Inc., 4150 Network Circle, Santa Clara, - * CA 95054 USA or visit www.sun.com if you need additional information or - * have any questions. + * http://www.gnu.org/licenses/gpl-2.0.html * * GPL HEADER END */ /* * Copyright (c) 2009, 2010, Oracle and/or its affiliates. All rights reserved. * Use is subject to license terms. + * + * Copyright (c) 2011, 2016, Intel Corporation. */ /* * This file is part of Lustre, http://www.lustre.org/ @@ -51,11 +47,11 @@ * - move all stuff to libcfs * - don't allow cur_bits != max_bits without setting of CFS_HASH_REHASH * - ignore hs_rwlock if without CFS_HASH_REHASH setting - * - buckets are allocated one by one(intead of contiguous memory), + * - buckets are allocated one by one(instead of contiguous memory), * to avoid unnecessary cacheline conflict * * 2010-03-01: Liang Zhen - * - "bucket" is a group of hlist_head now, user can speicify bucket size + * - "bucket" is a group of hlist_head now, user can specify bucket size * by bkt_bits of cfs_hash_create(), all hlist_heads in a bucket share * one lock for reducing memory overhead. * @@ -106,117 +102,120 @@ * Now we support both locked iteration & lockless iteration of hash * table. Also, user can break the iteration by return 1 in callback. */ +#include +#include +#include #include #if CFS_HASH_DEBUG_LEVEL >= CFS_HASH_DEBUG_1 -static unsigned int warn_on_depth = 0; -CFS_MODULE_PARM(warn_on_depth, "i", uint, 0644, - "warning when hash depth is high."); +static unsigned int warn_on_depth = 8; +module_param(warn_on_depth, uint, 0644); +MODULE_PARM_DESC(warn_on_depth, "warning when hash depth is high."); #endif +struct cfs_wi_sched *cfs_sched_rehash; + static inline void -cfs_hash_nl_lock(cfs_hash_lock_t *lock, int exclusive) {} +cfs_hash_nl_lock(union cfs_hash_lock *lock, int exclusive) {} static inline void -cfs_hash_nl_unlock(cfs_hash_lock_t *lock, int exclusive) {} +cfs_hash_nl_unlock(union cfs_hash_lock *lock, int exclusive) {} static inline void -cfs_hash_spin_lock(cfs_hash_lock_t *lock, int exclusive) +cfs_hash_spin_lock(union cfs_hash_lock *lock, int exclusive) + __acquires(&lock->spin) { - cfs_spin_lock(&lock->spin); + spin_lock(&lock->spin); } static inline void -cfs_hash_spin_unlock(cfs_hash_lock_t *lock, int exclusive) +cfs_hash_spin_unlock(union cfs_hash_lock *lock, int exclusive) + __releases(&lock->spin) { - cfs_spin_unlock(&lock->spin); + spin_unlock(&lock->spin); } static inline void -cfs_hash_rw_lock(cfs_hash_lock_t *lock, int exclusive) +cfs_hash_rw_lock(union cfs_hash_lock *lock, int exclusive) + __acquires(&lock->rw) { - if (!exclusive) - cfs_read_lock(&lock->rw); - else - cfs_write_lock(&lock->rw); + if (!exclusive) + read_lock(&lock->rw); + else + write_lock(&lock->rw); } static inline void -cfs_hash_rw_unlock(cfs_hash_lock_t *lock, int exclusive) +cfs_hash_rw_unlock(union cfs_hash_lock *lock, int exclusive) + __releases(&lock->rw) { - if (!exclusive) - cfs_read_unlock(&lock->rw); - else - cfs_write_unlock(&lock->rw); + if (!exclusive) + read_unlock(&lock->rw); + else + write_unlock(&lock->rw); } /** No lock hash */ -static cfs_hash_lock_ops_t cfs_hash_nl_lops = -{ - .hs_lock = cfs_hash_nl_lock, - .hs_unlock = cfs_hash_nl_unlock, - .hs_bkt_lock = cfs_hash_nl_lock, - .hs_bkt_unlock = cfs_hash_nl_unlock, +static struct cfs_hash_lock_ops cfs_hash_nl_lops = { + .hs_lock = cfs_hash_nl_lock, + .hs_unlock = cfs_hash_nl_unlock, + .hs_bkt_lock = cfs_hash_nl_lock, + .hs_bkt_unlock = cfs_hash_nl_unlock, }; /** no bucket lock, one spinlock to protect everything */ -static cfs_hash_lock_ops_t cfs_hash_nbl_lops = -{ - .hs_lock = cfs_hash_spin_lock, - .hs_unlock = cfs_hash_spin_unlock, - .hs_bkt_lock = cfs_hash_nl_lock, - .hs_bkt_unlock = cfs_hash_nl_unlock, +static struct cfs_hash_lock_ops cfs_hash_nbl_lops = { + .hs_lock = cfs_hash_spin_lock, + .hs_unlock = cfs_hash_spin_unlock, + .hs_bkt_lock = cfs_hash_nl_lock, + .hs_bkt_unlock = cfs_hash_nl_unlock, }; /** spin bucket lock, rehash is enabled */ -static cfs_hash_lock_ops_t cfs_hash_bkt_spin_lops = -{ - .hs_lock = cfs_hash_rw_lock, - .hs_unlock = cfs_hash_rw_unlock, - .hs_bkt_lock = cfs_hash_spin_lock, - .hs_bkt_unlock = cfs_hash_spin_unlock, +static struct cfs_hash_lock_ops cfs_hash_bkt_spin_lops = { + .hs_lock = cfs_hash_rw_lock, + .hs_unlock = cfs_hash_rw_unlock, + .hs_bkt_lock = cfs_hash_spin_lock, + .hs_bkt_unlock = cfs_hash_spin_unlock, }; /** rw bucket lock, rehash is enabled */ -static cfs_hash_lock_ops_t cfs_hash_bkt_rw_lops = -{ - .hs_lock = cfs_hash_rw_lock, - .hs_unlock = cfs_hash_rw_unlock, - .hs_bkt_lock = cfs_hash_rw_lock, - .hs_bkt_unlock = cfs_hash_rw_unlock, +static struct cfs_hash_lock_ops cfs_hash_bkt_rw_lops = { + .hs_lock = cfs_hash_rw_lock, + .hs_unlock = cfs_hash_rw_unlock, + .hs_bkt_lock = cfs_hash_rw_lock, + .hs_bkt_unlock = cfs_hash_rw_unlock, }; /** spin bucket lock, rehash is disabled */ -static cfs_hash_lock_ops_t cfs_hash_nr_bkt_spin_lops = -{ - .hs_lock = cfs_hash_nl_lock, - .hs_unlock = cfs_hash_nl_unlock, - .hs_bkt_lock = cfs_hash_spin_lock, - .hs_bkt_unlock = cfs_hash_spin_unlock, +static struct cfs_hash_lock_ops cfs_hash_nr_bkt_spin_lops = { + .hs_lock = cfs_hash_nl_lock, + .hs_unlock = cfs_hash_nl_unlock, + .hs_bkt_lock = cfs_hash_spin_lock, + .hs_bkt_unlock = cfs_hash_spin_unlock, }; /** rw bucket lock, rehash is disabled */ -static cfs_hash_lock_ops_t cfs_hash_nr_bkt_rw_lops = -{ - .hs_lock = cfs_hash_nl_lock, - .hs_unlock = cfs_hash_nl_unlock, - .hs_bkt_lock = cfs_hash_rw_lock, - .hs_bkt_unlock = cfs_hash_rw_unlock, +static struct cfs_hash_lock_ops cfs_hash_nr_bkt_rw_lops = { + .hs_lock = cfs_hash_nl_lock, + .hs_unlock = cfs_hash_nl_unlock, + .hs_bkt_lock = cfs_hash_rw_lock, + .hs_bkt_unlock = cfs_hash_rw_unlock, }; static void -cfs_hash_lock_setup(cfs_hash_t *hs) +cfs_hash_lock_setup(struct cfs_hash *hs) { - if (cfs_hash_with_no_lock(hs)) { - hs->hs_lops = &cfs_hash_nl_lops; + if (cfs_hash_with_no_lock(hs)) { + hs->hs_lops = &cfs_hash_nl_lops; - } else if (cfs_hash_with_no_bktlock(hs)) { - hs->hs_lops = &cfs_hash_nbl_lops; - cfs_spin_lock_init(&hs->hs_lock.spin); + } else if (cfs_hash_with_no_bktlock(hs)) { + hs->hs_lops = &cfs_hash_nbl_lops; + spin_lock_init(&hs->hs_lock.spin); - } else if (cfs_hash_with_rehash(hs)) { - cfs_rwlock_init(&hs->hs_lock.rw); + } else if (cfs_hash_with_rehash(hs)) { + rwlock_init(&hs->hs_lock.rw); if (cfs_hash_with_rw_bktlock(hs)) hs->hs_lops = &cfs_hash_bkt_rw_lops; @@ -238,215 +237,224 @@ cfs_hash_lock_setup(cfs_hash_t *hs) * Simple hash head without depth tracking * new element is always added to head of hlist */ -typedef struct { - cfs_hlist_head_t hh_head; /**< entries list */ -} cfs_hash_head_t; +struct cfs_hash_head { + struct hlist_head hh_head; /**< entries list */ +}; static int -cfs_hash_hh_hhead_size(cfs_hash_t *hs) +cfs_hash_hh_hhead_size(struct cfs_hash *hs) { - return sizeof(cfs_hash_head_t); + return sizeof(struct cfs_hash_head); } -static cfs_hlist_head_t * -cfs_hash_hh_hhead(cfs_hash_t *hs, cfs_hash_bd_t *bd) +static struct hlist_head * +cfs_hash_hh_hhead(struct cfs_hash *hs, struct cfs_hash_bd *bd) { - cfs_hash_head_t *head = (cfs_hash_head_t *)&bd->bd_bucket->hsb_head[0]; + struct cfs_hash_head *head; - return &head[bd->bd_offset].hh_head; + head = (struct cfs_hash_head *)&bd->bd_bucket->hsb_head[0]; + return &head[bd->bd_offset].hh_head; } static int -cfs_hash_hh_hnode_add(cfs_hash_t *hs, cfs_hash_bd_t *bd, - cfs_hlist_node_t *hnode) +cfs_hash_hh_hnode_add(struct cfs_hash *hs, struct cfs_hash_bd *bd, + struct hlist_node *hnode) { - cfs_hlist_add_head(hnode, cfs_hash_hh_hhead(hs, bd)); - return -1; /* unknown depth */ + hlist_add_head(hnode, cfs_hash_hh_hhead(hs, bd)); + return -1; /* unknown depth */ } static int -cfs_hash_hh_hnode_del(cfs_hash_t *hs, cfs_hash_bd_t *bd, - cfs_hlist_node_t *hnode) +cfs_hash_hh_hnode_del(struct cfs_hash *hs, struct cfs_hash_bd *bd, + struct hlist_node *hnode) { - cfs_hlist_del_init(hnode); - return -1; /* unknown depth */ + hlist_del_init(hnode); + return -1; /* unknown depth */ } /** * Simple hash head with depth tracking * new element is always added to head of hlist */ -typedef struct { - cfs_hlist_head_t hd_head; /**< entries list */ - unsigned int hd_depth; /**< list length */ -} cfs_hash_head_dep_t; +struct cfs_hash_head_dep { + struct hlist_head hd_head; /**< entries list */ + unsigned int hd_depth; /**< list length */ +}; static int -cfs_hash_hd_hhead_size(cfs_hash_t *hs) +cfs_hash_hd_hhead_size(struct cfs_hash *hs) { - return sizeof(cfs_hash_head_dep_t); + return sizeof(struct cfs_hash_head_dep); } -static cfs_hlist_head_t * -cfs_hash_hd_hhead(cfs_hash_t *hs, cfs_hash_bd_t *bd) +static struct hlist_head * +cfs_hash_hd_hhead(struct cfs_hash *hs, struct cfs_hash_bd *bd) { - cfs_hash_head_dep_t *head; + struct cfs_hash_head_dep *head; - head = (cfs_hash_head_dep_t *)&bd->bd_bucket->hsb_head[0]; - return &head[bd->bd_offset].hd_head; + head = (struct cfs_hash_head_dep *)&bd->bd_bucket->hsb_head[0]; + return &head[bd->bd_offset].hd_head; } static int -cfs_hash_hd_hnode_add(cfs_hash_t *hs, cfs_hash_bd_t *bd, - cfs_hlist_node_t *hnode) +cfs_hash_hd_hnode_add(struct cfs_hash *hs, struct cfs_hash_bd *bd, + struct hlist_node *hnode) { - cfs_hash_head_dep_t *hh = container_of(cfs_hash_hd_hhead(hs, bd), - cfs_hash_head_dep_t, hd_head); - cfs_hlist_add_head(hnode, &hh->hd_head); - return ++hh->hd_depth; + struct cfs_hash_head_dep *hh; + + hh = container_of(cfs_hash_hd_hhead(hs, bd), + struct cfs_hash_head_dep, hd_head); + hlist_add_head(hnode, &hh->hd_head); + return ++hh->hd_depth; } static int -cfs_hash_hd_hnode_del(cfs_hash_t *hs, cfs_hash_bd_t *bd, - cfs_hlist_node_t *hnode) +cfs_hash_hd_hnode_del(struct cfs_hash *hs, struct cfs_hash_bd *bd, + struct hlist_node *hnode) { - cfs_hash_head_dep_t *hh = container_of(cfs_hash_hd_hhead(hs, bd), - cfs_hash_head_dep_t, hd_head); - cfs_hlist_del_init(hnode); - return --hh->hd_depth; + struct cfs_hash_head_dep *hh; + + hh = container_of(cfs_hash_hd_hhead(hs, bd), + struct cfs_hash_head_dep, hd_head); + hlist_del_init(hnode); + return --hh->hd_depth; } /** * double links hash head without depth tracking * new element is always added to tail of hlist */ -typedef struct { - cfs_hlist_head_t dh_head; /**< entries list */ - cfs_hlist_node_t *dh_tail; /**< the last entry */ -} cfs_hash_dhead_t; +struct cfs_hash_dhead { + struct hlist_head dh_head; /**< entries list */ + struct hlist_node *dh_tail; /**< the last entry */ +}; static int -cfs_hash_dh_hhead_size(cfs_hash_t *hs) +cfs_hash_dh_hhead_size(struct cfs_hash *hs) { - return sizeof(cfs_hash_dhead_t); + return sizeof(struct cfs_hash_dhead); } -static cfs_hlist_head_t * -cfs_hash_dh_hhead(cfs_hash_t *hs, cfs_hash_bd_t *bd) +static struct hlist_head * +cfs_hash_dh_hhead(struct cfs_hash *hs, struct cfs_hash_bd *bd) { - cfs_hash_dhead_t *head; + struct cfs_hash_dhead *head; - head = (cfs_hash_dhead_t *)&bd->bd_bucket->hsb_head[0]; - return &head[bd->bd_offset].dh_head; + head = (struct cfs_hash_dhead *)&bd->bd_bucket->hsb_head[0]; + return &head[bd->bd_offset].dh_head; } static int -cfs_hash_dh_hnode_add(cfs_hash_t *hs, cfs_hash_bd_t *bd, - cfs_hlist_node_t *hnode) +cfs_hash_dh_hnode_add(struct cfs_hash *hs, struct cfs_hash_bd *bd, + struct hlist_node *hnode) { - cfs_hash_dhead_t *dh = container_of(cfs_hash_dh_hhead(hs, bd), - cfs_hash_dhead_t, dh_head); + struct cfs_hash_dhead *dh; - if (dh->dh_tail != NULL) /* not empty */ - cfs_hlist_add_after(dh->dh_tail, hnode); - else /* empty list */ - cfs_hlist_add_head(hnode, &dh->dh_head); - dh->dh_tail = hnode; - return -1; /* unknown depth */ + dh = container_of(cfs_hash_dh_hhead(hs, bd), + struct cfs_hash_dhead, dh_head); + if (dh->dh_tail != NULL) /* not empty */ + hlist_add_behind(hnode, dh->dh_tail); + else /* empty list */ + hlist_add_head(hnode, &dh->dh_head); + dh->dh_tail = hnode; + return -1; /* unknown depth */ } static int -cfs_hash_dh_hnode_del(cfs_hash_t *hs, cfs_hash_bd_t *bd, - cfs_hlist_node_t *hnd) +cfs_hash_dh_hnode_del(struct cfs_hash *hs, struct cfs_hash_bd *bd, + struct hlist_node *hnd) { - cfs_hash_dhead_t *dh = container_of(cfs_hash_dh_hhead(hs, bd), - cfs_hash_dhead_t, dh_head); + struct cfs_hash_dhead *dh; - if (hnd->next == NULL) { /* it's the tail */ - dh->dh_tail = (hnd->pprev == &dh->dh_head.first) ? NULL : - container_of(hnd->pprev, cfs_hlist_node_t, next); - } - cfs_hlist_del_init(hnd); - return -1; /* unknown depth */ + dh = container_of(cfs_hash_dh_hhead(hs, bd), + struct cfs_hash_dhead, dh_head); + if (hnd->next == NULL) { /* it's the tail */ + dh->dh_tail = (hnd->pprev == &dh->dh_head.first) ? NULL : + container_of(hnd->pprev, struct hlist_node, next); + } + hlist_del_init(hnd); + return -1; /* unknown depth */ } /** * double links hash head with depth tracking * new element is always added to tail of hlist */ -typedef struct { - cfs_hlist_head_t dd_head; /**< entries list */ - cfs_hlist_node_t *dd_tail; /**< the last entry */ - unsigned int dd_depth; /**< list length */ -} cfs_hash_dhead_dep_t; +struct cfs_hash_dhead_dep { + struct hlist_head dd_head; /**< entries list */ + struct hlist_node *dd_tail; /**< the last entry */ + unsigned int dd_depth; /**< list length */ +}; static int -cfs_hash_dd_hhead_size(cfs_hash_t *hs) +cfs_hash_dd_hhead_size(struct cfs_hash *hs) { - return sizeof(cfs_hash_dhead_dep_t); + return sizeof(struct cfs_hash_dhead_dep); } -static cfs_hlist_head_t * -cfs_hash_dd_hhead(cfs_hash_t *hs, cfs_hash_bd_t *bd) +static struct hlist_head * +cfs_hash_dd_hhead(struct cfs_hash *hs, struct cfs_hash_bd *bd) { - cfs_hash_dhead_dep_t *head; + struct cfs_hash_dhead_dep *head; - head = (cfs_hash_dhead_dep_t *)&bd->bd_bucket->hsb_head[0]; - return &head[bd->bd_offset].dd_head; + head = (struct cfs_hash_dhead_dep *)&bd->bd_bucket->hsb_head[0]; + return &head[bd->bd_offset].dd_head; } static int -cfs_hash_dd_hnode_add(cfs_hash_t *hs, cfs_hash_bd_t *bd, - cfs_hlist_node_t *hnode) +cfs_hash_dd_hnode_add(struct cfs_hash *hs, struct cfs_hash_bd *bd, + struct hlist_node *hnode) { - cfs_hash_dhead_dep_t *dh = container_of(cfs_hash_dd_hhead(hs, bd), - cfs_hash_dhead_dep_t, dd_head); + struct cfs_hash_dhead_dep *dh; - if (dh->dd_tail != NULL) /* not empty */ - cfs_hlist_add_after(dh->dd_tail, hnode); - else /* empty list */ - cfs_hlist_add_head(hnode, &dh->dd_head); - dh->dd_tail = hnode; - return ++dh->dd_depth; + dh = container_of(cfs_hash_dd_hhead(hs, bd), + struct cfs_hash_dhead_dep, dd_head); + if (dh->dd_tail != NULL) /* not empty */ + hlist_add_behind(hnode, dh->dd_tail); + else /* empty list */ + hlist_add_head(hnode, &dh->dd_head); + dh->dd_tail = hnode; + return ++dh->dd_depth; } static int -cfs_hash_dd_hnode_del(cfs_hash_t *hs, cfs_hash_bd_t *bd, - cfs_hlist_node_t *hnd) +cfs_hash_dd_hnode_del(struct cfs_hash *hs, struct cfs_hash_bd *bd, + struct hlist_node *hnd) { - cfs_hash_dhead_dep_t *dh = container_of(cfs_hash_dd_hhead(hs, bd), - cfs_hash_dhead_dep_t, dd_head); + struct cfs_hash_dhead_dep *dh; - if (hnd->next == NULL) { /* it's the tail */ - dh->dd_tail = (hnd->pprev == &dh->dd_head.first) ? NULL : - container_of(hnd->pprev, cfs_hlist_node_t, next); - } - cfs_hlist_del_init(hnd); - return --dh->dd_depth; + dh = container_of(cfs_hash_dd_hhead(hs, bd), + struct cfs_hash_dhead_dep, dd_head); + if (hnd->next == NULL) { /* it's the tail */ + dh->dd_tail = (hnd->pprev == &dh->dd_head.first) ? NULL : + container_of(hnd->pprev, struct hlist_node, next); + } + hlist_del_init(hnd); + return --dh->dd_depth; } -static cfs_hash_hlist_ops_t cfs_hash_hh_hops = { +static struct cfs_hash_hlist_ops cfs_hash_hh_hops = { .hop_hhead = cfs_hash_hh_hhead, .hop_hhead_size = cfs_hash_hh_hhead_size, .hop_hnode_add = cfs_hash_hh_hnode_add, .hop_hnode_del = cfs_hash_hh_hnode_del, }; -static cfs_hash_hlist_ops_t cfs_hash_hd_hops = { +static struct cfs_hash_hlist_ops cfs_hash_hd_hops = { .hop_hhead = cfs_hash_hd_hhead, .hop_hhead_size = cfs_hash_hd_hhead_size, .hop_hnode_add = cfs_hash_hd_hnode_add, .hop_hnode_del = cfs_hash_hd_hnode_del, }; -static cfs_hash_hlist_ops_t cfs_hash_dh_hops = { +static struct cfs_hash_hlist_ops cfs_hash_dh_hops = { .hop_hhead = cfs_hash_dh_hhead, .hop_hhead_size = cfs_hash_dh_hhead_size, .hop_hnode_add = cfs_hash_dh_hnode_add, .hop_hnode_del = cfs_hash_dh_hnode_del, }; -static cfs_hash_hlist_ops_t cfs_hash_dd_hops = { +static struct cfs_hash_hlist_ops cfs_hash_dd_hops = { .hop_hhead = cfs_hash_dd_hhead, .hop_hhead_size = cfs_hash_dd_hhead_size, .hop_hnode_add = cfs_hash_dd_hnode_add, @@ -454,7 +462,7 @@ static cfs_hash_hlist_ops_t cfs_hash_dd_hops = { }; static void -cfs_hash_hlist_setup(cfs_hash_t *hs) +cfs_hash_hlist_setup(struct cfs_hash *hs) { if (cfs_hash_with_add_tail(hs)) { hs->hs_hops = cfs_hash_with_depth(hs) ? @@ -466,8 +474,8 @@ cfs_hash_hlist_setup(cfs_hash_t *hs) } static void -cfs_hash_bd_from_key(cfs_hash_t *hs, cfs_hash_bucket_t **bkts, - unsigned int bits, const void *key, cfs_hash_bd_t *bd) +cfs_hash_bd_from_key(struct cfs_hash *hs, struct cfs_hash_bucket **bkts, + unsigned int bits, const void *key, struct cfs_hash_bd *bd) { unsigned int index = cfs_hash_id(hs, key, (1U << bits) - 1); @@ -478,7 +486,7 @@ cfs_hash_bd_from_key(cfs_hash_t *hs, cfs_hash_bucket_t **bkts, } void -cfs_hash_bd_get(cfs_hash_t *hs, const void *key, cfs_hash_bd_t *bd) +cfs_hash_bd_get(struct cfs_hash *hs, const void *key, struct cfs_hash_bd *bd) { /* NB: caller should hold hs->hs_rwlock if REHASH is set */ if (likely(hs->hs_rehash_buckets == NULL)) { @@ -490,10 +498,10 @@ cfs_hash_bd_get(cfs_hash_t *hs, const void *key, cfs_hash_bd_t *bd) hs->hs_rehash_bits, key, bd); } } -CFS_EXPORT_SYMBOL(cfs_hash_bd_get); +EXPORT_SYMBOL(cfs_hash_bd_get); static inline void -cfs_hash_bd_dep_record(cfs_hash_t *hs, cfs_hash_bd_t *bd, int dep_cur) +cfs_hash_bd_dep_record(struct cfs_hash *hs, struct cfs_hash_bd *bd, int dep_cur) { if (likely(dep_cur <= bd->bd_bucket->hsb_depmax)) return; @@ -504,64 +512,64 @@ cfs_hash_bd_dep_record(cfs_hash_t *hs, cfs_hash_bd_t *bd, int dep_cur) max(warn_on_depth, hs->hs_dep_max) >= dep_cur)) return; - cfs_spin_lock(&hs->hs_dep_lock); - hs->hs_dep_max = dep_cur; - hs->hs_dep_bkt = bd->bd_bucket->hsb_index; - hs->hs_dep_off = bd->bd_offset; - hs->hs_dep_bits = hs->hs_cur_bits; - cfs_spin_unlock(&hs->hs_dep_lock); + spin_lock(&hs->hs_dep_lock); + hs->hs_dep_max = dep_cur; + hs->hs_dep_bkt = bd->bd_bucket->hsb_index; + hs->hs_dep_off = bd->bd_offset; + hs->hs_dep_bits = hs->hs_cur_bits; + spin_unlock(&hs->hs_dep_lock); - cfs_wi_schedule(&hs->hs_dep_wi); + cfs_wi_schedule(cfs_sched_rehash, &hs->hs_dep_wi); # endif } void -cfs_hash_bd_add_locked(cfs_hash_t *hs, cfs_hash_bd_t *bd, - cfs_hlist_node_t *hnode) +cfs_hash_bd_add_locked(struct cfs_hash *hs, struct cfs_hash_bd *bd, + struct hlist_node *hnode) { - int rc; + int rc; - rc = hs->hs_hops->hop_hnode_add(hs, bd, hnode); - cfs_hash_bd_dep_record(hs, bd, rc); - bd->bd_bucket->hsb_version++; - if (unlikely(bd->bd_bucket->hsb_version == 0)) - bd->bd_bucket->hsb_version++; - bd->bd_bucket->hsb_count++; + rc = hs->hs_hops->hop_hnode_add(hs, bd, hnode); + cfs_hash_bd_dep_record(hs, bd, rc); + bd->bd_bucket->hsb_version++; + if (unlikely(bd->bd_bucket->hsb_version == 0)) + bd->bd_bucket->hsb_version++; + bd->bd_bucket->hsb_count++; - if (cfs_hash_with_counter(hs)) - cfs_atomic_inc(&hs->hs_count); - if (!cfs_hash_with_no_itemref(hs)) - cfs_hash_get(hs, hnode); + if (cfs_hash_with_counter(hs)) + atomic_inc(&hs->hs_count); + if (!cfs_hash_with_no_itemref(hs)) + cfs_hash_get(hs, hnode); } -CFS_EXPORT_SYMBOL(cfs_hash_bd_add_locked); +EXPORT_SYMBOL(cfs_hash_bd_add_locked); void -cfs_hash_bd_del_locked(cfs_hash_t *hs, cfs_hash_bd_t *bd, - cfs_hlist_node_t *hnode) +cfs_hash_bd_del_locked(struct cfs_hash *hs, struct cfs_hash_bd *bd, + struct hlist_node *hnode) { - hs->hs_hops->hop_hnode_del(hs, bd, hnode); + hs->hs_hops->hop_hnode_del(hs, bd, hnode); - LASSERT(bd->bd_bucket->hsb_count > 0); - bd->bd_bucket->hsb_count--; - bd->bd_bucket->hsb_version++; - if (unlikely(bd->bd_bucket->hsb_version == 0)) - bd->bd_bucket->hsb_version++; + LASSERT(bd->bd_bucket->hsb_count > 0); + bd->bd_bucket->hsb_count--; + bd->bd_bucket->hsb_version++; + if (unlikely(bd->bd_bucket->hsb_version == 0)) + bd->bd_bucket->hsb_version++; - if (cfs_hash_with_counter(hs)) { - LASSERT(cfs_atomic_read(&hs->hs_count) > 0); - cfs_atomic_dec(&hs->hs_count); - } - if (!cfs_hash_with_no_itemref(hs)) - cfs_hash_put_locked(hs, hnode); + if (cfs_hash_with_counter(hs)) { + LASSERT(atomic_read(&hs->hs_count) > 0); + atomic_dec(&hs->hs_count); + } + if (!cfs_hash_with_no_itemref(hs)) + cfs_hash_put_locked(hs, hnode); } -CFS_EXPORT_SYMBOL(cfs_hash_bd_del_locked); +EXPORT_SYMBOL(cfs_hash_bd_del_locked); void -cfs_hash_bd_move_locked(cfs_hash_t *hs, cfs_hash_bd_t *bd_old, - cfs_hash_bd_t *bd_new, cfs_hlist_node_t *hnode) +cfs_hash_bd_move_locked(struct cfs_hash *hs, struct cfs_hash_bd *bd_old, + struct cfs_hash_bd *bd_new, struct hlist_node *hnode) { - cfs_hash_bucket_t *obkt = bd_old->bd_bucket; - cfs_hash_bucket_t *nbkt = bd_new->bd_bucket; + struct cfs_hash_bucket *obkt = bd_old->bd_bucket; + struct cfs_hash_bucket *nbkt = bd_new->bd_bucket; int rc; if (cfs_hash_bd_compare(bd_old, bd_new) == 0) @@ -583,7 +591,6 @@ cfs_hash_bd_move_locked(cfs_hash_t *hs, cfs_hash_bd_t *bd_old, if (unlikely(nbkt->hsb_version == 0)) nbkt->hsb_version++; } -CFS_EXPORT_SYMBOL(cfs_hash_bd_move_locked); enum { /** always set, for sanity (avoid ZERO intent) */ @@ -596,7 +603,7 @@ enum { CFS_HS_LOOKUP_MASK_DEL = 1 << 3, }; -typedef enum cfs_hash_lookup_intent { +enum cfs_hash_lookup_intent { /** return item w/o refcount */ CFS_HS_LOOKUP_IT_PEEK = CFS_HS_LOOKUP_MASK_FIND, /** return item with refcount */ @@ -611,25 +618,25 @@ typedef enum cfs_hash_lookup_intent { /** delete if existed */ CFS_HS_LOOKUP_IT_FINDDEL = (CFS_HS_LOOKUP_MASK_FIND | CFS_HS_LOOKUP_MASK_DEL) -} cfs_hash_lookup_intent_t; +}; -static cfs_hlist_node_t * -cfs_hash_bd_lookup_intent(cfs_hash_t *hs, cfs_hash_bd_t *bd, - const void *key, cfs_hlist_node_t *hnode, - cfs_hash_lookup_intent_t intent) +static struct hlist_node * +cfs_hash_bd_lookup_intent(struct cfs_hash *hs, struct cfs_hash_bd *bd, + const void *key, struct hlist_node *hnode, + enum cfs_hash_lookup_intent intent) { - cfs_hlist_head_t *hhead = cfs_hash_bd_hhead(hs, bd); - cfs_hlist_node_t *ehnode; - cfs_hlist_node_t *match; - int intent_add = (intent & CFS_HS_LOOKUP_MASK_ADD) != 0; + struct hlist_head *hhead = cfs_hash_bd_hhead(hs, bd); + struct hlist_node *ehnode; + struct hlist_node *match; + int intent_add = (intent & CFS_HS_LOOKUP_MASK_ADD) != 0; - /* with this function, we can avoid a lot of useless refcount ops, - * which are expensive atomic operations most time. */ - match = intent_add ? NULL : hnode; - cfs_hlist_for_each(ehnode, hhead) { - if (!cfs_hash_keycmp(hs, key, ehnode)) - continue; + /* with this function, we can avoid a lot of useless refcount ops, + * which are expensive atomic operations most time. */ + match = intent_add ? NULL : hnode; + hlist_for_each(ehnode, hhead) { + if (!cfs_hash_keycmp(hs, key, ehnode)) + continue; if (match != NULL && match != ehnode) /* can't match */ continue; @@ -654,40 +661,29 @@ cfs_hash_bd_lookup_intent(cfs_hash_t *hs, cfs_hash_bd_t *bd, return hnode; } -cfs_hlist_node_t * -cfs_hash_bd_lookup_locked(cfs_hash_t *hs, cfs_hash_bd_t *bd, const void *key) +struct hlist_node * +cfs_hash_bd_lookup_locked(struct cfs_hash *hs, struct cfs_hash_bd *bd, + const void *key) { - return cfs_hash_bd_lookup_intent(hs, bd, key, NULL, - CFS_HS_LOOKUP_IT_FIND); + return cfs_hash_bd_lookup_intent(hs, bd, key, NULL, + CFS_HS_LOOKUP_IT_FIND); } -CFS_EXPORT_SYMBOL(cfs_hash_bd_lookup_locked); +EXPORT_SYMBOL(cfs_hash_bd_lookup_locked); -cfs_hlist_node_t * -cfs_hash_bd_findadd_locked(cfs_hash_t *hs, cfs_hash_bd_t *bd, - const void *key, cfs_hlist_node_t *hnode, - int noref) +struct hlist_node * +cfs_hash_bd_peek_locked(struct cfs_hash *hs, struct cfs_hash_bd *bd, + const void *key) { - return cfs_hash_bd_lookup_intent(hs, bd, key, hnode, - CFS_HS_LOOKUP_IT_ADD | - (!noref * CFS_HS_LOOKUP_MASK_REF)); + return cfs_hash_bd_lookup_intent(hs, bd, key, NULL, + CFS_HS_LOOKUP_IT_PEEK); } -CFS_EXPORT_SYMBOL(cfs_hash_bd_findadd_locked); - -cfs_hlist_node_t * -cfs_hash_bd_finddel_locked(cfs_hash_t *hs, cfs_hash_bd_t *bd, - const void *key, cfs_hlist_node_t *hnode) -{ - /* hnode can be NULL, we find the first item with @key */ - return cfs_hash_bd_lookup_intent(hs, bd, key, hnode, - CFS_HS_LOOKUP_IT_FINDDEL); -} -CFS_EXPORT_SYMBOL(cfs_hash_bd_finddel_locked); +EXPORT_SYMBOL(cfs_hash_bd_peek_locked); static void -cfs_hash_multi_bd_lock(cfs_hash_t *hs, cfs_hash_bd_t *bds, +cfs_hash_multi_bd_lock(struct cfs_hash *hs, struct cfs_hash_bd *bds, unsigned n, int excl) { - cfs_hash_bucket_t *prev = NULL; + struct cfs_hash_bucket *prev = NULL; int i; /** @@ -707,10 +703,10 @@ cfs_hash_multi_bd_lock(cfs_hash_t *hs, cfs_hash_bd_t *bds, } static void -cfs_hash_multi_bd_unlock(cfs_hash_t *hs, cfs_hash_bd_t *bds, +cfs_hash_multi_bd_unlock(struct cfs_hash *hs, struct cfs_hash_bd *bds, unsigned n, int excl) { - cfs_hash_bucket_t *prev = NULL; + struct cfs_hash_bucket *prev = NULL; int i; cfs_hash_for_each_bd(bds, n, i) { @@ -721,30 +717,30 @@ cfs_hash_multi_bd_unlock(cfs_hash_t *hs, cfs_hash_bd_t *bds, } } -static cfs_hlist_node_t * -cfs_hash_multi_bd_lookup_locked(cfs_hash_t *hs, cfs_hash_bd_t *bds, - unsigned n, const void *key) +static struct hlist_node * +cfs_hash_multi_bd_lookup_locked(struct cfs_hash *hs, struct cfs_hash_bd *bds, + unsigned n, const void *key) { - cfs_hlist_node_t *ehnode; - unsigned i; + struct hlist_node *ehnode; + unsigned i; - cfs_hash_for_each_bd(bds, n, i) { - ehnode = cfs_hash_bd_lookup_intent(hs, &bds[i], key, NULL, - CFS_HS_LOOKUP_IT_FIND); - if (ehnode != NULL) - return ehnode; - } - return NULL; + cfs_hash_for_each_bd(bds, n, i) { + ehnode = cfs_hash_bd_lookup_intent(hs, &bds[i], key, NULL, + CFS_HS_LOOKUP_IT_FIND); + if (ehnode != NULL) + return ehnode; + } + return NULL; } -static cfs_hlist_node_t * -cfs_hash_multi_bd_findadd_locked(cfs_hash_t *hs, - cfs_hash_bd_t *bds, unsigned n, const void *key, - cfs_hlist_node_t *hnode, int noref) +static struct hlist_node * +cfs_hash_multi_bd_findadd_locked(struct cfs_hash *hs, struct cfs_hash_bd *bds, + unsigned n, const void *key, + struct hlist_node *hnode, int noref) { - cfs_hlist_node_t *ehnode; - int intent; - unsigned i; + struct hlist_node *ehnode; + int intent; + unsigned i; LASSERT(hnode != NULL); intent = CFS_HS_LOOKUP_IT_PEEK | (!noref * CFS_HS_LOOKUP_MASK_REF); @@ -759,7 +755,7 @@ cfs_hash_multi_bd_findadd_locked(cfs_hash_t *hs, if (i == 1) { /* only one bucket */ cfs_hash_bd_add_locked(hs, &bds[0], hnode); } else { - cfs_hash_bd_t mybd; + struct cfs_hash_bd mybd; cfs_hash_bd_get(hs, key, &mybd); cfs_hash_bd_add_locked(hs, &mybd, hnode); @@ -768,25 +764,25 @@ cfs_hash_multi_bd_findadd_locked(cfs_hash_t *hs, return hnode; } -static cfs_hlist_node_t * -cfs_hash_multi_bd_finddel_locked(cfs_hash_t *hs, cfs_hash_bd_t *bds, - unsigned n, const void *key, - cfs_hlist_node_t *hnode) +static struct hlist_node * +cfs_hash_multi_bd_finddel_locked(struct cfs_hash *hs, struct cfs_hash_bd *bds, + unsigned n, const void *key, + struct hlist_node *hnode) { - cfs_hlist_node_t *ehnode; - unsigned i; + struct hlist_node *ehnode; + unsigned i; - cfs_hash_for_each_bd(bds, n, i) { - ehnode = cfs_hash_bd_lookup_intent(hs, &bds[i], key, hnode, - CFS_HS_LOOKUP_IT_FINDDEL); - if (ehnode != NULL) - return ehnode; - } - return NULL; + cfs_hash_for_each_bd(bds, n, i) { + ehnode = cfs_hash_bd_lookup_intent(hs, &bds[i], key, hnode, + CFS_HS_LOOKUP_IT_FINDDEL); + if (ehnode != NULL) + return ehnode; + } + return NULL; } static void -cfs_hash_bd_order(cfs_hash_bd_t *bd1, cfs_hash_bd_t *bd2) +cfs_hash_bd_order(struct cfs_hash_bd *bd1, struct cfs_hash_bd *bd2) { int rc; @@ -803,17 +799,14 @@ cfs_hash_bd_order(cfs_hash_bd_t *bd1, cfs_hash_bd_t *bd2) if (rc == 0) { bd2->bd_bucket = NULL; - } else if (rc > 0) { /* swab bd1 and bd2 */ - cfs_hash_bd_t tmp; - - tmp = *bd2; - *bd2 = *bd1; - *bd1 = tmp; + } else if (rc > 0) { + swap(*bd1, *bd2); /* swab bd1 and bd2 */ } } void -cfs_hash_dual_bd_get(cfs_hash_t *hs, const void *key, cfs_hash_bd_t *bds) +cfs_hash_dual_bd_get(struct cfs_hash *hs, const void *key, + struct cfs_hash_bd *bds) { /* NB: caller should hold hs_lock.rw if REHASH is set */ cfs_hash_bd_from_key(hs, hs->hs_buckets, @@ -830,50 +823,44 @@ cfs_hash_dual_bd_get(cfs_hash_t *hs, const void *key, cfs_hash_bd_t *bds) cfs_hash_bd_order(&bds[0], &bds[1]); } -CFS_EXPORT_SYMBOL(cfs_hash_dual_bd_get); void -cfs_hash_dual_bd_lock(cfs_hash_t *hs, cfs_hash_bd_t *bds, int excl) +cfs_hash_dual_bd_lock(struct cfs_hash *hs, struct cfs_hash_bd *bds, int excl) { cfs_hash_multi_bd_lock(hs, bds, 2, excl); } -CFS_EXPORT_SYMBOL(cfs_hash_dual_bd_lock); void -cfs_hash_dual_bd_unlock(cfs_hash_t *hs, cfs_hash_bd_t *bds, int excl) +cfs_hash_dual_bd_unlock(struct cfs_hash *hs, struct cfs_hash_bd *bds, int excl) { cfs_hash_multi_bd_unlock(hs, bds, 2, excl); } -CFS_EXPORT_SYMBOL(cfs_hash_dual_bd_unlock); -cfs_hlist_node_t * -cfs_hash_dual_bd_lookup_locked(cfs_hash_t *hs, cfs_hash_bd_t *bds, +struct hlist_node * +cfs_hash_dual_bd_lookup_locked(struct cfs_hash *hs, struct cfs_hash_bd *bds, const void *key) { return cfs_hash_multi_bd_lookup_locked(hs, bds, 2, key); } -CFS_EXPORT_SYMBOL(cfs_hash_dual_bd_lookup_locked); -cfs_hlist_node_t * -cfs_hash_dual_bd_findadd_locked(cfs_hash_t *hs, cfs_hash_bd_t *bds, - const void *key, cfs_hlist_node_t *hnode, - int noref) +struct hlist_node * +cfs_hash_dual_bd_findadd_locked(struct cfs_hash *hs, struct cfs_hash_bd *bds, + const void *key, struct hlist_node *hnode, + int noref) { - return cfs_hash_multi_bd_findadd_locked(hs, bds, 2, key, - hnode, noref); + return cfs_hash_multi_bd_findadd_locked(hs, bds, 2, key, + hnode, noref); } -CFS_EXPORT_SYMBOL(cfs_hash_dual_bd_findadd_locked); -cfs_hlist_node_t * -cfs_hash_dual_bd_finddel_locked(cfs_hash_t *hs, cfs_hash_bd_t *bds, - const void *key, cfs_hlist_node_t *hnode) +struct hlist_node * +cfs_hash_dual_bd_finddel_locked(struct cfs_hash *hs, struct cfs_hash_bd *bds, + const void *key, struct hlist_node *hnode) { - return cfs_hash_multi_bd_finddel_locked(hs, bds, 2, key, hnode); + return cfs_hash_multi_bd_finddel_locked(hs, bds, 2, key, hnode); } -CFS_EXPORT_SYMBOL(cfs_hash_dual_bd_finddel_locked); static void -cfs_hash_buckets_free(cfs_hash_bucket_t **buckets, +cfs_hash_buckets_free(struct cfs_hash_bucket **buckets, int bkt_size, int prev_size, int size) { int i; @@ -891,11 +878,11 @@ cfs_hash_buckets_free(cfs_hash_bucket_t **buckets, * needed, the newly allocated buckets if allocation was needed and * successful, and NULL on error. */ -static cfs_hash_bucket_t ** -cfs_hash_buckets_realloc(cfs_hash_t *hs, cfs_hash_bucket_t **old_bkts, +static struct cfs_hash_bucket ** +cfs_hash_buckets_realloc(struct cfs_hash *hs, struct cfs_hash_bucket **old_bkts, unsigned int old_size, unsigned int new_size) { - cfs_hash_bucket_t **new_bkts; + struct cfs_hash_bucket **new_bkts; int i; LASSERT(old_size == 0 || old_bkts != NULL); @@ -912,9 +899,9 @@ cfs_hash_buckets_realloc(cfs_hash_t *hs, cfs_hash_bucket_t **old_bkts, min(old_size, new_size) * sizeof(*old_bkts)); } - for (i = old_size; i < new_size; i++) { - cfs_hlist_head_t *hhead; - cfs_hash_bd_t bd; + for (i = old_size; i < new_size; i++) { + struct hlist_head *hhead; + struct cfs_hash_bd bd; LIBCFS_ALLOC(new_bkts[i], cfs_hash_bkt_size(hs)); if (new_bkts[i] == NULL) { @@ -923,25 +910,25 @@ cfs_hash_buckets_realloc(cfs_hash_t *hs, cfs_hash_bucket_t **old_bkts, return NULL; } - new_bkts[i]->hsb_index = i; - new_bkts[i]->hsb_version = 1; /* shouldn't be zero */ - new_bkts[i]->hsb_depmax = -1; /* unknown */ - bd.bd_bucket = new_bkts[i]; - cfs_hash_bd_for_each_hlist(hs, &bd, hhead) - CFS_INIT_HLIST_HEAD(hhead); + new_bkts[i]->hsb_index = i; + new_bkts[i]->hsb_version = 1; /* shouldn't be zero */ + new_bkts[i]->hsb_depmax = -1; /* unknown */ + bd.bd_bucket = new_bkts[i]; + cfs_hash_bd_for_each_hlist(hs, &bd, hhead) + INIT_HLIST_HEAD(hhead); if (cfs_hash_with_no_lock(hs) || cfs_hash_with_no_bktlock(hs)) continue; - if (cfs_hash_with_rw_bktlock(hs)) - cfs_rwlock_init(&new_bkts[i]->hsb_lock.rw); - else if (cfs_hash_with_spin_bktlock(hs)) - cfs_spin_lock_init(&new_bkts[i]->hsb_lock.spin); - else - LBUG(); /* invalid use-case */ - } - return new_bkts; + if (cfs_hash_with_rw_bktlock(hs)) + rwlock_init(&new_bkts[i]->hsb_lock.rw); + else if (cfs_hash_with_spin_bktlock(hs)) + spin_lock_init(&new_bkts[i]->hsb_lock.spin); + else + LBUG(); /* invalid use-case */ + } + return new_bkts; } /** @@ -953,67 +940,66 @@ cfs_hash_buckets_realloc(cfs_hash_t *hs, cfs_hash_bucket_t **old_bkts, * @flags - CFS_HASH_REHASH enable synamic hash resizing * - CFS_HASH_SORT enable chained hash sort */ -static int cfs_hash_rehash_worker(cfs_workitem_t *wi); +static int cfs_hash_rehash_worker(struct cfs_workitem *wi); #if CFS_HASH_DEBUG_LEVEL >= CFS_HASH_DEBUG_1 -static int cfs_hash_dep_print(cfs_workitem_t *wi) -{ - cfs_hash_t *hs = container_of(wi, cfs_hash_t, hs_dep_wi); - int dep; - int bkt; - int off; - int bits; - - cfs_spin_lock(&hs->hs_dep_lock); - dep = hs->hs_dep_max; - bkt = hs->hs_dep_bkt; - off = hs->hs_dep_off; - bits = hs->hs_dep_bits; - cfs_spin_unlock(&hs->hs_dep_lock); - - LCONSOLE_WARN("#### HASH %s (bits: %d): max depth %d at bucket %d/%d\n", - hs->hs_name, bits, dep, bkt, off); - cfs_spin_lock(&hs->hs_dep_lock); - hs->hs_dep_bits = 0; /* mark as workitem done */ - cfs_spin_unlock(&hs->hs_dep_lock); - return 0; +static int cfs_hash_dep_print(struct cfs_workitem *wi) +{ + struct cfs_hash *hs = container_of(wi, struct cfs_hash, hs_dep_wi); + int dep; + int bkt; + int off; + int bits; + + spin_lock(&hs->hs_dep_lock); + dep = hs->hs_dep_max; + bkt = hs->hs_dep_bkt; + off = hs->hs_dep_off; + bits = hs->hs_dep_bits; + spin_unlock(&hs->hs_dep_lock); + + LCONSOLE_WARN("#### HASH %s (bits: %d): max depth %d at bucket %d/%d\n", + hs->hs_name, bits, dep, bkt, off); + spin_lock(&hs->hs_dep_lock); + hs->hs_dep_bits = 0; /* mark as workitem done */ + spin_unlock(&hs->hs_dep_lock); + return 0; } -static void cfs_hash_depth_wi_init(cfs_hash_t *hs) +static void cfs_hash_depth_wi_init(struct cfs_hash *hs) { - cfs_spin_lock_init(&hs->hs_dep_lock); - cfs_wi_init(&hs->hs_dep_wi, hs, - cfs_hash_dep_print, CFS_WI_SCHED_ANY); + spin_lock_init(&hs->hs_dep_lock); + cfs_wi_init(&hs->hs_dep_wi, hs, cfs_hash_dep_print); } -static void cfs_hash_depth_wi_cancel(cfs_hash_t *hs) +static void cfs_hash_depth_wi_cancel(struct cfs_hash *hs) { - if (cfs_wi_cancel(&hs->hs_dep_wi)) - return; + if (cfs_wi_deschedule(cfs_sched_rehash, &hs->hs_dep_wi)) + return; - cfs_spin_lock(&hs->hs_dep_lock); - while (hs->hs_dep_bits != 0) { - cfs_spin_unlock(&hs->hs_dep_lock); - cfs_cond_resched(); - cfs_spin_lock(&hs->hs_dep_lock); - } - cfs_spin_unlock(&hs->hs_dep_lock); + spin_lock(&hs->hs_dep_lock); + while (hs->hs_dep_bits != 0) { + spin_unlock(&hs->hs_dep_lock); + cond_resched(); + spin_lock(&hs->hs_dep_lock); + } + spin_unlock(&hs->hs_dep_lock); } #else /* CFS_HASH_DEBUG_LEVEL < CFS_HASH_DEBUG_1 */ -static inline void cfs_hash_depth_wi_init(cfs_hash_t *hs) {} -static inline void cfs_hash_depth_wi_cancel(cfs_hash_t *hs) {} +static inline void cfs_hash_depth_wi_init(struct cfs_hash *hs) {} +static inline void cfs_hash_depth_wi_cancel(struct cfs_hash *hs) {} #endif /* CFS_HASH_DEBUG_LEVEL >= CFS_HASH_DEBUG_1 */ -cfs_hash_t * +struct cfs_hash * cfs_hash_create(char *name, unsigned cur_bits, unsigned max_bits, unsigned bkt_bits, unsigned extra_bytes, unsigned min_theta, unsigned max_theta, - cfs_hash_ops_t *ops, unsigned flags) + struct cfs_hash_ops *ops, unsigned flags) { - cfs_hash_t *hs; + struct cfs_hash *hs; int len; ENTRY; @@ -1027,7 +1013,7 @@ cfs_hash_create(char *name, unsigned cur_bits, unsigned max_bits, LASSERT(ops->hs_object); LASSERT(ops->hs_keycmp); LASSERT(ops->hs_get != NULL); - LASSERT(ops->hs_put_locked != NULL); + LASSERT(ops->hs_put != NULL || ops->hs_put_locked != NULL); if ((flags & CFS_HASH_REHASH) != 0) flags |= CFS_HASH_COUNTER; /* must have counter */ @@ -1043,19 +1029,18 @@ cfs_hash_create(char *name, unsigned cur_bits, unsigned max_bits, len = (flags & CFS_HASH_BIGNAME) == 0 ? CFS_HASH_NAME_LEN : CFS_HASH_BIGNAME_LEN; - LIBCFS_ALLOC(hs, offsetof(cfs_hash_t, hs_name[len])); + LIBCFS_ALLOC(hs, offsetof(struct cfs_hash, hs_name[len])); if (hs == NULL) RETURN(NULL); - strncpy(hs->hs_name, name, len); - hs->hs_name[len - 1] = '\0'; - hs->hs_flags = flags; + strlcpy(hs->hs_name, name, len); + hs->hs_flags = flags; - cfs_atomic_set(&hs->hs_refcount, 1); - cfs_atomic_set(&hs->hs_count, 0); + atomic_set(&hs->hs_refcount, 1); + atomic_set(&hs->hs_count, 0); - cfs_hash_lock_setup(hs); - cfs_hash_hlist_setup(hs); + cfs_hash_lock_setup(hs); + cfs_hash_hlist_setup(hs); hs->hs_cur_bits = (__u8)cur_bits; hs->hs_min_bits = (__u8)cur_bits; @@ -1065,8 +1050,7 @@ cfs_hash_create(char *name, unsigned cur_bits, unsigned max_bits, hs->hs_ops = ops; hs->hs_extra_bytes = extra_bytes; hs->hs_rehash_bits = 0; - cfs_wi_init(&hs->hs_rehash_wi, hs, - cfs_hash_rehash_worker, CFS_WI_SCHED_ANY); + cfs_wi_init(&hs->hs_rehash_wi, hs, cfs_hash_rehash_worker); cfs_hash_depth_wi_init(hs); if (cfs_hash_with_rehash(hs)) @@ -1077,26 +1061,26 @@ cfs_hash_create(char *name, unsigned cur_bits, unsigned max_bits, if (hs->hs_buckets != NULL) return hs; - LIBCFS_FREE(hs, offsetof(cfs_hash_t, hs_name[len])); + LIBCFS_FREE(hs, offsetof(struct cfs_hash, hs_name[len])); RETURN(NULL); } -CFS_EXPORT_SYMBOL(cfs_hash_create); +EXPORT_SYMBOL(cfs_hash_create); /** * Cleanup libcfs hash @hs. */ static void -cfs_hash_destroy(cfs_hash_t *hs) +cfs_hash_destroy(struct cfs_hash *hs) { - cfs_hlist_node_t *hnode; - cfs_hlist_node_t *pos; - cfs_hash_bd_t bd; - int i; - ENTRY; + struct hlist_node *hnode; + struct hlist_node *pos; + struct cfs_hash_bd bd; + int i; + ENTRY; - LASSERT(hs != NULL); - LASSERT(!cfs_hash_is_exiting(hs) && - !cfs_hash_is_iterating(hs)); + LASSERT(hs != NULL); + LASSERT(!cfs_hash_is_exiting(hs) && + !cfs_hash_is_iterating(hs)); /** * prohibit further rehashes, don't need any lock because @@ -1111,59 +1095,59 @@ cfs_hash_destroy(cfs_hash_t *hs) LASSERT(hs->hs_buckets != NULL && hs->hs_rehash_buckets == NULL); - cfs_hash_for_each_bucket(hs, &bd, i) { - cfs_hlist_head_t *hhead; + cfs_hash_for_each_bucket(hs, &bd, i) { + struct hlist_head *hhead; - LASSERT(bd.bd_bucket != NULL); - /* no need to take this lock, just for consistent code */ - cfs_hash_bd_lock(hs, &bd, 1); + LASSERT(bd.bd_bucket != NULL); + /* no need to take this lock, just for consistent code */ + cfs_hash_bd_lock(hs, &bd, 1); cfs_hash_bd_for_each_hlist(hs, &bd, hhead) { - cfs_hlist_for_each_safe(hnode, pos, hhead) { - LASSERTF(!cfs_hash_with_assert_empty(hs), - "hash %s bucket %u(%u) is not " - " empty: %u items left\n", - hs->hs_name, bd.bd_bucket->hsb_index, - bd.bd_offset, bd.bd_bucket->hsb_count); - /* can't assert key valicate, because we - * can interrupt rehash */ - cfs_hash_bd_del_locked(hs, &bd, hnode); - cfs_hash_exit(hs, hnode); - } - } - LASSERT(bd.bd_bucket->hsb_count == 0); - cfs_hash_bd_unlock(hs, &bd, 1); - cfs_cond_resched(); - } + hlist_for_each_safe(hnode, pos, hhead) { + LASSERTF(!cfs_hash_with_assert_empty(hs), + "hash %s bucket %u(%u) is not " + " empty: %u items left\n", + hs->hs_name, bd.bd_bucket->hsb_index, + bd.bd_offset, bd.bd_bucket->hsb_count); + /* can't assert key valicate, because we + * can interrupt rehash */ + cfs_hash_bd_del_locked(hs, &bd, hnode); + cfs_hash_exit(hs, hnode); + } + } + LASSERT(bd.bd_bucket->hsb_count == 0); + cfs_hash_bd_unlock(hs, &bd, 1); + cond_resched(); + } - LASSERT(cfs_atomic_read(&hs->hs_count) == 0); + LASSERT(atomic_read(&hs->hs_count) == 0); - cfs_hash_buckets_free(hs->hs_buckets, cfs_hash_bkt_size(hs), - 0, CFS_HASH_NBKT(hs)); - i = cfs_hash_with_bigname(hs) ? - CFS_HASH_BIGNAME_LEN : CFS_HASH_NAME_LEN; - LIBCFS_FREE(hs, offsetof(cfs_hash_t, hs_name[i])); + cfs_hash_buckets_free(hs->hs_buckets, cfs_hash_bkt_size(hs), + 0, CFS_HASH_NBKT(hs)); + i = cfs_hash_with_bigname(hs) ? + CFS_HASH_BIGNAME_LEN : CFS_HASH_NAME_LEN; + LIBCFS_FREE(hs, offsetof(struct cfs_hash, hs_name[i])); - EXIT; + EXIT; } -cfs_hash_t *cfs_hash_getref(cfs_hash_t *hs) +struct cfs_hash *cfs_hash_getref(struct cfs_hash *hs) { - if (cfs_atomic_inc_not_zero(&hs->hs_refcount)) - return hs; - return NULL; + if (atomic_inc_not_zero(&hs->hs_refcount)) + return hs; + return NULL; } -CFS_EXPORT_SYMBOL(cfs_hash_getref); +EXPORT_SYMBOL(cfs_hash_getref); -void cfs_hash_putref(cfs_hash_t *hs) +void cfs_hash_putref(struct cfs_hash *hs) { - if (cfs_atomic_dec_and_test(&hs->hs_refcount)) - cfs_hash_destroy(hs); + if (atomic_dec_and_test(&hs->hs_refcount)) + cfs_hash_destroy(hs); } -CFS_EXPORT_SYMBOL(cfs_hash_putref); +EXPORT_SYMBOL(cfs_hash_putref); static inline int -cfs_hash_rehash_bits(cfs_hash_t *hs) +cfs_hash_rehash_bits(struct cfs_hash *hs) { if (cfs_hash_with_no_lock(hs) || !cfs_hash_with_rehash(hs)) @@ -1200,10 +1184,10 @@ cfs_hash_rehash_bits(cfs_hash_t *hs) * - too many elements */ static inline int -cfs_hash_rehash_inline(cfs_hash_t *hs) +cfs_hash_rehash_inline(struct cfs_hash *hs) { - return !cfs_hash_with_nblk_change(hs) && - cfs_atomic_read(&hs->hs_count) < CFS_HASH_LOOP_HOG; + return !cfs_hash_with_nblk_change(hs) && + atomic_read(&hs->hs_count) < CFS_HASH_LOOP_HOG; } /** @@ -1211,12 +1195,12 @@ cfs_hash_rehash_inline(cfs_hash_t *hs) * ops->hs_get function will be called when the item is added. */ void -cfs_hash_add(cfs_hash_t *hs, const void *key, cfs_hlist_node_t *hnode) +cfs_hash_add(struct cfs_hash *hs, const void *key, struct hlist_node *hnode) { - cfs_hash_bd_t bd; + struct cfs_hash_bd bd; int bits; - LASSERT(cfs_hlist_unhashed(hnode)); + LASSERT(hlist_unhashed(hnode)); cfs_hash_lock(hs, 0); cfs_hash_bd_get_and_lock(hs, key, &bd, 1); @@ -1231,33 +1215,33 @@ cfs_hash_add(cfs_hash_t *hs, const void *key, cfs_hlist_node_t *hnode) if (bits > 0) cfs_hash_rehash(hs, cfs_hash_rehash_inline(hs)); } -CFS_EXPORT_SYMBOL(cfs_hash_add); +EXPORT_SYMBOL(cfs_hash_add); -static cfs_hlist_node_t * -cfs_hash_find_or_add(cfs_hash_t *hs, const void *key, - cfs_hlist_node_t *hnode, int noref) +static struct hlist_node * +cfs_hash_find_or_add(struct cfs_hash *hs, const void *key, + struct hlist_node *hnode, int noref) { - cfs_hlist_node_t *ehnode; - cfs_hash_bd_t bds[2]; - int bits = 0; + struct hlist_node *ehnode; + struct cfs_hash_bd bds[2]; + int bits = 0; - LASSERT(cfs_hlist_unhashed(hnode)); + LASSERTF(hlist_unhashed(hnode), "hnode = %p\n", hnode); - cfs_hash_lock(hs, 0); - cfs_hash_dual_bd_get_and_lock(hs, key, bds, 1); + cfs_hash_lock(hs, 0); + cfs_hash_dual_bd_get_and_lock(hs, key, bds, 1); - cfs_hash_key_validate(hs, key, hnode); - ehnode = cfs_hash_dual_bd_findadd_locked(hs, bds, key, - hnode, noref); - cfs_hash_dual_bd_unlock(hs, bds, 1); + cfs_hash_key_validate(hs, key, hnode); + ehnode = cfs_hash_dual_bd_findadd_locked(hs, bds, key, + hnode, noref); + cfs_hash_dual_bd_unlock(hs, bds, 1); - if (ehnode == hnode) /* new item added */ - bits = cfs_hash_rehash_bits(hs); - cfs_hash_unlock(hs, 0); - if (bits > 0) - cfs_hash_rehash(hs, cfs_hash_rehash_inline(hs)); + if (ehnode == hnode) /* new item added */ + bits = cfs_hash_rehash_bits(hs); + cfs_hash_unlock(hs, 0); + if (bits > 0) + cfs_hash_rehash(hs, cfs_hash_rehash_inline(hs)); - return ehnode; + return ehnode; } /** @@ -1266,12 +1250,13 @@ cfs_hash_find_or_add(cfs_hash_t *hs, const void *key, * Returns 0 on success or -EALREADY on key collisions. */ int -cfs_hash_add_unique(cfs_hash_t *hs, const void *key, cfs_hlist_node_t *hnode) +cfs_hash_add_unique(struct cfs_hash *hs, const void *key, + struct hlist_node *hnode) { - return cfs_hash_find_or_add(hs, key, hnode, 1) != hnode ? - -EALREADY : 0; + return cfs_hash_find_or_add(hs, key, hnode, 1) != hnode ? + -EALREADY : 0; } -CFS_EXPORT_SYMBOL(cfs_hash_add_unique); +EXPORT_SYMBOL(cfs_hash_add_unique); /** * Add item @hnode to libcfs hash @hs using @key. If this @key @@ -1280,14 +1265,14 @@ CFS_EXPORT_SYMBOL(cfs_hash_add_unique); * Otherwise ops->hs_get is called on the item which was added. */ void * -cfs_hash_findadd_unique(cfs_hash_t *hs, const void *key, - cfs_hlist_node_t *hnode) +cfs_hash_findadd_unique(struct cfs_hash *hs, const void *key, + struct hlist_node *hnode) { - hnode = cfs_hash_find_or_add(hs, key, hnode, 0); + hnode = cfs_hash_find_or_add(hs, key, hnode, 0); - return cfs_hash_object(hs, hnode); + return cfs_hash_object(hs, hnode); } -CFS_EXPORT_SYMBOL(cfs_hash_findadd_unique); +EXPORT_SYMBOL(cfs_hash_findadd_unique); /** * Delete item @hnode from the libcfs hash @hs using @key. The @key @@ -1297,19 +1282,24 @@ CFS_EXPORT_SYMBOL(cfs_hash_findadd_unique); * on the removed object. */ void * -cfs_hash_del(cfs_hash_t *hs, const void *key, cfs_hlist_node_t *hnode) +cfs_hash_del(struct cfs_hash *hs, const void *key, struct hlist_node *hnode) { void *obj = NULL; int bits = 0; - cfs_hash_bd_t bds[2]; + struct cfs_hash_bd bds[2]; cfs_hash_lock(hs, 0); cfs_hash_dual_bd_get_and_lock(hs, key, bds, 1); - if (bds[1].bd_bucket == NULL && hnode != NULL) - cfs_hash_bd_del_locked(hs, &bds[0], hnode); - else - hnode = cfs_hash_dual_bd_finddel_locked(hs, bds, key, hnode); + /* NB: do nothing if @hnode is not in hash table */ + if (hnode == NULL || !hlist_unhashed(hnode)) { + if (bds[1].bd_bucket == NULL && hnode != NULL) { + cfs_hash_bd_del_locked(hs, &bds[0], hnode); + } else { + hnode = cfs_hash_dual_bd_finddel_locked(hs, bds, + key, hnode); + } + } if (hnode != NULL) { obj = cfs_hash_object(hs, hnode); @@ -1323,7 +1313,7 @@ cfs_hash_del(cfs_hash_t *hs, const void *key, cfs_hlist_node_t *hnode) return obj; } -CFS_EXPORT_SYMBOL(cfs_hash_del); +EXPORT_SYMBOL(cfs_hash_del); /** * Delete item given @key in libcfs hash @hs. The first @key found in @@ -1332,11 +1322,11 @@ CFS_EXPORT_SYMBOL(cfs_hash_del); * will be returned and ops->hs_put is called on the removed object. */ void * -cfs_hash_del_key(cfs_hash_t *hs, const void *key) +cfs_hash_del_key(struct cfs_hash *hs, const void *key) { return cfs_hash_del(hs, key, NULL); } -CFS_EXPORT_SYMBOL(cfs_hash_del_key); +EXPORT_SYMBOL(cfs_hash_del_key); /** * Lookup an item using @key in the libcfs hash @hs and return it. @@ -1347,11 +1337,11 @@ CFS_EXPORT_SYMBOL(cfs_hash_del_key); * in the hash @hs NULL is returned. */ void * -cfs_hash_lookup(cfs_hash_t *hs, const void *key) +cfs_hash_lookup(struct cfs_hash *hs, const void *key) { void *obj = NULL; - cfs_hlist_node_t *hnode; - cfs_hash_bd_t bds[2]; + struct hlist_node *hnode; + struct cfs_hash_bd bds[2]; cfs_hash_lock(hs, 0); cfs_hash_dual_bd_get_and_lock(hs, key, bds, 0); @@ -1365,10 +1355,10 @@ cfs_hash_lookup(cfs_hash_t *hs, const void *key) return obj; } -CFS_EXPORT_SYMBOL(cfs_hash_lookup); +EXPORT_SYMBOL(cfs_hash_lookup); static void -cfs_hash_for_each_enter(cfs_hash_t *hs) +cfs_hash_for_each_enter(struct cfs_hash *hs) { LASSERT(!cfs_hash_is_exiting(hs)); @@ -1377,7 +1367,7 @@ cfs_hash_for_each_enter(cfs_hash_t *hs) /* * NB: it's race on cfs_has_t::hs_iterating, but doesn't matter * because it's just an unreliable signal to rehash-thread, - * rehash-thread will try to finsih rehash ASAP when seeing this. + * rehash-thread will try to finish rehash ASAP when seeing this. */ hs->hs_iterating = 1; @@ -1385,7 +1375,7 @@ cfs_hash_for_each_enter(cfs_hash_t *hs) hs->hs_iterators++; /* NB: iteration is mostly called by service thread, - * we tend to cancel pending rehash-requst, instead of + * we tend to cancel pending rehash-request, instead of * blocking service thread, we will relaunch rehash request * after iteration */ if (cfs_hash_is_rehashing(hs)) @@ -1394,24 +1384,24 @@ cfs_hash_for_each_enter(cfs_hash_t *hs) } static void -cfs_hash_for_each_exit(cfs_hash_t *hs) -{ - int remained; - int bits; - - if (!cfs_hash_with_rehash(hs)) - return; - cfs_hash_lock(hs, 1); - remained = --hs->hs_iterators; - bits = cfs_hash_rehash_bits(hs); - cfs_hash_unlock(hs, 1); - /* NB: it's race on cfs_has_t::hs_iterating, see above */ - if (remained == 0) - hs->hs_iterating = 0; - if (bits > 0) { - cfs_hash_rehash(hs, cfs_atomic_read(&hs->hs_count) < - CFS_HASH_LOOP_HOG); - } +cfs_hash_for_each_exit(struct cfs_hash *hs) +{ + int remained; + int bits; + + if (!cfs_hash_with_rehash(hs)) + return; + cfs_hash_lock(hs, 1); + remained = --hs->hs_iterators; + bits = cfs_hash_rehash_bits(hs); + cfs_hash_unlock(hs, 1); + /* NB: it's race on cfs_has_t::hs_iterating, see above */ + if (remained == 0) + hs->hs_iterating = 0; + if (bits > 0) { + cfs_hash_rehash(hs, atomic_read(&hs->hs_count) < + CFS_HASH_LOOP_HOG); + } } /** @@ -1425,69 +1415,69 @@ cfs_hash_for_each_exit(cfs_hash_t *hs) * cfs_hash_bd_del_locked */ static __u64 -cfs_hash_for_each_tight(cfs_hash_t *hs, cfs_hash_for_each_cb_t func, - void *data, int remove_safe) -{ - cfs_hlist_node_t *hnode; - cfs_hlist_node_t *pos; - cfs_hash_bd_t bd; - __u64 count = 0; - int excl = !!remove_safe; - int loop = 0; - int i; - ENTRY; - - cfs_hash_for_each_enter(hs); - - cfs_hash_lock(hs, 0); - LASSERT(!cfs_hash_is_rehashing(hs)); - - cfs_hash_for_each_bucket(hs, &bd, i) { - cfs_hlist_head_t *hhead; - - cfs_hash_bd_lock(hs, &bd, excl); - if (func == NULL) { /* only glimpse size */ - count += bd.bd_bucket->hsb_count; - cfs_hash_bd_unlock(hs, &bd, excl); - continue; - } - - cfs_hash_bd_for_each_hlist(hs, &bd, hhead) { - cfs_hlist_for_each_safe(hnode, pos, hhead) { - cfs_hash_bucket_validate(hs, &bd, hnode); - count++; - loop++; - if (func(hs, &bd, hnode, data)) { - cfs_hash_bd_unlock(hs, &bd, excl); - goto out; - } - } - } - cfs_hash_bd_unlock(hs, &bd, excl); - if (loop < CFS_HASH_LOOP_HOG) - continue; - loop = 0; - cfs_hash_unlock(hs, 0); - cfs_cond_resched(); - cfs_hash_lock(hs, 0); - } +cfs_hash_for_each_tight(struct cfs_hash *hs, cfs_hash_for_each_cb_t func, + void *data, int remove_safe) +{ + struct hlist_node *hnode; + struct hlist_node *pos; + struct cfs_hash_bd bd; + __u64 count = 0; + int excl = !!remove_safe; + int loop = 0; + int i; + ENTRY; + + cfs_hash_for_each_enter(hs); + + cfs_hash_lock(hs, 0); + LASSERT(!cfs_hash_is_rehashing(hs)); + + cfs_hash_for_each_bucket(hs, &bd, i) { + struct hlist_head *hhead; + + cfs_hash_bd_lock(hs, &bd, excl); + if (func == NULL) { /* only glimpse size */ + count += bd.bd_bucket->hsb_count; + cfs_hash_bd_unlock(hs, &bd, excl); + continue; + } + + cfs_hash_bd_for_each_hlist(hs, &bd, hhead) { + hlist_for_each_safe(hnode, pos, hhead) { + cfs_hash_bucket_validate(hs, &bd, hnode); + count++; + loop++; + if (func(hs, &bd, hnode, data)) { + cfs_hash_bd_unlock(hs, &bd, excl); + goto out; + } + } + } + cfs_hash_bd_unlock(hs, &bd, excl); + if (loop < CFS_HASH_LOOP_HOG) + continue; + loop = 0; + cfs_hash_unlock(hs, 0); + cond_resched(); + cfs_hash_lock(hs, 0); + } out: - cfs_hash_unlock(hs, 0); + cfs_hash_unlock(hs, 0); - cfs_hash_for_each_exit(hs); - RETURN(count); + cfs_hash_for_each_exit(hs); + RETURN(count); } -typedef struct { +struct cfs_hash_cond_arg { cfs_hash_cond_opt_cb_t func; void *arg; -} cfs_hash_cond_arg_t; +}; static int -cfs_hash_cond_del_locked(cfs_hash_t *hs, cfs_hash_bd_t *bd, - cfs_hlist_node_t *hnode, void *data) +cfs_hash_cond_del_locked(struct cfs_hash *hs, struct cfs_hash_bd *bd, + struct hlist_node *hnode, void *data) { - cfs_hash_cond_arg_t *cond = data; + struct cfs_hash_cond_arg *cond = data; if (cond->func(cfs_hash_object(hs, hnode), cond->arg)) cfs_hash_bd_del_locked(hs, bd, hnode); @@ -1500,59 +1490,59 @@ cfs_hash_cond_del_locked(cfs_hash_t *hs, cfs_hash_bd_t *bd, * any object be reference. */ void -cfs_hash_cond_del(cfs_hash_t *hs, cfs_hash_cond_opt_cb_t func, void *data) +cfs_hash_cond_del(struct cfs_hash *hs, cfs_hash_cond_opt_cb_t func, void *data) { - cfs_hash_cond_arg_t arg = { + struct cfs_hash_cond_arg arg = { .func = func, .arg = data, }; cfs_hash_for_each_tight(hs, cfs_hash_cond_del_locked, &arg, 1); } -CFS_EXPORT_SYMBOL(cfs_hash_cond_del); +EXPORT_SYMBOL(cfs_hash_cond_del); void -cfs_hash_for_each(cfs_hash_t *hs, +cfs_hash_for_each(struct cfs_hash *hs, cfs_hash_for_each_cb_t func, void *data) { cfs_hash_for_each_tight(hs, func, data, 0); } -CFS_EXPORT_SYMBOL(cfs_hash_for_each); +EXPORT_SYMBOL(cfs_hash_for_each); void -cfs_hash_for_each_safe(cfs_hash_t *hs, +cfs_hash_for_each_safe(struct cfs_hash *hs, cfs_hash_for_each_cb_t func, void *data) { cfs_hash_for_each_tight(hs, func, data, 1); } -CFS_EXPORT_SYMBOL(cfs_hash_for_each_safe); +EXPORT_SYMBOL(cfs_hash_for_each_safe); static int -cfs_hash_peek(cfs_hash_t *hs, cfs_hash_bd_t *bd, - cfs_hlist_node_t *hnode, void *data) +cfs_hash_peek(struct cfs_hash *hs, struct cfs_hash_bd *bd, + struct hlist_node *hnode, void *data) { - *(int *)data = 0; - return 1; /* return 1 to break the loop */ + *(int *)data = 0; + return 1; /* return 1 to break the loop */ } int -cfs_hash_is_empty(cfs_hash_t *hs) +cfs_hash_is_empty(struct cfs_hash *hs) { int empty = 1; cfs_hash_for_each_tight(hs, cfs_hash_peek, &empty, 0); return empty; } -CFS_EXPORT_SYMBOL(cfs_hash_is_empty); +EXPORT_SYMBOL(cfs_hash_is_empty); __u64 -cfs_hash_size_get(cfs_hash_t *hs) +cfs_hash_size_get(struct cfs_hash *hs) { - return cfs_hash_with_counter(hs) ? - cfs_atomic_read(&hs->hs_count) : - cfs_hash_for_each_tight(hs, NULL, NULL, 0); + return cfs_hash_with_counter(hs) ? + atomic_read(&hs->hs_count) : + cfs_hash_for_each_tight(hs, NULL, NULL, 0); } -CFS_EXPORT_SYMBOL(cfs_hash_size_get); +EXPORT_SYMBOL(cfs_hash_size_get); /* * cfs_hash_for_each_relax: @@ -1570,70 +1560,102 @@ CFS_EXPORT_SYMBOL(cfs_hash_size_get); * two cases, so iteration has to be stopped on change. */ static int -cfs_hash_for_each_relax(cfs_hash_t *hs, cfs_hash_for_each_cb_t func, void *data) -{ - cfs_hlist_node_t *hnode; - cfs_hlist_node_t *tmp; - cfs_hash_bd_t bd; - __u32 version; - int count = 0; - int stop_on_change; - int rc; - int i; - ENTRY; - - stop_on_change = cfs_hash_with_rehash_key(hs) || - !cfs_hash_with_no_itemref(hs) || - CFS_HOP(hs, put_locked) == NULL; - cfs_hash_lock(hs, 0); - LASSERT(!cfs_hash_is_rehashing(hs)); - - cfs_hash_for_each_bucket(hs, &bd, i) { - cfs_hlist_head_t *hhead; +cfs_hash_for_each_relax(struct cfs_hash *hs, cfs_hash_for_each_cb_t func, + void *data, int start) +{ + struct hlist_node *hnode; + struct hlist_node *next = NULL; + struct cfs_hash_bd bd; + __u32 version; + int count = 0; + int stop_on_change; + int has_put_locked; + int rc = 0; + int i, end = -1; + ENTRY; + + stop_on_change = cfs_hash_with_rehash_key(hs) || + !cfs_hash_with_no_itemref(hs); + has_put_locked = hs->hs_ops->hs_put_locked != NULL; + cfs_hash_lock(hs, 0); +again: + LASSERT(!cfs_hash_is_rehashing(hs)); + + cfs_hash_for_each_bucket(hs, &bd, i) { + struct hlist_head *hhead; + + if (i < start) + continue; + else if (end > 0 && i >= end) + break; cfs_hash_bd_lock(hs, &bd, 0); version = cfs_hash_bd_version_get(&bd); cfs_hash_bd_for_each_hlist(hs, &bd, hhead) { - for (hnode = hhead->first; hnode != NULL;) { - cfs_hash_bucket_validate(hs, &bd, hnode); - cfs_hash_get(hs, hnode); + hnode = hhead->first; + if (hnode == NULL) + continue; + cfs_hash_get(hs, hnode); + for (; hnode != NULL; hnode = next) { + cfs_hash_bucket_validate(hs, &bd, hnode); + next = hnode->next; + if (next != NULL) + cfs_hash_get(hs, next); cfs_hash_bd_unlock(hs, &bd, 0); cfs_hash_unlock(hs, 0); - rc = func(hs, &bd, hnode, data); - if (stop_on_change) - cfs_hash_put(hs, hnode); - cfs_cond_resched(); - count++; + rc = func(hs, &bd, hnode, data); + if (stop_on_change || !has_put_locked) + cfs_hash_put(hs, hnode); + + cond_resched(); + count++; cfs_hash_lock(hs, 0); cfs_hash_bd_lock(hs, &bd, 0); - if (!stop_on_change) { - tmp = hnode->next; - cfs_hash_put_locked(hs, hnode); - hnode = tmp; - } else { /* bucket changed? */ - if (version != - cfs_hash_bd_version_get(&bd)) - break; - /* safe to continue because no change */ - hnode = hnode->next; - } + if (stop_on_change) { + if (version != + cfs_hash_bd_version_get(&bd)) + rc = -EINTR; + } else if (has_put_locked) { + cfs_hash_put_locked(hs, hnode); + } if (rc) /* callback wants to break iteration */ break; } + if (next != NULL) { + if (has_put_locked) { + cfs_hash_put_locked(hs, next); + next = NULL; + } + break; + } else if (rc != 0) { + break; + } } cfs_hash_bd_unlock(hs, &bd, 0); + if (next != NULL && !has_put_locked) { + cfs_hash_put(hs, next); + next = NULL; + } + if (rc) /* callback wants to break iteration */ + break; } - cfs_hash_unlock(hs, 0); - return count; + if (start > 0 && rc == 0) { + end = start; + start = 0; + goto again; + } + + cfs_hash_unlock(hs, 0); + return count; } int -cfs_hash_for_each_nolock(cfs_hash_t *hs, - cfs_hash_for_each_cb_t func, void *data) +cfs_hash_for_each_nolock(struct cfs_hash *hs, + cfs_hash_for_each_cb_t func, void *data, int start) { ENTRY; @@ -1642,18 +1664,18 @@ cfs_hash_for_each_nolock(cfs_hash_t *hs, !cfs_hash_with_no_itemref(hs)) RETURN(-EOPNOTSUPP); - if (CFS_HOP(hs, get) == NULL || - (CFS_HOP(hs, put) == NULL && - CFS_HOP(hs, put_locked) == NULL)) - RETURN(-EOPNOTSUPP); + if (hs->hs_ops->hs_get == NULL || + (hs->hs_ops->hs_put == NULL && + hs->hs_ops->hs_put_locked == NULL)) + RETURN(-EOPNOTSUPP); - cfs_hash_for_each_enter(hs); - cfs_hash_for_each_relax(hs, func, data); - cfs_hash_for_each_exit(hs); + cfs_hash_for_each_enter(hs); + cfs_hash_for_each_relax(hs, func, data, start); + cfs_hash_for_each_exit(hs); - RETURN(0); + RETURN(0); } -CFS_EXPORT_SYMBOL(cfs_hash_for_each_nolock); +EXPORT_SYMBOL(cfs_hash_for_each_nolock); /** * For each hash bucket in the libcfs hash @hs call the passed callback @@ -1667,7 +1689,7 @@ CFS_EXPORT_SYMBOL(cfs_hash_for_each_nolock); * the required locking is in place to prevent concurrent insertions. */ int -cfs_hash_for_each_empty(cfs_hash_t *hs, +cfs_hash_for_each_empty(struct cfs_hash *hs, cfs_hash_for_each_cb_t func, void *data) { unsigned i = 0; @@ -1676,49 +1698,49 @@ cfs_hash_for_each_empty(cfs_hash_t *hs, if (cfs_hash_with_no_lock(hs)) return -EOPNOTSUPP; - if (CFS_HOP(hs, get) == NULL || - (CFS_HOP(hs, put) == NULL && - CFS_HOP(hs, put_locked) == NULL)) - return -EOPNOTSUPP; + if (hs->hs_ops->hs_get == NULL || + (hs->hs_ops->hs_put == NULL && + hs->hs_ops->hs_put_locked == NULL)) + return -EOPNOTSUPP; - cfs_hash_for_each_enter(hs); - while (cfs_hash_for_each_relax(hs, func, data)) { - CDEBUG(D_INFO, "Try to empty hash: %s, loop: %u\n", - hs->hs_name, i++); - } - cfs_hash_for_each_exit(hs); - RETURN(0); + cfs_hash_for_each_enter(hs); + while (cfs_hash_for_each_relax(hs, func, data, 0)) { + CDEBUG(D_INFO, "Try to empty hash: %s, loop: %u\n", + hs->hs_name, i++); + } + cfs_hash_for_each_exit(hs); + RETURN(0); } -CFS_EXPORT_SYMBOL(cfs_hash_for_each_empty); +EXPORT_SYMBOL(cfs_hash_for_each_empty); void -cfs_hash_hlist_for_each(cfs_hash_t *hs, unsigned hindex, - cfs_hash_for_each_cb_t func, void *data) +cfs_hash_hlist_for_each(struct cfs_hash *hs, unsigned hindex, + cfs_hash_for_each_cb_t func, void *data) { - cfs_hlist_head_t *hhead; - cfs_hlist_node_t *hnode; - cfs_hash_bd_t bd; + struct hlist_head *hhead; + struct hlist_node *hnode; + struct cfs_hash_bd bd; cfs_hash_for_each_enter(hs); cfs_hash_lock(hs, 0); if (hindex >= CFS_HASH_NHLIST(hs)) goto out; - cfs_hash_bd_index_set(hs, hindex, &bd); + cfs_hash_bd_index_set(hs, hindex, &bd); - cfs_hash_bd_lock(hs, &bd, 0); - hhead = cfs_hash_bd_hhead(hs, &bd); - cfs_hlist_for_each(hnode, hhead) { - if (func(hs, &bd, hnode, data)) - break; - } - cfs_hash_bd_unlock(hs, &bd, 0); - out: - cfs_hash_unlock(hs, 0); - cfs_hash_for_each_exit(hs); + cfs_hash_bd_lock(hs, &bd, 0); + hhead = cfs_hash_bd_hhead(hs, &bd); + hlist_for_each(hnode, hhead) { + if (func(hs, &bd, hnode, data)) + break; + } + cfs_hash_bd_unlock(hs, &bd, 0); +out: + cfs_hash_unlock(hs, 0); + cfs_hash_for_each_exit(hs); } -CFS_EXPORT_SYMBOL(cfs_hash_hlist_for_each); +EXPORT_SYMBOL(cfs_hash_hlist_for_each); /* * For each item in the libcfs hash @hs which matches the @key call @@ -1727,34 +1749,34 @@ CFS_EXPORT_SYMBOL(cfs_hash_hlist_for_each); * is held so the callback must never sleep. */ void -cfs_hash_for_each_key(cfs_hash_t *hs, const void *key, - cfs_hash_for_each_cb_t func, void *data) +cfs_hash_for_each_key(struct cfs_hash *hs, const void *key, + cfs_hash_for_each_cb_t func, void *data) { - cfs_hlist_node_t *hnode; - cfs_hash_bd_t bds[2]; - unsigned i; + struct hlist_node *hnode; + struct cfs_hash_bd bds[2]; + unsigned i; - cfs_hash_lock(hs, 0); + cfs_hash_lock(hs, 0); - cfs_hash_dual_bd_get_and_lock(hs, key, bds, 0); + cfs_hash_dual_bd_get_and_lock(hs, key, bds, 0); - cfs_hash_for_each_bd(bds, 2, i) { - cfs_hlist_head_t *hlist = cfs_hash_bd_hhead(hs, &bds[i]); + cfs_hash_for_each_bd(bds, 2, i) { + struct hlist_head *hlist = cfs_hash_bd_hhead(hs, &bds[i]); - cfs_hlist_for_each(hnode, hlist) { - cfs_hash_bucket_validate(hs, &bds[i], hnode); + hlist_for_each(hnode, hlist) { + cfs_hash_bucket_validate(hs, &bds[i], hnode); - if (cfs_hash_keycmp(hs, key, hnode)) { - if (func(hs, &bds[i], hnode, data)) - break; - } - } - } + if (cfs_hash_keycmp(hs, key, hnode)) { + if (func(hs, &bds[i], hnode, data)) + break; + } + } + } - cfs_hash_dual_bd_unlock(hs, bds, 0); - cfs_hash_unlock(hs, 0); + cfs_hash_dual_bd_unlock(hs, bds, 0); + cfs_hash_unlock(hs, 0); } -CFS_EXPORT_SYMBOL(cfs_hash_for_each_key); +EXPORT_SYMBOL(cfs_hash_for_each_key); /** * Rehash the libcfs hash @hs to the given @bits. This can be used @@ -1768,7 +1790,7 @@ CFS_EXPORT_SYMBOL(cfs_hash_for_each_key); * theta thresholds for @hs are tunable via cfs_hash_set_theta(). */ void -cfs_hash_rehash_cancel_locked(cfs_hash_t *hs) +cfs_hash_rehash_cancel_locked(struct cfs_hash *hs) { int i; @@ -1779,34 +1801,32 @@ cfs_hash_rehash_cancel_locked(cfs_hash_t *hs) if (!cfs_hash_is_rehashing(hs)) return; - if (cfs_wi_cancel(&hs->hs_rehash_wi)) { + if (cfs_wi_deschedule(cfs_sched_rehash, &hs->hs_rehash_wi)) { hs->hs_rehash_bits = 0; return; } for (i = 2; cfs_hash_is_rehashing(hs); i++) { - cfs_hash_unlock(hs, 1); - /* raise console warning while waiting too long */ - CDEBUG(IS_PO2(i >> 3) ? D_WARNING : D_INFO, - "hash %s is still rehashing, rescheded %d\n", - hs->hs_name, i - 1); - cfs_cond_resched(); - cfs_hash_lock(hs, 1); - } + cfs_hash_unlock(hs, 1); + /* raise console warning while waiting too long */ + CDEBUG(is_power_of_2(i >> 3) ? D_WARNING : D_INFO, + "hash %s is still rehashing, rescheded %d\n", + hs->hs_name, i - 1); + cond_resched(); + cfs_hash_lock(hs, 1); + } } -CFS_EXPORT_SYMBOL(cfs_hash_rehash_cancel_locked); void -cfs_hash_rehash_cancel(cfs_hash_t *hs) +cfs_hash_rehash_cancel(struct cfs_hash *hs) { cfs_hash_lock(hs, 1); cfs_hash_rehash_cancel_locked(hs); cfs_hash_unlock(hs, 1); } -CFS_EXPORT_SYMBOL(cfs_hash_rehash_cancel); int -cfs_hash_rehash(cfs_hash_t *hs, int do_rehash) +cfs_hash_rehash(struct cfs_hash *hs, int do_rehash) { int rc; @@ -1823,7 +1843,7 @@ cfs_hash_rehash(cfs_hash_t *hs, int do_rehash) hs->hs_rehash_bits = rc; if (!do_rehash) { /* launch and return */ - cfs_wi_schedule(&hs->hs_rehash_wi); + cfs_wi_schedule(cfs_sched_rehash, &hs->hs_rehash_wi); cfs_hash_unlock(hs, 1); return 0; } @@ -1833,53 +1853,52 @@ cfs_hash_rehash(cfs_hash_t *hs, int do_rehash) return cfs_hash_rehash_worker(&hs->hs_rehash_wi); } -CFS_EXPORT_SYMBOL(cfs_hash_rehash); static int -cfs_hash_rehash_bd(cfs_hash_t *hs, cfs_hash_bd_t *old) -{ - cfs_hash_bd_t new; - cfs_hlist_head_t *hhead; - cfs_hlist_node_t *hnode; - cfs_hlist_node_t *pos; - void *key; - int c = 0; - - /* hold cfs_hash_lock(hs, 1), so don't need any bucket lock */ - cfs_hash_bd_for_each_hlist(hs, old, hhead) { - cfs_hlist_for_each_safe(hnode, pos, hhead) { - key = cfs_hash_key(hs, hnode); - LASSERT(key != NULL); - /* Validate hnode is in the correct bucket. */ - cfs_hash_bucket_validate(hs, old, hnode); - /* - * Delete from old hash bucket; move to new bucket. - * ops->hs_key must be defined. - */ - cfs_hash_bd_from_key(hs, hs->hs_rehash_buckets, - hs->hs_rehash_bits, key, &new); - cfs_hash_bd_move_locked(hs, old, &new, hnode); - c++; - } - } - - return c; +cfs_hash_rehash_bd(struct cfs_hash *hs, struct cfs_hash_bd *old) +{ + struct cfs_hash_bd new; + struct hlist_head *hhead; + struct hlist_node *hnode; + struct hlist_node *pos; + void *key; + int c = 0; + + /* hold cfs_hash_lock(hs, 1), so don't need any bucket lock */ + cfs_hash_bd_for_each_hlist(hs, old, hhead) { + hlist_for_each_safe(hnode, pos, hhead) { + key = cfs_hash_key(hs, hnode); + LASSERT(key != NULL); + /* Validate hnode is in the correct bucket. */ + cfs_hash_bucket_validate(hs, old, hnode); + /* + * Delete from old hash bucket; move to new bucket. + * ops->hs_key must be defined. + */ + cfs_hash_bd_from_key(hs, hs->hs_rehash_buckets, + hs->hs_rehash_bits, key, &new); + cfs_hash_bd_move_locked(hs, old, &new, hnode); + c++; + } + } + return c; } static int -cfs_hash_rehash_worker(cfs_workitem_t *wi) -{ - cfs_hash_t *hs = container_of(wi, cfs_hash_t, hs_rehash_wi); - cfs_hash_bucket_t **bkts; - cfs_hash_bd_t bd; - unsigned int old_size; - unsigned int new_size; - int bsize; - int count = 0; - int rc = 0; - int i; +cfs_hash_rehash_worker(struct cfs_workitem *wi) +{ + struct cfs_hash *hs = + container_of(wi, struct cfs_hash, hs_rehash_wi); + struct cfs_hash_bucket **bkts; + struct cfs_hash_bd bd; + unsigned int old_size; + unsigned int new_size; + int bsize; + int count = 0; + int rc = 0; + int i; - LASSERT (hs != NULL && cfs_hash_with_rehash(hs)); + LASSERT(hs != NULL && cfs_hash_with_rehash(hs)); cfs_hash_lock(hs, 0); LASSERT(cfs_hash_is_rehashing(hs)); @@ -1938,11 +1957,11 @@ cfs_hash_rehash_worker(cfs_workitem_t *wi) continue; } - count = 0; - cfs_hash_unlock(hs, 1); - cfs_cond_resched(); - cfs_hash_lock(hs, 1); - } + count = 0; + cfs_hash_unlock(hs, 1); + cond_resched(); + cfs_hash_lock(hs, 1); + } hs->hs_rehash_count++; @@ -1953,17 +1972,17 @@ cfs_hash_rehash_worker(cfs_workitem_t *wi) hs->hs_cur_bits = hs->hs_rehash_bits; out: hs->hs_rehash_bits = 0; - if (rc == -ESRCH) - cfs_wi_exit(wi); /* never be scheduled again */ + if (rc == -ESRCH) /* never be scheduled again */ + cfs_wi_exit(cfs_sched_rehash, wi); bsize = cfs_hash_bkt_size(hs); cfs_hash_unlock(hs, 1); /* can't refer to @hs anymore because it could be destroyed */ if (bkts != NULL) cfs_hash_buckets_free(bkts, bsize, new_size, old_size); if (rc != 0) - CDEBUG(D_INFO, "early quit of of rehashing: %d\n", rc); - /* cfs_workitem require us to always return 0 */ - return 0; + CDEBUG(D_INFO, "early quit of rehashing: %d\n", rc); + /* return 1 only if cfs_wi_exit is called */ + return rc == -ESRCH; } /** @@ -1976,14 +1995,14 @@ cfs_hash_rehash_worker(cfs_workitem_t *wi) * the registered cfs_hash_get() and cfs_hash_put() functions will * not be called. */ -void cfs_hash_rehash_key(cfs_hash_t *hs, const void *old_key, - void *new_key, cfs_hlist_node_t *hnode) +void cfs_hash_rehash_key(struct cfs_hash *hs, const void *old_key, + void *new_key, struct hlist_node *hnode) { - cfs_hash_bd_t bds[3]; - cfs_hash_bd_t old_bds[2]; - cfs_hash_bd_t new_bd; + struct cfs_hash_bd bds[3]; + struct cfs_hash_bd old_bds[2]; + struct cfs_hash_bd new_bd; - LASSERT(!cfs_hlist_unhashed(hnode)); + LASSERT(!hlist_unhashed(hnode)); cfs_hash_lock(hs, 0); @@ -2007,25 +2026,22 @@ void cfs_hash_rehash_key(cfs_hash_t *hs, const void *old_key, } /* overwrite key inside locks, otherwise may screw up with * other operations, i.e: rehash */ - cfs_hash_keycpy(hs, new_key, hnode); + cfs_hash_keycpy(hs, hnode, new_key); cfs_hash_multi_bd_unlock(hs, bds, 3, 1); cfs_hash_unlock(hs, 0); } -CFS_EXPORT_SYMBOL(cfs_hash_rehash_key); +EXPORT_SYMBOL(cfs_hash_rehash_key); -int cfs_hash_debug_header(char *str, int size) +void cfs_hash_debug_header(struct seq_file *m) { - return snprintf(str, size, "%-*s%6s%6s%6s%6s%6s%6s%6s%7s%8s%8s%8s%s\n", - CFS_HASH_BIGNAME_LEN, - "name", "cur", "min", "max", "theta", "t-min", "t-max", - "flags", "rehash", "count", "maxdep", "maxdepb", - " distribution"); + seq_printf(m, "%-*s cur min max theta t-min t-max flags rehash count maxdep maxdepb distribution\n", + CFS_HASH_BIGNAME_LEN, "name"); } -CFS_EXPORT_SYMBOL(cfs_hash_debug_header); +EXPORT_SYMBOL(cfs_hash_debug_header); -static cfs_hash_bucket_t ** -cfs_hash_full_bkts(cfs_hash_t *hs) +static struct cfs_hash_bucket ** +cfs_hash_full_bkts(struct cfs_hash *hs) { /* NB: caller should hold hs->hs_rwlock if REHASH is set */ if (hs->hs_rehash_buckets == NULL) @@ -2037,7 +2053,7 @@ cfs_hash_full_bkts(cfs_hash_t *hs) } static unsigned int -cfs_hash_full_nbkt(cfs_hash_t *hs) +cfs_hash_full_nbkt(struct cfs_hash *hs) { /* NB: caller should hold hs->hs_rwlock if REHASH is set */ if (hs->hs_rehash_buckets == NULL) @@ -2048,77 +2064,60 @@ cfs_hash_full_nbkt(cfs_hash_t *hs) CFS_HASH_RH_NBKT(hs) : CFS_HASH_NBKT(hs); } -int cfs_hash_debug_str(cfs_hash_t *hs, char *str, int size) -{ - int dist[8] = { 0, }; - int maxdep = -1; - int maxdepb = -1; - int total = 0; - int c = 0; - int theta; - int i; - - if (str == NULL || size == 0) - return 0; - - cfs_hash_lock(hs, 0); - theta = __cfs_hash_theta(hs); - - c += snprintf(str + c, size - c, "%-*s ", - CFS_HASH_BIGNAME_LEN, hs->hs_name); - c += snprintf(str + c, size - c, "%5d ", 1 << hs->hs_cur_bits); - c += snprintf(str + c, size - c, "%5d ", 1 << hs->hs_min_bits); - c += snprintf(str + c, size - c, "%5d ", 1 << hs->hs_max_bits); - c += snprintf(str + c, size - c, "%d.%03d ", - __cfs_hash_theta_int(theta), - __cfs_hash_theta_frac(theta)); - c += snprintf(str + c, size - c, "%d.%03d ", - __cfs_hash_theta_int(hs->hs_min_theta), - __cfs_hash_theta_frac(hs->hs_min_theta)); - c += snprintf(str + c, size - c, "%d.%03d ", - __cfs_hash_theta_int(hs->hs_max_theta), - __cfs_hash_theta_frac(hs->hs_max_theta)); - c += snprintf(str + c, size - c, " 0x%02x ", hs->hs_flags); - c += snprintf(str + c, size - c, "%6d ", hs->hs_rehash_count); - - /* - * The distribution is a summary of the chained hash depth in - * each of the libcfs hash buckets. Each buckets hsb_count is - * divided by the hash theta value and used to generate a - * histogram of the hash distribution. A uniform hash will - * result in all hash buckets being close to the average thus - * only the first few entries in the histogram will be non-zero. - * If you hash function results in a non-uniform hash the will - * be observable by outlier bucks in the distribution histogram. - * - * Uniform hash distribution: 128/128/0/0/0/0/0/0 - * Non-Uniform hash distribution: 128/125/0/0/0/0/2/1 - */ - for (i = 0; i < cfs_hash_full_nbkt(hs); i++) { - cfs_hash_bd_t bd; - - bd.bd_bucket = cfs_hash_full_bkts(hs)[i]; - cfs_hash_bd_lock(hs, &bd, 0); - if (maxdep < bd.bd_bucket->hsb_depmax) { - maxdep = bd.bd_bucket->hsb_depmax; -#ifdef __KERNEL__ - maxdepb = cfs_ffz(~maxdep); -#endif - } - total += bd.bd_bucket->hsb_count; - dist[min(__cfs_fls(bd.bd_bucket->hsb_count/max(theta,1)),7)]++; - cfs_hash_bd_unlock(hs, &bd, 0); - } - - c += snprintf(str + c, size - c, "%7d ", total); - c += snprintf(str + c, size - c, "%7d ", maxdep); - c += snprintf(str + c, size - c, "%7d ", maxdepb); - for (i = 0; i < 8; i++) - c += snprintf(str + c, size - c, "%d%c", dist[i], - (i == 7) ? '\n' : '/'); - - cfs_hash_unlock(hs, 0); - - return c; -} -CFS_EXPORT_SYMBOL(cfs_hash_debug_str); +void cfs_hash_debug_str(struct cfs_hash *hs, struct seq_file *m) +{ + int dist[8] = { 0, }; + int maxdep = -1; + int maxdepb = -1; + int total = 0; + int theta; + int i; + + cfs_hash_lock(hs, 0); + theta = __cfs_hash_theta(hs); + + seq_printf(m, "%-*s %5d %5d %5d %d.%03d %d.%03d %d.%03d 0x%02x %6d ", + CFS_HASH_BIGNAME_LEN, hs->hs_name, + 1 << hs->hs_cur_bits, 1 << hs->hs_min_bits, + 1 << hs->hs_max_bits, + __cfs_hash_theta_int(theta), __cfs_hash_theta_frac(theta), + __cfs_hash_theta_int(hs->hs_min_theta), + __cfs_hash_theta_frac(hs->hs_min_theta), + __cfs_hash_theta_int(hs->hs_max_theta), + __cfs_hash_theta_frac(hs->hs_max_theta), + hs->hs_flags, hs->hs_rehash_count); + + /* + * The distribution is a summary of the chained hash depth in + * each of the libcfs hash buckets. Each buckets hsb_count is + * divided by the hash theta value and used to generate a + * histogram of the hash distribution. A uniform hash will + * result in all hash buckets being close to the average thus + * only the first few entries in the histogram will be non-zero. + * If you hash function results in a non-uniform hash the will + * be observable by outlier bucks in the distribution histogram. + * + * Uniform hash distribution: 128/128/0/0/0/0/0/0 + * Non-Uniform hash distribution: 128/125/0/0/0/0/2/1 + */ + for (i = 0; i < cfs_hash_full_nbkt(hs); i++) { + struct cfs_hash_bd bd; + + bd.bd_bucket = cfs_hash_full_bkts(hs)[i]; + cfs_hash_bd_lock(hs, &bd, 0); + if (maxdep < bd.bd_bucket->hsb_depmax) { + maxdep = bd.bd_bucket->hsb_depmax; + maxdepb = ffz(~maxdep); + } + total += bd.bd_bucket->hsb_count; + dist[min(fls(bd.bd_bucket->hsb_count / max(theta, 1)), 7)]++; + cfs_hash_bd_unlock(hs, &bd, 0); + } + + seq_printf(m, "%7d %7d %7d ", total, maxdep, maxdepb); + for (i = 0; i < 8; i++) + seq_printf(m, "%d%c", dist[i], (i == 7) ? '\n' : '/'); + + cfs_hash_unlock(hs, 0); +} +EXPORT_SYMBOL(cfs_hash_debug_str);