Whamcloud - gitweb
b=11089
[fs/lustre-release.git] / lustre / obdclass / class_hash.c
1 /* -*- mode: c; c-basic-offset: 8; indent-tabs-mode: nil; -*-
2  * vim:expandtab:shiftwidth=8:tabstop=8:
3  *
4  *  Copyright (C) 2005 Cluster File Systems, Inc.
5  *   Author: YuZhangyong <yzy@clusterfs.com>
6  *
7  *   This file is part of Lustre, http://www.lustre.org/
8  *
9  *   No redistribution or use is permitted outside of Cluster File Systems, Inc.
10  *
11  *   Implement a hash class for hash process in lustre system.
12  */
13
14 #ifndef __KERNEL__
15 #include <liblustre.h>
16 #include <obd.h>
17 #endif
18
19 #include <obd_class.h>
20 #include <class_hash.h>
21 #include <lustre_export.h>
22 #include <obd_support.h>
23 #include <lustre_net.h>
24
25 int lustre_hash_init(struct lustre_class_hash_body **hash_body_new, 
26                      char *hashname, __u32 hashsize, 
27                      struct lustre_hash_operations *hash_operations)
28 {
29         int i, n = 0;
30         struct lustre_class_hash_body *hash_body = NULL;
31
32         LASSERT(hashsize > 0);
33         LASSERT(hash_operations != NULL);
34         ENTRY;
35
36         i = hashsize;
37         while (i != 0) {
38                 if (i & 0x1)
39                         n++;
40                 i >>= 1;
41         }
42
43         LASSERTF(n == 1, "hashsize %u isn't 2^n\n", hashsize);
44
45         /* alloc space for hash_body */   
46         OBD_ALLOC(hash_body, sizeof(*hash_body)); 
47
48         if (hash_body == NULL) {
49                 CERROR("Cannot alloc space for hash body, hashname = %s \n", 
50                         hashname);
51                 RETURN(-ENOMEM);
52         }
53
54         LASSERT(hashname != NULL && 
55                 strlen(hashname) <= sizeof(hash_body->hashname));
56         strcpy(hash_body->hashname, hashname);
57         hash_body->lchb_hash_max_size = hashsize;      
58         hash_body->lchb_hash_operations = hash_operations;  
59
60         /* alloc space for the hash tables */
61         OBD_ALLOC(hash_body->lchb_hash_tables, 
62                   sizeof(*hash_body->lchb_hash_tables) * hash_body->lchb_hash_max_size);
63
64         if (hash_body->lchb_hash_tables == NULL) {
65                 OBD_FREE(hash_body, sizeof(*hash_body)); 
66                 CERROR("Cannot alloc space for hashtables, hashname = %s \n", 
67                         hash_body->hashname);
68                 RETURN(-ENOMEM);
69         }
70
71         spin_lock_init(&hash_body->lchb_lock); /* initialize the body lock */
72
73         for(i = 0 ; i < hash_body->lchb_hash_max_size; i++) {
74                 /* initial the bucket lock and list_head */
75                 INIT_HLIST_HEAD(&hash_body->lchb_hash_tables[i].lhb_head);
76                 spin_lock_init(&hash_body->lchb_hash_tables[i].lhb_lock);
77         }
78         *hash_body_new = hash_body;
79
80         RETURN(0);
81 }
82 EXPORT_SYMBOL(lustre_hash_init);
83
84 void lustre_hash_exit(struct lustre_class_hash_body **new_hash_body)
85 {
86         int i;
87         struct lustre_class_hash_body *hash_body = NULL;
88         ENTRY;
89
90         hash_body = *new_hash_body;
91
92         if (hash_body == NULL) {
93                 CWARN("hash body has been deleted\n");
94                 goto out_hash;
95         }
96
97         spin_lock(&hash_body->lchb_lock); /* lock the hash tables */
98
99         if (hash_body->lchb_hash_tables == NULL ) {
100                 spin_unlock(&hash_body->lchb_lock);
101                 CWARN("hash tables has been deleted\n");
102                 goto out_hash;   
103         }
104
105         for( i = 0; i < hash_body->lchb_hash_max_size; i++ ) {
106                 struct lustre_hash_bucket * bucket;
107                 struct hlist_node * actual_hnode, *pos;
108
109                 bucket = &hash_body->lchb_hash_tables[i];
110                 spin_lock(&bucket->lhb_lock); /* lock the bucket */
111                 hlist_for_each_safe(actual_hnode, pos, &(bucket->lhb_head)) {
112                         lustre_hash_delitem_nolock(hash_body, i, actual_hnode);
113                 }
114                 spin_unlock(&bucket->lhb_lock); 
115         }
116
117         /* free the hash_tables's memory space */
118         OBD_FREE(hash_body->lchb_hash_tables,
119                   sizeof(*hash_body->lchb_hash_tables) * hash_body->lchb_hash_max_size);     
120
121         hash_body->lchb_hash_tables = NULL;
122
123         spin_unlock(&hash_body->lchb_lock);
124
125 out_hash : 
126         /* free the hash_body's memory space */
127         if (hash_body != NULL) {
128                 OBD_FREE(hash_body, sizeof(*hash_body));
129                 *new_hash_body = NULL;
130         }
131         EXIT;
132 }
133 EXPORT_SYMBOL(lustre_hash_exit);
134
135 /*
136  * only allow unique @key in hashtables, if the same @key has existed 
137  * in hashtables, it will return with fails.
138  */
139 int lustre_hash_additem_unique(struct lustre_class_hash_body *hash_body, 
140                                void *key, struct hlist_node *actual_hnode)
141 {
142         int hashent;
143         struct lustre_hash_bucket *bucket = NULL;
144         struct lustre_hash_operations *hop = hash_body->lchb_hash_operations;
145         ENTRY;
146
147         LASSERT(hlist_unhashed(actual_hnode));
148         hashent = hop->lustre_hashfn(hash_body, key);
149
150         /* get the hash-bucket and lock it */
151         bucket = &hash_body->lchb_hash_tables[hashent];
152         spin_lock(&bucket->lhb_lock);
153
154         if ( (lustre_hash_getitem_in_bucket_nolock(hash_body, hashent, key)) != NULL) {
155                 /* the added-item exist in hashtables, so cannot add it again */
156                 spin_unlock(&bucket->lhb_lock);
157
158                 CWARN("Already found the key in hash [%s]\n", 
159                       hash_body->hashname);
160                 RETURN(-EALREADY);
161         }
162
163         hlist_add_head(actual_hnode, &(bucket->lhb_head));
164
165 #ifdef LUSTRE_HASH_DEBUG
166         /* hash distribute debug */
167         hash_body->lchb_hash_tables[hashent].lhb_item_count++; 
168         CDEBUG(D_INFO, "hashname[%s] bucket[%d] has [%d] hashitem\n", 
169                         hash_body->hashname, hashent, 
170                         hash_body->lchb_hash_tables[hashent].lhb_item_count);
171 #endif  
172         hop->lustre_hash_object_refcount_get(actual_hnode); 
173
174         spin_unlock(&bucket->lhb_lock);
175
176         RETURN(0);
177 }
178 EXPORT_SYMBOL(lustre_hash_additem_unique);
179
180 /*
181  * only allow unique @key in hashtables, if the same @key has existed
182  * in hashtables, it will return with fails.
183  */
184 void* lustre_hash_findadd_unique(struct lustre_class_hash_body *hash_body,
185                                      void *key, struct hlist_node *actual_hnode)
186 {
187         int hashent;
188         struct lustre_hash_bucket *bucket = NULL;
189         struct lustre_hash_operations *hop = hash_body->lchb_hash_operations;
190         struct hlist_node * hash_item_hnode = NULL;
191         void *obj;
192         ENTRY;
193
194         LASSERT(hlist_unhashed(actual_hnode));
195         hashent = hop->lustre_hashfn(hash_body, key);
196
197         /* get the hash-bucket and lock it */
198         bucket = &hash_body->lchb_hash_tables[hashent];
199         spin_lock(&bucket->lhb_lock);
200
201         hash_item_hnode = lustre_hash_getitem_in_bucket_nolock(hash_body,
202                                                                hashent, key);
203         if ( hash_item_hnode != NULL) {
204                 /* the added-item exist in hashtables, so cannot add it again */
205                 obj = hop->lustre_hash_object_refcount_get(hash_item_hnode);
206                 spin_unlock(&bucket->lhb_lock);
207                 RETURN(obj);
208         }
209
210         hlist_add_head(actual_hnode, &(bucket->lhb_head));
211
212 #ifdef LUSTRE_HASH_DEBUG
213         /* hash distribute debug */
214         hash_body->lchb_hash_tables[hashent].lhb_item_count++;
215         CDEBUG(D_INFO, "hashname[%s] bucket[%d] has [%d] hashitem\n",
216                         hash_body->hashname, hashent,
217                         hash_body->lchb_hash_tables[hashent].lhb_item_count);
218 #endif
219         obj = hop->lustre_hash_object_refcount_get(actual_hnode);
220
221         spin_unlock(&bucket->lhb_lock);
222
223         RETURN(obj);
224 }
225 EXPORT_SYMBOL(lustre_hash_findadd_unique);
226
227 /*
228  * this version of additem, it allow multi same @key <key, value> in hashtables. 
229  * in this additem version, we don't need to check if exist same @key in hash 
230  * tables, we only add it to related hashbucket.
231  * example: maybe same nid will be related to multi difference export
232  */
233 int lustre_hash_additem(struct lustre_class_hash_body *hash_body, void *key, 
234                          struct hlist_node *actual_hnode)
235 {
236         int hashent;
237         struct lustre_hash_bucket *bucket = NULL;
238         struct lustre_hash_operations *hop = hash_body->lchb_hash_operations;
239         ENTRY;
240
241         LASSERT(hlist_unhashed(actual_hnode));
242
243         hashent = hop->lustre_hashfn(hash_body, key);
244
245         /* get the hashbucket and lock it */
246         bucket = &hash_body->lchb_hash_tables[hashent];
247         spin_lock(&bucket->lhb_lock);
248
249         hlist_add_head(actual_hnode, &(bucket->lhb_head));
250
251 #ifdef LUSTRE_HASH_DEBUG
252         /* hash distribute debug */
253         hash_body->lchb_hash_tables[hashent].lhb_item_count++; 
254         CDEBUG(D_INFO, "hashname[%s] bucket[%d] has [%d] hashitem\n", 
255                         hash_body->hashname, hashent, 
256                         hash_body->lchb_hash_tables[hashent].lhb_item_count);
257 #endif  
258         hop->lustre_hash_object_refcount_get(actual_hnode); 
259
260         spin_unlock(&bucket->lhb_lock);
261
262         RETURN(0);
263 }
264 EXPORT_SYMBOL(lustre_hash_additem);
265
266
267 /*
268  * this version of delitem will delete a hashitem with given @key, 
269  * we need to search the <@key, @value> in hashbucket with @key, 
270  * if match, the hashitem will be delete. 
271  * we have a no-search version of delitem, it will directly delete a hashitem, 
272  * doesn't need to search it in hashtables, so it is a O(1) delete.
273  */
274 int lustre_hash_delitem_by_key(struct lustre_class_hash_body *hash_body, 
275                                void *key)
276 {
277         int hashent ;
278         struct hlist_node * hash_item;
279         struct lustre_hash_bucket *bucket = NULL;
280         struct lustre_hash_operations *hop = hash_body->lchb_hash_operations;
281         int retval = 0;
282         ENTRY;
283
284         hashent = hop->lustre_hashfn(hash_body, key);
285
286         /* first, lock the hashbucket */
287         bucket = &hash_body->lchb_hash_tables[hashent];
288         spin_lock(&bucket->lhb_lock);
289
290         /* get the hash_item from hash_bucket */
291         hash_item = lustre_hash_getitem_in_bucket_nolock(hash_body, hashent, 
292                                                          key);
293
294         if (hash_item == NULL) {
295                 spin_unlock(&bucket->lhb_lock);
296                 RETURN(-ENOENT);
297         }
298
299         /* call delitem_nolock() to delete the hash_item */
300         retval = lustre_hash_delitem_nolock(hash_body, hashent, hash_item);
301
302         spin_unlock(&bucket->lhb_lock);
303
304         RETURN(retval);
305 }
306 EXPORT_SYMBOL(lustre_hash_delitem_by_key);
307
308 /*
309  * the O(1) version of delete hash item, 
310  * it will directly delete the hashitem with given @hash_item,
311  * the parameter @key used to get the relation hash bucket and lock it.
312  */
313 int lustre_hash_delitem(struct lustre_class_hash_body *hash_body, 
314                         void *key, struct hlist_node * hash_item)
315 {  
316         int hashent = 0;
317         int retval = 0;
318         struct lustre_hash_bucket *bucket = NULL;
319         struct lustre_hash_operations *hop = hash_body->lchb_hash_operations;
320         ENTRY;
321
322         hashent = hop->lustre_hashfn(hash_body, key);
323
324         bucket = &hash_body->lchb_hash_tables[hashent];
325         spin_lock(&bucket->lhb_lock);
326
327         /* call delitem_nolock() to delete the hash_item */
328         retval = lustre_hash_delitem_nolock(hash_body, hashent, hash_item);
329
330         spin_unlock(&bucket->lhb_lock);
331
332         RETURN(retval);
333 }
334 EXPORT_SYMBOL(lustre_hash_delitem);
335
336 void lustre_hash_bucket_iterate(struct lustre_class_hash_body *hash_body,
337                                 void *key, hash_item_iterate_cb func, void *data)
338 {
339         int hashent, find = 0;
340         struct lustre_hash_bucket *bucket = NULL;
341         struct hlist_node *hash_item_node = NULL;
342         struct lustre_hash_operations *hop = hash_body->lchb_hash_operations;
343         struct obd_export *tmp = NULL;
344
345         ENTRY;
346
347         hashent = hop->lustre_hashfn(hash_body, key);
348         bucket = &hash_body->lchb_hash_tables[hashent];
349
350         spin_lock(&bucket->lhb_lock);
351         hlist_for_each(hash_item_node, &(bucket->lhb_head)) {
352                 find = hop->lustre_hash_key_compare(key, hash_item_node);
353                 if (find) {
354                         tmp = hop->lustre_hash_object_refcount_get(hash_item_node);
355                         func(tmp, data);
356                         hop->lustre_hash_object_refcount_put(hash_item_node);
357                 }
358         }
359         spin_unlock(&bucket->lhb_lock);
360 }
361 EXPORT_SYMBOL(lustre_hash_bucket_iterate);
362
363 void lustre_hash_iterate_all(struct lustre_class_hash_body *hash_body,
364                             hash_item_iterate_cb func, void *data)
365 {
366         int i;
367         struct lustre_hash_operations *hop = hash_body->lchb_hash_operations;
368         ENTRY;
369
370         for( i = 0; i < hash_body->lchb_hash_max_size; i++ ) {
371                 struct lustre_hash_bucket * bucket;
372                 struct hlist_node * actual_hnode, *pos;
373                 void *obj;
374
375                 bucket = &hash_body->lchb_hash_tables[i];
376 #ifdef LUSTRE_HASH_DEBUG
377                 CDEBUG(D_INFO, "idx %d - bucket %p\n", i, bucket);
378 #endif
379                 spin_lock(&bucket->lhb_lock); /* lock the bucket */
380                 hlist_for_each_safe(actual_hnode, pos, &(bucket->lhb_head)) {
381                         obj = hop->lustre_hash_object_refcount_get(actual_hnode);
382                         func(obj, data);
383                         hop->lustre_hash_object_refcount_put(actual_hnode);
384                 }
385                 spin_unlock(&bucket->lhb_lock);
386         }
387         EXIT;
388 }
389 EXPORT_SYMBOL(lustre_hash_iterate_all);
390
391
392 void * lustre_hash_get_object_by_key(struct lustre_class_hash_body *hash_body,
393                                      void *key)
394 {
395         int hashent ;
396         struct hlist_node * hash_item_hnode = NULL;
397         void * obj_value = NULL;
398         struct lustre_hash_bucket *bucket = NULL;
399         struct lustre_hash_operations * hop = hash_body->lchb_hash_operations;
400         ENTRY;
401
402         /* get the hash value from the given item */
403         hashent = hop->lustre_hashfn(hash_body, key);
404
405         bucket = &hash_body->lchb_hash_tables[hashent];
406         spin_lock(&bucket->lhb_lock); /* lock the bucket */
407
408         hash_item_hnode = lustre_hash_getitem_in_bucket_nolock(hash_body, 
409                                                                hashent, key);
410
411         if (hash_item_hnode == NULL) {
412                 spin_unlock(&bucket->lhb_lock); /* lock the bucket */
413                 RETURN(NULL);
414         }
415
416         obj_value = hop->lustre_hash_object_refcount_get(hash_item_hnode);
417         spin_unlock(&bucket->lhb_lock); /* lock the bucket */
418
419         RETURN(obj_value);
420 }
421 EXPORT_SYMBOL(lustre_hash_get_object_by_key);
422
423 /* string hashing using djb2 hash algorithm */
424 __u32 djb2_hashfn(struct lustre_class_hash_body *hash_body,  void* key,
425                   size_t size)
426 {
427         __u32 hash = 5381;
428         int i;
429         char *ptr = key;
430
431         LASSERT(key != NULL);
432
433         for( i = 0; i < size; i++ )
434                 hash = hash * 33 + ptr[i];
435
436         hash &= (hash_body->lchb_hash_max_size - 1);
437
438         RETURN(hash);
439 }
440
441 /*
442  * define (uuid <-> export) hash operations and function define
443  */
444
445 /* define the uuid hash operations */
446 struct lustre_hash_operations uuid_hash_operations = {
447         .lustre_hashfn = uuid_hashfn,
448         .lustre_hash_key_compare = uuid_hash_key_compare,
449         .lustre_hash_object_refcount_get = uuid_export_refcount_get,
450         .lustre_hash_object_refcount_put = uuid_export_refcount_put,
451 };
452
453 __u32 uuid_hashfn(struct lustre_class_hash_body *hash_body,  void * key)
454 {
455         struct obd_uuid * uuid_key = key;
456
457         return djb2_hashfn(hash_body, uuid_key->uuid, sizeof(uuid_key->uuid));
458 }
459
460 /* Note, it is impossible to find an export that is in failed state with
461  * this function */
462 int uuid_hash_key_compare(void *key, struct hlist_node *compared_hnode)
463 {
464         struct obd_export *export = NULL;
465         struct obd_uuid *uuid_key = NULL, *compared_uuid = NULL;
466
467         LASSERT( key != NULL);
468
469         uuid_key = (struct obd_uuid*)key;
470
471         export = hlist_entry(compared_hnode, struct obd_export, exp_uuid_hash);
472
473         compared_uuid = &export->exp_client_uuid;
474
475         RETURN(obd_uuid_equals(uuid_key, compared_uuid) &&
476                !export->exp_failed);
477 }
478
479 void * uuid_export_refcount_get(struct hlist_node * actual_hnode)
480 {
481         struct obd_export *export = NULL;
482
483         LASSERT(actual_hnode != NULL);
484
485         export = hlist_entry(actual_hnode, struct obd_export, exp_uuid_hash);
486
487         LASSERT(export != NULL);
488
489         class_export_get(export);
490
491         RETURN(export);
492 }
493
494 void uuid_export_refcount_put(struct hlist_node * actual_hnode)
495 {
496         struct obd_export *export = NULL;
497
498         LASSERT(actual_hnode != NULL);
499
500         export = hlist_entry(actual_hnode, struct obd_export, exp_uuid_hash);
501
502         LASSERT(export != NULL);
503
504         class_export_put(export);
505 }
506
507 /*
508  * define (nid <-> export) hash operations and function define
509  */
510
511 /* define the nid hash operations */
512 struct lustre_hash_operations nid_hash_operations = {
513         .lustre_hashfn = nid_hashfn,
514         .lustre_hash_key_compare = nid_hash_key_compare,
515         .lustre_hash_object_refcount_get = nid_export_refcount_get,
516         .lustre_hash_object_refcount_put = nid_export_refcount_put,
517 };
518
519 __u32 nid_hashfn(struct lustre_class_hash_body *hash_body,  void * key)
520 {
521         return djb2_hashfn(hash_body, key, sizeof(lnet_nid_t));
522 }
523
524 /* Note, it is impossible to find an export that is in failed state with
525  * this function */
526 int nid_hash_key_compare(void *key, struct hlist_node *compared_hnode)
527 {
528         struct obd_export *export = NULL;
529         lnet_nid_t *nid_key = NULL;
530
531         LASSERT( key != NULL);
532
533         nid_key = (lnet_nid_t*)key;
534
535         export = hlist_entry(compared_hnode, struct obd_export, exp_nid_hash);
536
537         return (export->exp_connection->c_peer.nid == *nid_key &&
538                 !export->exp_failed);
539 }
540
541 void *nid_export_refcount_get(struct hlist_node *actual_hnode)
542 {
543         struct obd_export *export = NULL;
544
545         LASSERT(actual_hnode != NULL);
546
547         export = hlist_entry(actual_hnode, struct obd_export, exp_nid_hash);
548
549         LASSERT(export != NULL);
550
551         class_export_get(export);
552
553         RETURN(export);
554 }
555
556 void nid_export_refcount_put(struct hlist_node *actual_hnode)
557 {
558         struct obd_export *export = NULL;
559
560         LASSERT(actual_hnode != NULL);
561
562         export = hlist_entry(actual_hnode, struct obd_export, exp_nid_hash);
563
564         LASSERT(export != NULL);
565
566         class_export_put(export);
567 }
568
569 /*
570  * define (net_peer <-> connection) hash operations and function define
571  */
572
573 /* define the conn hash operations */
574 struct lustre_hash_operations conn_hash_operations = {
575         .lustre_hashfn = conn_hashfn,
576         .lustre_hash_key_compare = conn_hash_key_compare,
577         .lustre_hash_object_refcount_get = conn_refcount_get,
578         .lustre_hash_object_refcount_put = conn_refcount_put,
579 };
580 EXPORT_SYMBOL(conn_hash_operations);
581
582 __u32 conn_hashfn(struct lustre_class_hash_body *hash_body,  void * key)
583 {
584         return djb2_hashfn(hash_body, key, sizeof(lnet_process_id_t));
585 }
586
587 int conn_hash_key_compare(void *key, struct hlist_node *compared_hnode)
588 {
589         struct ptlrpc_connection *c = NULL;
590         lnet_process_id_t *conn_key = NULL;
591
592         LASSERT( key != NULL);
593
594         conn_key = (lnet_process_id_t*)key;
595
596         c = hlist_entry(compared_hnode, struct ptlrpc_connection, c_hash);
597
598         return (conn_key->nid == c->c_peer.nid &&
599                 conn_key->pid == c->c_peer.pid);
600 }
601
602 void *conn_refcount_get(struct hlist_node *actual_hnode)
603 {
604         struct ptlrpc_connection *c = NULL;
605
606         LASSERT(actual_hnode != NULL);
607
608         c = hlist_entry(actual_hnode, struct ptlrpc_connection, c_hash);
609
610         LASSERT(c != NULL);
611
612         atomic_inc(&c->c_refcount);
613
614         RETURN(c);
615 }
616
617 void conn_refcount_put(struct hlist_node *actual_hnode)
618 {
619         struct ptlrpc_connection *c = NULL;
620
621         LASSERT(actual_hnode != NULL);
622
623         c = hlist_entry(actual_hnode, struct ptlrpc_connection, c_hash);
624
625         LASSERT(c != NULL);
626
627         atomic_dec(&c->c_refcount);
628 }
629
630 /*******************************************************************************/
631 /* ( nid<>nidstats ) hash operations define */
632
633 struct lustre_hash_operations nid_stat_hash_operations = {
634         .lustre_hashfn = nid_hashfn,
635         .lustre_hash_key_compare = nidstats_hash_key_compare,
636         .lustre_hash_object_refcount_get = nidstats_refcount_get,
637         .lustre_hash_object_refcount_put = nidstats_refcount_put,
638 };
639 EXPORT_SYMBOL(nid_stat_hash_operations);
640
641 int nidstats_hash_key_compare(void *key, struct hlist_node * compared_hnode)
642 {
643         struct nid_stat *data;
644         lnet_nid_t *nid_key;
645
646         LASSERT( key != NULL);
647
648         nid_key = (lnet_nid_t*)key;
649         data = hlist_entry(compared_hnode, struct nid_stat, nid_hash);
650
651         return (data->nid == *nid_key);
652 }
653
654 void* nidstats_refcount_get(struct hlist_node * actual_hnode)
655 {
656         struct nid_stat *data;
657
658         data = hlist_entry(actual_hnode, struct nid_stat, nid_hash);
659         data->nid_exp_ref_count++;
660
661         RETURN(data);
662 }
663
664 void nidstats_refcount_put(struct hlist_node * actual_hnode)
665 {
666         struct nid_stat *data;
667
668         data = hlist_entry(actual_hnode, struct nid_stat, nid_hash);
669         data->nid_exp_ref_count--;
670
671 }
672
673 /*******************************************************************************/