X-Git-Url: https://git.whamcloud.com/?p=fs%2Flustre-release.git;a=blobdiff_plain;f=lustre%2Fptlrpc%2Fconnection.c;h=c87569e56e3a2bc8375b07fdf8bdf0cd740f9885;hp=f0c625dcc1d08897f02f10e521def79f1be33240;hb=89836033e22f37d021fae878144da479b741caad;hpb=6869932b552ac705f411de3362f01bd50c1f6f7d diff --git a/lustre/ptlrpc/connection.c b/lustre/ptlrpc/connection.c index f0c625d..c87569e 100644 --- a/lustre/ptlrpc/connection.c +++ b/lustre/ptlrpc/connection.c @@ -26,7 +26,7 @@ * GPL HEADER END */ /* - * Copyright 2008 Sun Microsystems, Inc. All rights reserved + * Copyright (c) 2002, 2010, Oracle and/or its affiliates. All rights reserved. * Use is subject to license terms. */ /* @@ -44,210 +44,201 @@ #endif #include "ptlrpc_internal.h" -#include -static spinlock_t conn_lock; -static struct list_head conn_list; -static struct list_head conn_unused_list; -static struct lustre_class_hash_body *conn_hash_body; -static struct lustre_class_hash_body *conn_unused_hash_body; +static cfs_hash_t *conn_hash = NULL; +static cfs_hash_ops_t conn_hash_ops; -extern struct lustre_hash_operations conn_hash_operations; - -void ptlrpc_dump_connections(void) +struct ptlrpc_connection * +ptlrpc_connection_get(lnet_process_id_t peer, lnet_nid_t self, + struct obd_uuid *uuid) { - struct list_head *tmp; - struct ptlrpc_connection *c; + struct ptlrpc_connection *conn, *conn2; ENTRY; - list_for_each(tmp, &conn_list) { - c = list_entry(tmp, struct ptlrpc_connection, c_link); - CERROR("Connection %p/%s has refcount %d (nid=%s->%s)\n", - c, c->c_remote_uuid.uuid, atomic_read(&c->c_refcount), - libcfs_nid2str(c->c_self), - libcfs_nid2str(c->c_peer.nid)); + conn = cfs_hash_lookup(conn_hash, &peer); + if (conn) + GOTO(out, conn); + + OBD_ALLOC_PTR(conn); + if (!conn) + RETURN(NULL); + + conn->c_peer = peer; + conn->c_self = self; + CFS_INIT_HLIST_NODE(&conn->c_hash); + cfs_atomic_set(&conn->c_refcount, 1); + if (uuid) + obd_str2uuid(&conn->c_remote_uuid, uuid->uuid); + + /* + * Add the newly created conn to the hash, on key collision we + * lost a racing addition and must destroy our newly allocated + * connection. The object which exists in the has will be + * returned and may be compared against out object. + */ + conn2 = cfs_hash_findadd_unique(conn_hash, &peer, &conn->c_hash); + if (conn != conn2) { + OBD_FREE_PTR(conn); + conn = conn2; } EXIT; +out: + CDEBUG(D_INFO, "conn=%p refcount %d to %s\n", + conn, cfs_atomic_read(&conn->c_refcount), + libcfs_nid2str(conn->c_peer.nid)); + return conn; } -struct ptlrpc_connection* -ptlrpc_lookup_conn_locked (lnet_process_id_t peer) +int ptlrpc_connection_put(struct ptlrpc_connection *conn) { - struct ptlrpc_connection *c; + int rc = 0; + ENTRY; - c = lustre_hash_get_object_by_key(conn_hash_body, &peer); - if (c != NULL) - return c; + if (!conn) + RETURN(rc); + + LASSERT(cfs_atomic_read(&conn->c_refcount) > 1); + + /* + * We do not remove connection from hashtable and + * do not free it even if last caller released ref, + * as we want to have it cached for the case it is + * needed again. + * + * Deallocating it and later creating new connection + * again would be wastful. This way we also avoid + * expensive locking to protect things from get/put + * race when found cached connection is freed by + * ptlrpc_connection_put(). + * + * It will be freed later in module unload time, + * when ptlrpc_connection_fini()->lh_exit->conn_exit() + * path is called. + */ + if (cfs_atomic_dec_return(&conn->c_refcount) == 1) + rc = 1; - c = lustre_hash_get_object_by_key(conn_unused_hash_body, &peer); - if (c != NULL) - return c; + CDEBUG(D_INFO, "PUT conn=%p refcount %d to %s\n", + conn, cfs_atomic_read(&conn->c_refcount), + libcfs_nid2str(conn->c_peer.nid)); - return NULL; + RETURN(rc); } - -struct ptlrpc_connection *ptlrpc_get_connection(lnet_process_id_t peer, - lnet_nid_t self, struct obd_uuid *uuid) +struct ptlrpc_connection * +ptlrpc_connection_addref(struct ptlrpc_connection *conn) { - struct ptlrpc_connection *c; - struct ptlrpc_connection *c2; - int rc = 0; ENTRY; - CDEBUG(D_INFO, "self %s peer %s\n", - libcfs_nid2str(self), libcfs_id2str(peer)); - - spin_lock(&conn_lock); - - c = ptlrpc_lookup_conn_locked(peer); - - spin_unlock(&conn_lock); - - if (c != NULL) - RETURN (c); - - OBD_ALLOC(c, sizeof(*c)); - if (c == NULL) - RETURN (NULL); - - atomic_set(&c->c_refcount, 1); - c->c_peer = peer; - c->c_self = self; - INIT_HLIST_NODE(&c->c_hash); - CFS_INIT_LIST_HEAD(&c->c_link); - if (uuid != NULL) - obd_str2uuid(&c->c_remote_uuid, uuid->uuid); - - spin_lock(&conn_lock); - - c2 = ptlrpc_lookup_conn_locked(peer); - if (c2 == NULL) { - list_add(&c->c_link, &conn_list); - rc = lustre_hash_additem_unique(conn_hash_body, &peer, - &c->c_hash); - if (rc != 0) { - list_del(&c->c_link); - CERROR("Cannot add connection to conn_hash_body\n"); - goto out_conn; - } - } - -out_conn: - - spin_unlock(&conn_lock); + cfs_atomic_inc(&conn->c_refcount); + CDEBUG(D_INFO, "conn=%p refcount %d to %s\n", + conn, cfs_atomic_read(&conn->c_refcount), + libcfs_nid2str(conn->c_peer.nid)); - if (c2 == NULL && rc == 0) - RETURN (c); - - if (c != NULL) - OBD_FREE(c, sizeof(*c)); - RETURN (c2); + RETURN(conn); } -int ptlrpc_put_connection(struct ptlrpc_connection *c) +int ptlrpc_connection_init(void) { - int rc = 0; ENTRY; - if (c == NULL) { - CERROR("NULL connection\n"); - RETURN(0); - } - - CDEBUG (D_INFO, "connection=%p refcount %d to %s\n", - c, atomic_read(&c->c_refcount) - 1, - libcfs_nid2str(c->c_peer.nid)); - - spin_lock(&conn_lock); - LASSERT(!hlist_unhashed(&c->c_hash)); - spin_unlock(&conn_lock); - - if (atomic_dec_return(&c->c_refcount) == 1) { - - spin_lock(&conn_lock); + conn_hash = cfs_hash_create("CONN_HASH", + HASH_CONN_CUR_BITS, + HASH_CONN_MAX_BITS, + HASH_CONN_BKT_BITS, 0, + CFS_HASH_MIN_THETA, + CFS_HASH_MAX_THETA, + &conn_hash_ops, CFS_HASH_DEFAULT); + if (!conn_hash) + RETURN(-ENOMEM); + + RETURN(0); +} - lustre_hash_delitem(conn_hash_body, &c->c_peer, &c->c_hash); - list_del(&c->c_link); +void ptlrpc_connection_fini(void) { + ENTRY; + cfs_hash_putref(conn_hash); + EXIT; +} - list_add(&c->c_link, &conn_unused_list); - rc = lustre_hash_additem_unique(conn_unused_hash_body, &c->c_peer, - &c->c_hash); - if (rc != 0) { - spin_unlock(&conn_lock); - CERROR("Cannot hash connection to conn_hash_body\n"); - GOTO(ret, rc); - } +/* + * Hash operations for net_peer<->connection + */ +static unsigned +conn_hashfn(cfs_hash_t *hs, const void *key, unsigned mask) +{ + return cfs_hash_djb2_hash(key, sizeof(lnet_process_id_t), mask); +} - spin_unlock(&conn_lock); - rc = 1; - - } +static int +conn_keycmp(const void *key, cfs_hlist_node_t *hnode) +{ + struct ptlrpc_connection *conn; + const lnet_process_id_t *conn_key; - if (atomic_read(&c->c_refcount) < 0) - CERROR("connection %p refcount %d!\n", - c, atomic_read(&c->c_refcount)); -ret : + LASSERT(key != NULL); + conn_key = (lnet_process_id_t*)key; + conn = cfs_hlist_entry(hnode, struct ptlrpc_connection, c_hash); - RETURN(rc); + return conn_key->nid == conn->c_peer.nid && + conn_key->pid == conn->c_peer.pid; } -struct ptlrpc_connection *ptlrpc_connection_addref(struct ptlrpc_connection *c) +static void * +conn_key(cfs_hlist_node_t *hnode) { - ENTRY; - atomic_inc(&c->c_refcount); - CDEBUG (D_INFO, "connection=%p refcount %d to %s\n", - c, atomic_read(&c->c_refcount), - libcfs_nid2str(c->c_peer.nid)); - RETURN(c); + struct ptlrpc_connection *conn; + conn = cfs_hlist_entry(hnode, struct ptlrpc_connection, c_hash); + return &conn->c_peer; } -int ptlrpc_init_connection(void) +static void * +conn_object(cfs_hlist_node_t *hnode) { - int rc = 0; - CFS_INIT_LIST_HEAD(&conn_list); - rc = lustre_hash_init(&conn_hash_body, "CONN_HASH", - 128, &conn_hash_operations); - if (rc) - GOTO(ret, rc); - - CFS_INIT_LIST_HEAD(&conn_unused_list); - rc = lustre_hash_init(&conn_unused_hash_body, "CONN_UNUSED_HASH", - 128, &conn_hash_operations); - if (rc) - GOTO(ret, rc); - - spin_lock_init(&conn_lock); -ret: - if (rc) { - lustre_hash_exit(&conn_hash_body); - lustre_hash_exit(&conn_unused_hash_body); - } - RETURN(rc); + return cfs_hlist_entry(hnode, struct ptlrpc_connection, c_hash); } -void ptlrpc_cleanup_connection(void) +static void +conn_get(cfs_hash_t *hs, cfs_hlist_node_t *hnode) { - struct list_head *tmp, *pos; - struct ptlrpc_connection *c; + struct ptlrpc_connection *conn; - spin_lock(&conn_lock); + conn = cfs_hlist_entry(hnode, struct ptlrpc_connection, c_hash); + cfs_atomic_inc(&conn->c_refcount); +} - lustre_hash_exit(&conn_unused_hash_body); - list_for_each_safe(tmp, pos, &conn_unused_list) { - c = list_entry(tmp, struct ptlrpc_connection, c_link); - list_del(&c->c_link); - OBD_FREE(c, sizeof(*c)); - } +static void +conn_put_locked(cfs_hash_t *hs, cfs_hlist_node_t *hnode) +{ + struct ptlrpc_connection *conn; - lustre_hash_exit(&conn_hash_body); - list_for_each_safe(tmp, pos, &conn_list) { - c = list_entry(tmp, struct ptlrpc_connection, c_link); - CERROR("Connection %p/%s has refcount %d (nid=%s)\n", - c, c->c_remote_uuid.uuid, atomic_read(&c->c_refcount), - libcfs_nid2str(c->c_peer.nid)); - list_del(&c->c_link); - OBD_FREE(c, sizeof(*c)); - } - spin_unlock(&conn_lock); + conn = cfs_hlist_entry(hnode, struct ptlrpc_connection, c_hash); + cfs_atomic_dec(&conn->c_refcount); } + +static void +conn_exit(cfs_hash_t *hs, cfs_hlist_node_t *hnode) +{ + struct ptlrpc_connection *conn; + + conn = cfs_hlist_entry(hnode, struct ptlrpc_connection, c_hash); + /* + * Nothing should be left. Connection user put it and + * connection also was deleted from table by this time + * so we should have 0 refs. + */ + LASSERTF(cfs_atomic_read(&conn->c_refcount) == 0, + "Busy connection with %d refs\n", + cfs_atomic_read(&conn->c_refcount)); + OBD_FREE_PTR(conn); +} + +static cfs_hash_ops_t conn_hash_ops = { + .hs_hash = conn_hashfn, + .hs_keycmp = conn_keycmp, + .hs_key = conn_key, + .hs_object = conn_object, + .hs_get = conn_get, + .hs_put_locked = conn_put_locked, + .hs_exit = conn_exit, +};