X-Git-Url: https://git.whamcloud.com/?p=fs%2Flustre-release.git;a=blobdiff_plain;f=lustre%2Fptlrpc%2Fconnection.c;h=36d962f6e39fad96501416aff93c0f4bead01b5a;hp=f3dba0f6a47e89952c4a368d92879c4d3a0b65b2;hb=e2af7fb3c91dfb13d34d8e1b2f2df8c09621f768;hpb=4df1229c4d6be064038ad8806d213ea146eab932 diff --git a/lustre/ptlrpc/connection.c b/lustre/ptlrpc/connection.c index f3dba0f..36d962f 100644 --- a/lustre/ptlrpc/connection.c +++ b/lustre/ptlrpc/connection.c @@ -1,6 +1,4 @@ -/* -*- mode: c; c-basic-offset: 8; indent-tabs-mode: nil; -*- - * vim:expandtab:shiftwidth=8:tabstop=8: - * +/* * GPL HEADER START * * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. @@ -26,8 +24,10 @@ * GPL HEADER END */ /* - * Copyright 2008 Sun Microsystems, Inc. All rights reserved + * Copyright (c) 2002, 2010, Oracle and/or its affiliates. All rights reserved. * Use is subject to license terms. + * + * Copyright (c) 2011, Whamcloud, Inc. */ /* * This file is part of Lustre, http://www.lustre.org/ @@ -44,208 +44,201 @@ #endif #include "ptlrpc_internal.h" -#include -static spinlock_t conn_lock; -static struct list_head conn_list; -static struct lustre_class_hash_body *conn_hash_body; -static struct lustre_class_hash_body *conn_unused_hash_body; +static cfs_hash_t *conn_hash = NULL; +static cfs_hash_ops_t conn_hash_ops; -extern struct lustre_hash_operations conn_hash_operations; - -void ptlrpc_dump_connection(void *obj, void *data) +struct ptlrpc_connection * +ptlrpc_connection_get(lnet_process_id_t peer, lnet_nid_t self, + struct obd_uuid *uuid) { - struct ptlrpc_connection *c = obj; + struct ptlrpc_connection *conn, *conn2; + ENTRY; - CERROR("Connection %p/%s has refcount %d (nid=%s->%s)\n", - c, c->c_remote_uuid.uuid, atomic_read(&c->c_refcount), - libcfs_nid2str(c->c_self), - libcfs_nid2str(c->c_peer.nid)); + conn = cfs_hash_lookup(conn_hash, &peer); + if (conn) + GOTO(out, conn); + + OBD_ALLOC_PTR(conn); + if (!conn) + RETURN(NULL); + + conn->c_peer = peer; + conn->c_self = self; + CFS_INIT_HLIST_NODE(&conn->c_hash); + cfs_atomic_set(&conn->c_refcount, 1); + if (uuid) + obd_str2uuid(&conn->c_remote_uuid, uuid->uuid); + + /* + * Add the newly created conn to the hash, on key collision we + * lost a racing addition and must destroy our newly allocated + * connection. The object which exists in the has will be + * returned and may be compared against out object. + */ + conn2 = cfs_hash_findadd_unique(conn_hash, &peer, &conn->c_hash); + if (conn != conn2) { + OBD_FREE_PTR(conn); + conn = conn2; + } + EXIT; +out: + CDEBUG(D_INFO, "conn=%p refcount %d to %s\n", + conn, cfs_atomic_read(&conn->c_refcount), + libcfs_nid2str(conn->c_peer.nid)); + return conn; } -void ptlrpc_dump_connections(void) +int ptlrpc_connection_put(struct ptlrpc_connection *conn) { + int rc = 0; ENTRY; - lustre_hash_iterate_all(conn_hash_body, ptlrpc_dump_connection, NULL); + if (!conn) + RETURN(rc); + + LASSERT(cfs_atomic_read(&conn->c_refcount) > 1); + + /* + * We do not remove connection from hashtable and + * do not free it even if last caller released ref, + * as we want to have it cached for the case it is + * needed again. + * + * Deallocating it and later creating new connection + * again would be wastful. This way we also avoid + * expensive locking to protect things from get/put + * race when found cached connection is freed by + * ptlrpc_connection_put(). + * + * It will be freed later in module unload time, + * when ptlrpc_connection_fini()->lh_exit->conn_exit() + * path is called. + */ + if (cfs_atomic_dec_return(&conn->c_refcount) == 1) + rc = 1; - EXIT; -} + CDEBUG(D_INFO, "PUT conn=%p refcount %d to %s\n", + conn, cfs_atomic_read(&conn->c_refcount), + libcfs_nid2str(conn->c_peer.nid)); -struct ptlrpc_connection* -ptlrpc_lookup_conn_locked (lnet_process_id_t peer) -{ - struct ptlrpc_connection *c = NULL; - int rc; - - c = lustre_hash_get_object_by_key(conn_hash_body, &peer); - if (c != NULL) - return c; - - c = lustre_hash_get_object_by_key(conn_unused_hash_body, &peer); - if (c != NULL) { - lustre_hash_delitem(conn_unused_hash_body, &peer, &c->c_hash); - rc = lustre_hash_additem_unique(conn_hash_body, &peer, - &c->c_hash); - if (rc) { - /* can't add - try with new item */ - OBD_FREE_PTR(c); - list_del(&c->c_link); - c = NULL; - } - } - - return c; + RETURN(rc); } - -struct ptlrpc_connection *ptlrpc_get_connection(lnet_process_id_t peer, - lnet_nid_t self, struct obd_uuid *uuid) +struct ptlrpc_connection * +ptlrpc_connection_addref(struct ptlrpc_connection *conn) { - struct ptlrpc_connection *c; - struct ptlrpc_connection *c2; - int rc = 0; ENTRY; - CDEBUG(D_INFO, "self %s peer %s\n", - libcfs_nid2str(self), libcfs_id2str(peer)); - - spin_lock(&conn_lock); - c = ptlrpc_lookup_conn_locked(peer); - spin_unlock(&conn_lock); - - if (c != NULL) - RETURN (c); - - OBD_ALLOC_PTR(c); - if (c == NULL) - RETURN (NULL); - - atomic_set(&c->c_refcount, 1); - c->c_peer = peer; - c->c_self = self; - INIT_HLIST_NODE(&c->c_hash); - CFS_INIT_LIST_HEAD(&c->c_link); - if (uuid != NULL) - obd_str2uuid(&c->c_remote_uuid, uuid->uuid); - - spin_lock(&conn_lock); - - c2 = ptlrpc_lookup_conn_locked(peer); - if (c2 == NULL) { - rc = lustre_hash_additem_unique(conn_hash_body, &peer, - &c->c_hash); - if (rc != 0) { - CERROR("Cannot add connection to conn_hash_body\n"); - goto out_conn; - } - list_add(&c->c_link, &conn_list); - } + cfs_atomic_inc(&conn->c_refcount); + CDEBUG(D_INFO, "conn=%p refcount %d to %s\n", + conn, cfs_atomic_read(&conn->c_refcount), + libcfs_nid2str(conn->c_peer.nid)); -out_conn: - spin_unlock(&conn_lock); - - if (c2 == NULL && rc == 0) - RETURN (c); - - if (c != NULL) - OBD_FREE(c, sizeof(*c)); - RETURN (c2); + RETURN(conn); } -int ptlrpc_put_connection(struct ptlrpc_connection *c) +int ptlrpc_connection_init(void) { - int rc = 0; ENTRY; - if (c == NULL) { - CERROR("NULL connection\n"); - RETURN(0); - } + conn_hash = cfs_hash_create("CONN_HASH", + HASH_CONN_CUR_BITS, + HASH_CONN_MAX_BITS, + HASH_CONN_BKT_BITS, 0, + CFS_HASH_MIN_THETA, + CFS_HASH_MAX_THETA, + &conn_hash_ops, CFS_HASH_DEFAULT); + if (!conn_hash) + RETURN(-ENOMEM); + + RETURN(0); +} - CDEBUG (D_INFO, "connection=%p refcount %d to %s\n", - c, atomic_read(&c->c_refcount) - 1, - libcfs_nid2str(c->c_peer.nid)); +void ptlrpc_connection_fini(void) { + ENTRY; + cfs_hash_putref(conn_hash); + EXIT; +} - spin_lock(&conn_lock); - LASSERT(!hlist_unhashed(&c->c_hash)); - spin_unlock(&conn_lock); +/* + * Hash operations for net_peer<->connection + */ +static unsigned +conn_hashfn(cfs_hash_t *hs, const void *key, unsigned mask) +{ + return cfs_hash_djb2_hash(key, sizeof(lnet_process_id_t), mask); +} - if (atomic_dec_return(&c->c_refcount) == 1) { +static int +conn_keycmp(const void *key, cfs_hlist_node_t *hnode) +{ + struct ptlrpc_connection *conn; + const lnet_process_id_t *conn_key; - spin_lock(&conn_lock); - lustre_hash_delitem(conn_hash_body, &c->c_peer, &c->c_hash); - rc = lustre_hash_additem_unique(conn_unused_hash_body, &c->c_peer, - &c->c_hash); - spin_lock(&conn_lock); - if (rc != 0) { - CERROR("Cannot hash connection to conn_hash_body\n"); - GOTO(ret, rc); - } + LASSERT(key != NULL); + conn_key = (lnet_process_id_t*)key; + conn = cfs_hlist_entry(hnode, struct ptlrpc_connection, c_hash); - rc = 1; - } + return conn_key->nid == conn->c_peer.nid && + conn_key->pid == conn->c_peer.pid; +} - if (atomic_read(&c->c_refcount) < 0) - CERROR("connection %p refcount %d!\n", - c, atomic_read(&c->c_refcount)); -ret : +static void * +conn_key(cfs_hlist_node_t *hnode) +{ + struct ptlrpc_connection *conn; + conn = cfs_hlist_entry(hnode, struct ptlrpc_connection, c_hash); + return &conn->c_peer; +} - RETURN(rc); +static void * +conn_object(cfs_hlist_node_t *hnode) +{ + return cfs_hlist_entry(hnode, struct ptlrpc_connection, c_hash); } -struct ptlrpc_connection *ptlrpc_connection_addref(struct ptlrpc_connection *c) +static void +conn_get(cfs_hash_t *hs, cfs_hlist_node_t *hnode) { - ENTRY; - atomic_inc(&c->c_refcount); - CDEBUG (D_INFO, "connection=%p refcount %d to %s\n", - c, atomic_read(&c->c_refcount), - libcfs_nid2str(c->c_peer.nid)); - RETURN(c); + struct ptlrpc_connection *conn; + + conn = cfs_hlist_entry(hnode, struct ptlrpc_connection, c_hash); + cfs_atomic_inc(&conn->c_refcount); } -int ptlrpc_init_connection(void) +static void +conn_put_locked(cfs_hash_t *hs, cfs_hlist_node_t *hnode) { - int rc = 0; - CFS_INIT_LIST_HEAD(&conn_list); - rc = lustre_hash_init(&conn_hash_body, "CONN_HASH", - 128, &conn_hash_operations); - if (rc) - GOTO(ret, rc); - - rc = lustre_hash_init(&conn_unused_hash_body, "CONN_UNUSED_HASH", - 128, &conn_hash_operations); - if (rc) - GOTO(ret, rc); - - spin_lock_init(&conn_lock); -ret: - if (rc) { - lustre_hash_exit(&conn_hash_body); - lustre_hash_exit(&conn_unused_hash_body); - } - RETURN(rc); + struct ptlrpc_connection *conn; + + conn = cfs_hlist_entry(hnode, struct ptlrpc_connection, c_hash); + cfs_atomic_dec(&conn->c_refcount); } -void ptlrpc_cleanup_connection(void) +static void +conn_exit(cfs_hash_t *hs, cfs_hlist_node_t *hnode) { - struct list_head *tmp, *pos; - struct ptlrpc_connection *c; - - spin_lock(&conn_lock); - - lustre_hash_exit(&conn_unused_hash_body); - lustre_hash_exit(&conn_hash_body); - - list_for_each_safe(tmp, pos, &conn_list) { - c = list_entry(tmp, struct ptlrpc_connection, c_link); - if (atomic_read(&c->c_refcount)) - CERROR("Connection %p/%s has refcount %d (nid=%s)\n", - c, c->c_remote_uuid.uuid, - atomic_read(&c->c_refcount), - libcfs_nid2str(c->c_peer.nid)); - list_del(&c->c_link); - OBD_FREE(c, sizeof(*c)); - } - spin_unlock(&conn_lock); + struct ptlrpc_connection *conn; + + conn = cfs_hlist_entry(hnode, struct ptlrpc_connection, c_hash); + /* + * Nothing should be left. Connection user put it and + * connection also was deleted from table by this time + * so we should have 0 refs. + */ + LASSERTF(cfs_atomic_read(&conn->c_refcount) == 0, + "Busy connection with %d refs\n", + cfs_atomic_read(&conn->c_refcount)); + OBD_FREE_PTR(conn); } + +static cfs_hash_ops_t conn_hash_ops = { + .hs_hash = conn_hashfn, + .hs_keycmp = conn_keycmp, + .hs_key = conn_key, + .hs_object = conn_object, + .hs_get = conn_get, + .hs_put_locked = conn_put_locked, + .hs_exit = conn_exit, +};