X-Git-Url: https://git.whamcloud.com/?p=fs%2Flustre-release.git;a=blobdiff_plain;f=lustre%2Fptlrpc%2Fconnection.c;h=a41b2021d6ea3ed7eb6172503f8c100d0df828b7;hp=7a1d47b013386331ea2c04c1adae1786ddbd560e;hb=6e3ec5812ebd1b5ecf7cae584f429b013ffe7431;hpb=c65d56a58b74bc2507ed1028b7738838a84e08a1 diff --git a/lustre/ptlrpc/connection.c b/lustre/ptlrpc/connection.c index 7a1d47b..a41b202 100644 --- a/lustre/ptlrpc/connection.c +++ b/lustre/ptlrpc/connection.c @@ -1,164 +1,238 @@ /* -*- mode: c; c-basic-offset: 8; indent-tabs-mode: nil; -*- * vim:expandtab:shiftwidth=8:tabstop=8: * - * Copyright (C) 2002 Cluster File Systems, Inc. + * GPL HEADER START * - * This file is part of Lustre, http://www.lustre.org. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * - * Lustre is free software; you can redistribute it and/or - * modify it under the terms of version 2 of the GNU General Public - * License as published by the Free Software Foundation. + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License version 2 only, + * as published by the Free Software Foundation. * - * Lustre is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. + * This program is distributed in the hope that it will be useful, but + * WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * General Public License version 2 for more details (a copy is included + * in the LICENSE file that accompanied this code). * - * You should have received a copy of the GNU General Public License - * along with Lustre; if not, write to the Free Software - * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA. + * You should have received a copy of the GNU General Public License + * version 2 along with this program; If not, see + * http://www.sun.com/software/products/lustre/docs/GPLv2.pdf * + * Please contact Sun Microsystems, Inc., 4150 Network Circle, Santa Clara, + * CA 95054 USA or visit www.sun.com if you need additional information or + * have any questions. + * + * GPL HEADER END + */ +/* + * Copyright 2008 Sun Microsystems, Inc. All rights reserved + * Use is subject to license terms. + */ +/* + * This file is part of Lustre, http://www.lustre.org/ + * Lustre is a trademark of Sun Microsystems, Inc. */ #define DEBUG_SUBSYSTEM S_RPC +#ifdef __KERNEL__ +#include +#include +#include +#else +#include +#endif + +#include "ptlrpc_internal.h" + +static cfs_hash_t *conn_hash = NULL; +static cfs_hash_ops_t conn_hash_ops; + +struct ptlrpc_connection * +ptlrpc_connection_get(lnet_process_id_t peer, lnet_nid_t self, + struct obd_uuid *uuid) +{ + struct ptlrpc_connection *conn, *conn2; + ENTRY; + + conn = cfs_hash_lookup(conn_hash, &peer); + if (conn) + GOTO(out, conn); -#include -#include + OBD_ALLOC_PTR(conn); + if (!conn) + RETURN(NULL); -static spinlock_t conn_lock; -static struct list_head conn_list; -static struct list_head conn_unused_list; + conn->c_peer = peer; + conn->c_self = self; + CFS_INIT_HLIST_NODE(&conn->c_hash); + cfs_atomic_set(&conn->c_refcount, 1); + if (uuid) + obd_str2uuid(&conn->c_remote_uuid, uuid->uuid); + + /* + * Add the newly created conn to the hash, on key collision we + * lost a racing addition and must destroy our newly allocated + * connection. The object which exists in the has will be + * returned and may be compared against out object. + */ + conn2 = cfs_hash_findadd_unique(conn_hash, &peer, &conn->c_hash); + if (conn != conn2) { + OBD_FREE_PTR(conn); + conn = conn2; + } + EXIT; +out: + CDEBUG(D_INFO, "conn=%p refcount %d to %s\n", + conn, cfs_atomic_read(&conn->c_refcount), + libcfs_nid2str(conn->c_peer.nid)); + return conn; +} -/* If UUID is NULL, c->c_remote_uuid must be all zeroes - * If UUID is non-NULL, c->c_remote_uuid must match. */ -static int match_connection_uuid(struct ptlrpc_connection *c, obd_uuid_t uuid) +int ptlrpc_connection_put(struct ptlrpc_connection *conn) { - obd_uuid_t zero_uuid = {0}; + int rc = 0; + ENTRY; - if (uuid) - return memcmp(c->c_remote_uuid, uuid, sizeof(uuid)); + if (!conn) + RETURN(rc); + + LASSERT(!cfs_hlist_unhashed(&conn->c_hash)); + + /* + * We do not remove connection from hashtable and + * do not free it even if last caller released ref, + * as we want to have it cached for the case it is + * needed again. + * + * Deallocating it and later creating new connection + * again would be wastful. This way we also avoid + * expensive locking to protect things from get/put + * race when found cached connection is freed by + * ptlrpc_connection_put(). + * + * It will be freed later in module unload time, + * when ptlrpc_connection_fini()->lh_exit->conn_exit() + * path is called. + */ + if (cfs_atomic_dec_return(&conn->c_refcount) == 1) + rc = 1; + + CDEBUG(D_INFO, "PUT conn=%p refcount %d to %s\n", + conn, cfs_atomic_read(&conn->c_refcount), + libcfs_nid2str(conn->c_peer.nid)); - return memcmp(c->c_remote_uuid, zero_uuid, sizeof(zero_uuid)); + RETURN(rc); } -struct ptlrpc_connection *ptlrpc_get_connection(struct lustre_peer *peer, - obd_uuid_t uuid) +struct ptlrpc_connection * +ptlrpc_connection_addref(struct ptlrpc_connection *conn) { - struct list_head *tmp, *pos; - struct ptlrpc_connection *c; ENTRY; - CDEBUG(D_INFO, "peer is %08x %08lx %08lx\n", - peer->peer_nid, peer->peer_ni.nal_idx, peer->peer_ni.handle_idx); - - spin_lock(&conn_lock); - list_for_each(tmp, &conn_list) { - c = list_entry(tmp, struct ptlrpc_connection, c_link); - if (memcmp(peer, &c->c_peer, sizeof(*peer)) == 0 && - !match_connection_uuid(c, uuid)) { - ptlrpc_connection_addref(c); - GOTO(out, c); - } - } + cfs_atomic_inc(&conn->c_refcount); + CDEBUG(D_INFO, "conn=%p refcount %d to %s\n", + conn, cfs_atomic_read(&conn->c_refcount), + libcfs_nid2str(conn->c_peer.nid)); - list_for_each_safe(tmp, pos, &conn_unused_list) { - c = list_entry(tmp, struct ptlrpc_connection, c_link); - if (memcmp(peer, &c->c_peer, sizeof(*peer)) == 0 && - !match_connection_uuid(c, uuid)) { - ptlrpc_connection_addref(c); - list_del(&c->c_link); - list_add(&c->c_link, &conn_list); - GOTO(out, c); - } - } + RETURN(conn); +} - /* FIXME: this should be a slab once we can validate slab addresses - * without OOPSing */ - OBD_ALLOC(c, sizeof(*c)); - if (c == NULL) - GOTO(out, c); - - c->c_level = LUSTRE_CONN_NEW; - c->c_xid_in = 1; - c->c_xid_out = 1; - c->c_generation = 1; - c->c_epoch = 1; - c->c_bootcount = 0; - if (uuid) - strcpy(c->c_remote_uuid, uuid); - INIT_LIST_HEAD(&c->c_delayed_head); - INIT_LIST_HEAD(&c->c_sending_head); - INIT_LIST_HEAD(&c->c_dying_head); - INIT_LIST_HEAD(&c->c_imports); - INIT_LIST_HEAD(&c->c_exports); - atomic_set(&c->c_refcount, 0); - ptlrpc_connection_addref(c); - spin_lock_init(&c->c_lock); - - memcpy(&c->c_peer, peer, sizeof(c->c_peer)); - list_add(&c->c_link, &conn_list); +int ptlrpc_connection_init(void) +{ + ENTRY; + + conn_hash = cfs_hash_create("CONN_HASH", + HASH_CONN_CUR_BITS, + HASH_CONN_MAX_BITS, + &conn_hash_ops, CFS_HASH_REHASH); + if (!conn_hash) + RETURN(-ENOMEM); + RETURN(0); +} + +void ptlrpc_connection_fini(void) { + ENTRY; + cfs_hash_destroy(conn_hash); EXIT; - out: - spin_unlock(&conn_lock); - return c; } -int ptlrpc_put_connection(struct ptlrpc_connection *c) +/* + * Hash operations for net_peer<->connection + */ +static unsigned +conn_hashfn(cfs_hash_t *hs, void *key, unsigned mask) { - int rc = 0; - ENTRY; + return cfs_hash_djb2_hash(key, sizeof(lnet_process_id_t), mask); +} - CDEBUG(D_INFO, "connection=%p refcount %d\n", - c, atomic_read(&c->c_refcount) - 1); - if (atomic_dec_and_test(&c->c_refcount)) { - spin_lock(&conn_lock); - list_del(&c->c_link); - list_add(&c->c_link, &conn_unused_list); - spin_unlock(&conn_lock); - rc = 1; - } - if (atomic_read(&c->c_refcount) < 0) - CERROR("connection %p refcount %d!\n", - c, atomic_read(&c->c_refcount)); +static int +conn_compare(void *key, cfs_hlist_node_t *hnode) +{ + struct ptlrpc_connection *conn; + lnet_process_id_t *conn_key; - RETURN(rc); + LASSERT(key != NULL); + conn_key = (lnet_process_id_t*)key; + conn = cfs_hlist_entry(hnode, struct ptlrpc_connection, c_hash); + + return conn_key->nid == conn->c_peer.nid && + conn_key->pid == conn->c_peer.pid; } -struct ptlrpc_connection *ptlrpc_connection_addref(struct ptlrpc_connection *c) +static void * +conn_key(cfs_hlist_node_t *hnode) { - ENTRY; - CDEBUG(D_INFO, "connection=%p refcount %d\n", - c, atomic_read(&c->c_refcount) + 1); - atomic_inc(&c->c_refcount); - RETURN(c); + struct ptlrpc_connection *conn; + conn = cfs_hlist_entry(hnode, struct ptlrpc_connection, c_hash); + return &conn->c_peer; } -void ptlrpc_init_connection(void) +static void * +conn_get(cfs_hlist_node_t *hnode) { - INIT_LIST_HEAD(&conn_list); - INIT_LIST_HEAD(&conn_unused_list); - conn_lock = SPIN_LOCK_UNLOCKED; + struct ptlrpc_connection *conn; + + conn = cfs_hlist_entry(hnode, struct ptlrpc_connection, c_hash); + cfs_atomic_inc(&conn->c_refcount); + + return conn; } -void ptlrpc_cleanup_connection(void) +static void * +conn_put(cfs_hlist_node_t *hnode) { - struct list_head *tmp, *pos; - struct ptlrpc_connection *c; - - spin_lock(&conn_lock); - list_for_each_safe(tmp, pos, &conn_unused_list) { - c = list_entry(tmp, struct ptlrpc_connection, c_link); - list_del(&c->c_link); - OBD_FREE(c, sizeof(*c)); - } - list_for_each_safe(tmp, pos, &conn_list) { - c = list_entry(tmp, struct ptlrpc_connection, c_link); - CERROR("Connection %p has refcount %d at cleanup (nid=%lu)!\n", - c, atomic_read(&c->c_refcount), - (unsigned long)c->c_peer.peer_nid); - list_del(&c->c_link); - OBD_FREE(c, sizeof(*c)); - } - spin_unlock(&conn_lock); + struct ptlrpc_connection *conn; + + conn = cfs_hlist_entry(hnode, struct ptlrpc_connection, c_hash); + cfs_atomic_dec(&conn->c_refcount); + + return conn; } + +static void +conn_exit(cfs_hlist_node_t *hnode) +{ + struct ptlrpc_connection *conn; + + conn = cfs_hlist_entry(hnode, struct ptlrpc_connection, c_hash); + /* + * Nothing should be left. Connection user put it and + * connection also was deleted from table by this time + * so we should have 0 refs. + */ + LASSERTF(cfs_atomic_read(&conn->c_refcount) == 0, + "Busy connection with %d refs\n", + cfs_atomic_read(&conn->c_refcount)); + OBD_FREE_PTR(conn); +} + +static cfs_hash_ops_t conn_hash_ops = { + .hs_hash = conn_hashfn, + .hs_compare = conn_compare, + .hs_key = conn_key, + .hs_get = conn_get, + .hs_put = conn_put, + .hs_exit = conn_exit, +};