From 4c689a573fafcfa1ca7474a275f958e00b1deddc Mon Sep 17 00:00:00 2001 From: Amir Shehata Date: Fri, 6 Nov 2015 12:41:01 -0800 Subject: [PATCH] LU-3322 lnet: make connect parameters persistent Store map-on-demand and peertx credits in the peer, since the peer is persistent. Also made sure that when assigning the parameters received on the connection to the peer structure through create, that if another peer is added before grabbing the lock we assign these parameters to it as well. Signed-off-by: Amir Shehata Change-Id: Ie68f1ba1349d15b0a31eff9a2ca454df8e408ea9 Reviewed-on: http://review.whamcloud.com/17074 Tested-by: Jenkins Reviewed-by: Doug Oucharek Reviewed-by: James Simmons Tested-by: Maloo Reviewed-by: Oleg Drokin --- lnet/klnds/o2iblnd/o2iblnd.c | 14 +++++--------- lnet/klnds/o2iblnd/o2iblnd.h | 8 ++++++-- lnet/klnds/o2iblnd/o2iblnd_cb.c | 36 ++++++++++++++++++++++++------------ 3 files changed, 35 insertions(+), 23 deletions(-) diff --git a/lnet/klnds/o2iblnd/o2iblnd.c b/lnet/klnds/o2iblnd/o2iblnd.c index c7edbba..5576ee8 100644 --- a/lnet/klnds/o2iblnd/o2iblnd.c +++ b/lnet/klnds/o2iblnd/o2iblnd.c @@ -337,6 +337,8 @@ kiblnd_create_peer(lnet_ni_t *ni, kib_peer_t **peerp, lnet_nid_t nid) peer->ibp_nid = nid; peer->ibp_error = 0; peer->ibp_last_alive = 0; + peer->ibp_max_frags = IBLND_CFG_RDMA_FRAGS; + peer->ibp_queue_depth = *kiblnd_tunables.kib_peertxcredits; atomic_set(&peer->ibp_refcount, 1); /* 1 ref for caller */ INIT_LIST_HEAD(&peer->ibp_list); /* not in the peer table yet */ @@ -704,7 +706,7 @@ kiblnd_get_completion_vector(kib_conn_t *conn, int cpt) kib_conn_t * kiblnd_create_conn(kib_peer_t *peer, struct rdma_cm_id *cmid, - int state, int version, kib_connparams_t *cp) + int state, int version) { /* CAVEAT EMPTOR: * If the new conn is created successfully it takes over the caller's @@ -758,14 +760,8 @@ kiblnd_create_conn(kib_peer_t *peer, struct rdma_cm_id *cmid, conn->ibc_peer = peer; /* I take the caller's ref */ cmid->context = conn; /* for future CM callbacks */ conn->ibc_cmid = cmid; - - if (cp == NULL) { - conn->ibc_max_frags = IBLND_CFG_RDMA_FRAGS; - conn->ibc_queue_depth = *kiblnd_tunables.kib_peertxcredits; - } else { - conn->ibc_max_frags = cp->ibcp_max_frags; - conn->ibc_queue_depth = cp->ibcp_queue_depth; - } + conn->ibc_max_frags = peer->ibp_max_frags; + conn->ibc_queue_depth = peer->ibp_queue_depth; INIT_LIST_HEAD(&conn->ibc_early_rxs); INIT_LIST_HEAD(&conn->ibc_tx_noops); diff --git a/lnet/klnds/o2iblnd/o2iblnd.h b/lnet/klnds/o2iblnd/o2iblnd.h index c03de37..bdb7ac5 100644 --- a/lnet/klnds/o2iblnd/o2iblnd.h +++ b/lnet/klnds/o2iblnd/o2iblnd.h @@ -736,6 +736,10 @@ typedef struct kib_peer int ibp_error; /* when (in jiffies) I was last alive */ cfs_time_t ibp_last_alive; + /* max map_on_demand */ + __u16 ibp_max_frags; + /* max_peer_credits */ + __u16 ibp_queue_depth; } kib_peer_t; extern kib_data_t kiblnd_data; @@ -1096,8 +1100,8 @@ int kiblnd_close_stale_conns_locked (kib_peer_t *peer, int version, __u64 incarnation); int kiblnd_close_peer_conns_locked (kib_peer_t *peer, int why); -kib_conn_t *kiblnd_create_conn (kib_peer_t *peer, struct rdma_cm_id *cmid, - int state, int version, kib_connparams_t *cp); +kib_conn_t *kiblnd_create_conn(kib_peer_t *peer, struct rdma_cm_id *cmid, + int state, int version); void kiblnd_destroy_conn (kib_conn_t *conn); void kiblnd_close_conn (kib_conn_t *conn, int error); void kiblnd_close_conn_locked (kib_conn_t *conn, int error); diff --git a/lnet/klnds/o2iblnd/o2iblnd_cb.c b/lnet/klnds/o2iblnd/o2iblnd_cb.c index 1223906..ff64291 100644 --- a/lnet/klnds/o2iblnd/o2iblnd_cb.c +++ b/lnet/klnds/o2iblnd/o2iblnd_cb.c @@ -2312,6 +2312,10 @@ kiblnd_passive_connect(struct rdma_cm_id *cmid, void *priv, int priv_nob) goto failed; } + /* We have validated the peer's parameters so use those */ + peer->ibp_max_frags = reqmsg->ibm_u.connparams.ibcp_max_frags; + peer->ibp_queue_depth = reqmsg->ibm_u.connparams.ibcp_queue_depth; + write_lock_irqsave(g_lock, flags); peer2 = kiblnd_find_peer_locked(nid); @@ -2350,6 +2354,12 @@ kiblnd_passive_connect(struct rdma_cm_id *cmid, void *priv, int priv_nob) peer2->ibp_accepting++; kiblnd_peer_addref(peer2); + /* Race with kiblnd_launch_tx (active connect) to create peer + * so copy validated parameters since we now know what the + * peer's limits are */ + peer2->ibp_max_frags = peer->ibp_max_frags; + peer2->ibp_queue_depth = peer->ibp_queue_depth; + write_unlock_irqrestore(g_lock, flags); kiblnd_peer_decref(peer); peer = peer2; @@ -2372,8 +2382,7 @@ kiblnd_passive_connect(struct rdma_cm_id *cmid, void *priv, int priv_nob) write_unlock_irqrestore(g_lock, flags); } - conn = kiblnd_create_conn(peer, cmid, IBLND_CONN_PASSIVE_WAIT, version, - &reqmsg->ibm_u.connparams); + conn = kiblnd_create_conn(peer, cmid, IBLND_CONN_PASSIVE_WAIT, version); if (conn == NULL) { kiblnd_peer_connect_failed(peer, 0, -ENOMEM); kiblnd_peer_decref(peer); @@ -2383,10 +2392,9 @@ kiblnd_passive_connect(struct rdma_cm_id *cmid, void *priv, int priv_nob) /* conn now "owns" cmid, so I return success from here on to ensure the * CM callback doesn't destroy cmid. */ - conn->ibc_incarnation = reqmsg->ibm_srcstamp; - conn->ibc_credits = reqmsg->ibm_u.connparams.ibcp_queue_depth; - conn->ibc_reserved_credits = reqmsg->ibm_u.connparams.ibcp_queue_depth; + conn->ibc_credits = conn->ibc_queue_depth; + conn->ibc_reserved_credits = conn->ibc_queue_depth; LASSERT(conn->ibc_credits + conn->ibc_reserved_credits + IBLND_OOB_MSGS(version) <= IBLND_RX_MSGS(conn)); @@ -2395,10 +2403,8 @@ kiblnd_passive_connect(struct rdma_cm_id *cmid, void *priv, int priv_nob) kiblnd_init_msg(ackmsg, IBLND_MSG_CONNACK, sizeof(ackmsg->ibm_u.connparams)); - ackmsg->ibm_u.connparams.ibcp_queue_depth = - reqmsg->ibm_u.connparams.ibcp_queue_depth; - ackmsg->ibm_u.connparams.ibcp_max_frags = - reqmsg->ibm_u.connparams.ibcp_max_frags; + ackmsg->ibm_u.connparams.ibcp_queue_depth = conn->ibc_queue_depth; + ackmsg->ibm_u.connparams.ibcp_max_frags = conn->ibc_max_frags; ackmsg->ibm_u.connparams.ibcp_max_msg_size = IBLND_MSG_SIZE; kiblnd_pack_msg(ni, ackmsg, version, 0, nid, reqmsg->ibm_srcstamp); @@ -2488,6 +2494,9 @@ kiblnd_reconnect (kib_conn_t *conn, int version, break; case IBLND_REJECT_RDMA_FRAGS: + if (!cp) + goto failed; + if (conn->ibc_max_frags <= cp->ibcp_max_frags) { CNETERR("Unsupported max frags, peer supports %d\n", cp->ibcp_max_frags); @@ -2498,18 +2507,21 @@ kiblnd_reconnect (kib_conn_t *conn, int version, goto failed; } - conn->ibc_max_frags = cp->ibcp_max_frags; + peer->ibp_max_frags = cp->ibcp_max_frags; reason = "rdma fragments"; break; case IBLND_REJECT_MSG_QUEUE_SIZE: + if (!cp) + goto failed; + if (conn->ibc_queue_depth <= cp->ibcp_queue_depth) { CNETERR("Unsupported queue depth, peer supports %d\n", cp->ibcp_queue_depth); goto failed; } - conn->ibc_queue_depth = cp->ibcp_queue_depth; + peer->ibp_queue_depth = cp->ibcp_queue_depth; reason = "queue depth"; break; @@ -2787,7 +2799,7 @@ kiblnd_active_connect (struct rdma_cm_id *cmid) read_unlock_irqrestore(&kiblnd_data.kib_global_lock, flags); conn = kiblnd_create_conn(peer, cmid, IBLND_CONN_ACTIVE_CONNECT, - version, NULL); + version); if (conn == NULL) { kiblnd_peer_connect_failed(peer, 1, -ENOMEM); kiblnd_peer_decref(peer); /* lose cmid's ref */ -- 1.8.3.1