From 84f3af43c4bdeb1744736f44cd746dd4b6e8fa6d Mon Sep 17 00:00:00 2001 From: Amir Shehata Date: Mon, 25 Jun 2018 20:59:07 -0700 Subject: [PATCH 1/1] LU-9120 lnet: calculate the lnd timeout Calculate the LND timeout based on the transaction timeout and the retry count. Both of these are user defined values. Whenever they are set the lnd timeout is calculated. The LNDs use these timeouts instead of the LND timeout module parameter. Retry count can be set to 0, which means no retries. In that case the LND timeout will default to 5 seconds, which is the same as the default transaction timeout. Test-Parameters: forbuildonly Signed-off-by: Amir Shehata Change-Id: I5a37caa2b69df155211864735ba8b275fc2d34bb Reviewed-on: https://review.whamcloud.com/32770 Reviewed-by: Olaf Weber Reviewed-by: Sonia Sharma Tested-by: Jenkins --- lnet/include/lnet/lib-lnet.h | 2 ++ lnet/klnds/o2iblnd/o2iblnd_cb.c | 49 +++++++++++++++++++++-------------------- lnet/klnds/socklnd/socklnd.c | 6 ++--- lnet/klnds/socklnd/socklnd_cb.c | 27 ++++++++++++----------- lnet/lnet/api-ni.c | 9 ++++++++ 5 files changed, 53 insertions(+), 40 deletions(-) diff --git a/lnet/include/lnet/lib-lnet.h b/lnet/include/lnet/lib-lnet.h index 44cda89..0af6d32 100644 --- a/lnet/include/lnet/lib-lnet.h +++ b/lnet/include/lnet/lib-lnet.h @@ -75,6 +75,7 @@ extern struct lnet the_lnet; /* THE network */ /* default timeout */ #define DEFAULT_PEER_TIMEOUT 180 +#define LNET_LND_DEFAULT_TIMEOUT 5 static inline int lnet_is_route_alive(struct lnet_route *route) { @@ -796,6 +797,7 @@ void lnet_md_deconstruct(struct lnet_libmd *lmd, struct lnet_md *umd); struct page *lnet_kvaddr_to_page(unsigned long vaddr); int lnet_cpt_of_md(struct lnet_libmd *md, unsigned int offset); +unsigned int lnet_get_lnd_timeout(void); void lnet_register_lnd(struct lnet_lnd *lnd); void lnet_unregister_lnd(struct lnet_lnd *lnd); diff --git a/lnet/klnds/o2iblnd/o2iblnd_cb.c b/lnet/klnds/o2iblnd/o2iblnd_cb.c index c5b0bc4..ebd74e9 100644 --- a/lnet/klnds/o2iblnd/o2iblnd_cb.c +++ b/lnet/klnds/o2iblnd/o2iblnd_cb.c @@ -1278,7 +1278,7 @@ kiblnd_queue_tx_locked(struct kib_tx *tx, struct kib_conn *conn) LASSERT(!tx->tx_queued); /* not queued for sending already */ LASSERT(conn->ibc_state >= IBLND_CONN_ESTABLISHED); - timeout_ns = *kiblnd_tunables.kib_timeout * NSEC_PER_SEC; + timeout_ns = lnet_get_lnd_timeout() * NSEC_PER_SEC; tx->tx_queued = 1; tx->tx_deadline = ktime_add_ns(ktime_get(), timeout_ns); @@ -1404,21 +1404,21 @@ kiblnd_connect_peer(struct kib_peer_ni *peer_ni) kiblnd_peer_addref(peer_ni); /* cmid's ref */ - if (*kiblnd_tunables.kib_use_priv_port) { - rc = kiblnd_resolve_addr(cmid, &srcaddr, &dstaddr, - *kiblnd_tunables.kib_timeout * 1000); - } else { - rc = rdma_resolve_addr(cmid, - (struct sockaddr *)&srcaddr, - (struct sockaddr *)&dstaddr, - *kiblnd_tunables.kib_timeout * 1000); - } - if (rc != 0) { - /* Can't initiate address resolution: */ - CERROR("Can't resolve addr for %s: %d\n", - libcfs_nid2str(peer_ni->ibp_nid), rc); - goto failed2; - } + if (*kiblnd_tunables.kib_use_priv_port) { + rc = kiblnd_resolve_addr(cmid, &srcaddr, &dstaddr, + lnet_get_lnd_timeout() * 1000); + } else { + rc = rdma_resolve_addr(cmid, + (struct sockaddr *)&srcaddr, + (struct sockaddr *)&dstaddr, + lnet_get_lnd_timeout() * 1000); + } + if (rc != 0) { + /* Can't initiate address resolution: */ + CERROR("Can't resolve addr for %s: %d\n", + libcfs_nid2str(peer_ni->ibp_nid), rc); + goto failed2; + } return; @@ -3153,9 +3153,9 @@ kiblnd_cm_callback(struct rdma_cm_id *cmid, struct rdma_cm_event *event) CNETERR("Can't resolve address for %s: %d\n", libcfs_nid2str(peer_ni->ibp_nid), event->status); rc = event->status; - } else { - rc = rdma_resolve_route( - cmid, *kiblnd_tunables.kib_timeout * 1000); + } else { + rc = rdma_resolve_route( + cmid, lnet_get_lnd_timeout() * 1000); if (rc == 0) { struct kib_net *net = peer_ni->ibp_ni->ni_data; struct kib_dev *dev = net->ibn_dev; @@ -3560,6 +3560,7 @@ kiblnd_connd (void *arg) const int n = 4; const int p = 1; int chunk = kiblnd_data.kib_peer_hash_size; + unsigned int lnd_timeout; spin_unlock_irqrestore(lock, flags); dropped_lock = 1; @@ -3572,11 +3573,11 @@ kiblnd_connd (void *arg) * connection within (n+1)/n times the timeout * interval. */ - if (*kiblnd_tunables.kib_timeout > n * p) - chunk = (chunk * n * p) / - *kiblnd_tunables.kib_timeout; - if (chunk == 0) - chunk = 1; + lnd_timeout = lnet_get_lnd_timeout(); + if (lnd_timeout > n * p) + chunk = (chunk * n * p) / lnd_timeout; + if (chunk == 0) + chunk = 1; for (i = 0; i < chunk; i++) { kiblnd_check_conns(peer_index); diff --git a/lnet/klnds/socklnd/socklnd.c b/lnet/klnds/socklnd/socklnd.c index 1e36746..0f5b15a 100644 --- a/lnet/klnds/socklnd/socklnd.c +++ b/lnet/klnds/socklnd/socklnd.c @@ -1288,7 +1288,7 @@ ksocknal_create_conn(struct lnet_ni *ni, struct ksock_route *route, /* Set the deadline for the outgoing HELLO to drain */ conn->ksnc_tx_bufnob = sock->sk->sk_wmem_queued; conn->ksnc_tx_deadline = ktime_get_seconds() + - *ksocknal_tunables.ksnd_timeout; + lnet_get_lnd_timeout(); smp_mb(); /* order with adding to peer_ni's conn list */ list_add(&conn->ksnc_list, &peer_ni->ksnp_conns); @@ -1669,7 +1669,7 @@ ksocknal_destroy_conn(struct ksock_conn *conn) switch (conn->ksnc_rx_state) { case SOCKNAL_RX_LNET_PAYLOAD: last_rcv = conn->ksnc_rx_deadline - - *ksocknal_tunables.ksnd_timeout; + lnet_get_lnd_timeout(); CERROR("Completing partial receive from %s[%d], " "ip %pI4h:%d, with error, wanted: %d, left: %d, " "last alive is %lld secs ago\n", @@ -1843,7 +1843,7 @@ ksocknal_query(struct lnet_ni *ni, lnet_nid_t nid, time64_t *when) if (bufnob < conn->ksnc_tx_bufnob) { /* something got ACKed */ conn->ksnc_tx_deadline = ktime_get_seconds() + - *ksocknal_tunables.ksnd_timeout; + lnet_get_lnd_timeout(); peer_ni->ksnp_last_alive = now; conn->ksnc_tx_bufnob = bufnob; } diff --git a/lnet/klnds/socklnd/socklnd_cb.c b/lnet/klnds/socklnd/socklnd_cb.c index fb16e04..2eed8b1 100644 --- a/lnet/klnds/socklnd/socklnd_cb.c +++ b/lnet/klnds/socklnd/socklnd_cb.c @@ -223,7 +223,7 @@ ksocknal_transmit(struct ksock_conn *conn, struct ksock_tx *tx) /* allocated send buffer bytes < computed; infer * something got ACKed */ conn->ksnc_tx_deadline = ktime_get_seconds() + - *ksocknal_tunables.ksnd_timeout; + lnet_get_lnd_timeout(); conn->ksnc_peer->ksnp_last_alive = ktime_get_seconds(); conn->ksnc_tx_bufnob = bufnob; smp_mb(); @@ -272,7 +272,7 @@ ksocknal_recv_iov(struct ksock_conn *conn) conn->ksnc_peer->ksnp_last_alive = ktime_get_seconds(); conn->ksnc_rx_deadline = ktime_get_seconds() + - *ksocknal_tunables.ksnd_timeout; + lnet_get_lnd_timeout(); smp_mb(); /* order with setting rx_started */ conn->ksnc_rx_started = 1; @@ -316,7 +316,7 @@ ksocknal_recv_kiov(struct ksock_conn *conn) conn->ksnc_peer->ksnp_last_alive = ktime_get_seconds(); conn->ksnc_rx_deadline = ktime_get_seconds() + - *ksocknal_tunables.ksnd_timeout; + lnet_get_lnd_timeout(); smp_mb(); /* order with setting rx_started */ conn->ksnc_rx_started = 1; @@ -489,7 +489,7 @@ ksocknal_check_zc_req(struct ksock_tx *tx) /* ZC_REQ is going to be pinned to the peer_ni */ tx->tx_deadline = ktime_get_seconds() + - *ksocknal_tunables.ksnd_timeout; + lnet_get_lnd_timeout(); LASSERT (tx->tx_msg.ksm_zc_cookies[0] == 0); @@ -770,7 +770,7 @@ ksocknal_queue_tx_locked(struct ksock_tx *tx, struct ksock_conn *conn) if (list_empty(&conn->ksnc_tx_queue) && bufnob == 0) { /* First packet starts the timeout */ conn->ksnc_tx_deadline = ktime_get_seconds() + - *ksocknal_tunables.ksnd_timeout; + lnet_get_lnd_timeout(); if (conn->ksnc_tx_bufnob > 0) /* something got ACKed */ conn->ksnc_peer->ksnp_last_alive = ktime_get_seconds(); conn->ksnc_tx_bufnob = 0; @@ -947,7 +947,7 @@ ksocknal_launch_packet(struct lnet_ni *ni, struct ksock_tx *tx, ksocknal_find_connecting_route_locked (peer_ni) != NULL) { /* the message is going to be pinned to the peer_ni */ tx->tx_deadline = ktime_get_seconds() + - *ksocknal_tunables.ksnd_timeout; + lnet_get_lnd_timeout(); /* Queue the message until a connection is established */ list_add_tail(&tx->tx_list, &peer_ni->ksnp_tx_queue); @@ -1762,7 +1762,7 @@ ksocknal_recv_hello(struct lnet_ni *ni, struct ksock_conn *conn, /* socket type set on active connections - not set on passive */ LASSERT(!active == !(conn->ksnc_type != SOCKLND_CONN_NONE)); - timeout = active ? *ksocknal_tunables.ksnd_timeout : + timeout = active ? lnet_get_lnd_timeout() : lnet_acceptor_timeout(); rc = lnet_sock_read(sock, &hello->kshm_magic, @@ -1897,7 +1897,7 @@ ksocknal_connect(struct ksock_route *route) int retry_later = 0; int rc = 0; - deadline = ktime_get_seconds() + *ksocknal_tunables.ksnd_timeout; + deadline = ktime_get_seconds() + lnet_get_lnd_timeout(); write_lock_bh(&ksocknal_data.ksnd_global_lock); @@ -2636,6 +2636,7 @@ int ksocknal_reaper(void *arg) const int n = 4; const int p = 1; int chunk = ksocknal_data.ksnd_peer_hash_size; + unsigned int lnd_timeout; /* Time to check for timeouts on a few more peers: I do * checks every 'p' seconds on a proportion of the peer_ni @@ -2644,11 +2645,11 @@ int ksocknal_reaper(void *arg) * timeout on any connection within (n+1)/n times the * timeout interval. */ - if (*ksocknal_tunables.ksnd_timeout > n * p) - chunk = (chunk * n * p) / - *ksocknal_tunables.ksnd_timeout; - if (chunk == 0) - chunk = 1; + lnd_timeout = lnet_get_lnd_timeout(); + if (lnd_timeout > n * p) + chunk = (chunk * n * p) / lnd_timeout; + if (chunk == 0) + chunk = 1; for (i = 0; i < chunk; i++) { ksocknal_check_peer_timeouts (peer_index); diff --git a/lnet/lnet/api-ni.c b/lnet/lnet/api-ni.c index 2a11c0c..aa35054 100644 --- a/lnet/lnet/api-ni.c +++ b/lnet/lnet/api-ni.c @@ -149,6 +149,8 @@ module_param(lnet_retry_count, uint, 0444); MODULE_PARM_DESC(lnet_retry_count, "Maximum number of times to retry transmitting a message"); +unsigned lnet_lnd_timeout = LNET_LND_DEFAULT_TIMEOUT; + /* * This sequence number keeps track of how many times DLC was used to * update the local NIs. It is incremented when a NI is added or @@ -573,6 +575,13 @@ static struct lnet_lnd *lnet_find_lnd_by_type(__u32 type) return NULL; } +unsigned int +lnet_get_lnd_timeout(void) +{ + return lnet_lnd_timeout; +} +EXPORT_SYMBOL(lnet_get_lnd_timeout); + void lnet_register_lnd(struct lnet_lnd *lnd) { -- 1.8.3.1