From 0316f2ce528c3c0db1c9a52ec836dc69c8ff5c55 Mon Sep 17 00:00:00 2001 From: Alexander Boyko Date: Wed, 14 Oct 2020 04:20:58 -0400 Subject: [PATCH] LU-14031 ptlrpc: decrease time between reconnection When a connection get a timeout or get an error reply from a sever, the next attempt happens after PING_INTERVAL. It is equal to obd_timeout/4. When a first reconnection fails, a second go to failover pair. And a third connection go to a original server. Only 3 reconnection before server evicts client base on blocking ast timeout. Some times a first failed and the last is a bit late, so client is evicted. It is better to try reconnect with a timeout equal to a connection request deadline, it would increase a number of attempts in 5 times for a large obd_timeout. For example, obd_timeout=200 - [ 1597902357, CONNECTING ] - [ 1597902357, FULL ] - [ 1597902422, DISCONN ] - [ 1597902422, CONNECTING ] - [ 1597902433, DISCONN ] - [ 1597902473, CONNECTING ] - [ 1597902473, DISCONN ] <- ENODEV from a failover pair - [ 1597902523, CONNECTING ] - [ 1597902539, DISCONN ] The patch adds a logic to wakeup pinger for failed connection request with ETIMEDOUT or ENODEV. It adds imp_next_ping processing for ptlrpc_pinger_main() time_to_next_wake calculation, and fixes setting of imp_next_ping value. Lustre-commit: de8ed5f19f04136a4addcb3f91496f26478d03e7 Lustre-change: https://review.whamcloud.com/40244 HPE-bug-id: LUS-8520 Signed-off-by: Alexander Boyko Change-Id: Ia0891a8ead1922810037f7d71092cd57c061dab9 Reviewed-by: Andreas Dilger Reviewed-by: Alexey Lyashkov Reviewed-by: Vitaly Fertman Reviewed-on: https://review.whamcloud.com/44251 Tested-by: jenkins Tested-by: Maloo --- lustre/ptlrpc/pinger.c | 22 ++++++++++++++++++++-- 1 file changed, 20 insertions(+), 2 deletions(-) diff --git a/lustre/ptlrpc/pinger.c b/lustre/ptlrpc/pinger.c index ebdbbb5..f6f536c 100644 --- a/lustre/ptlrpc/pinger.c +++ b/lustre/ptlrpc/pinger.c @@ -176,9 +176,27 @@ static inline time64_t ptlrpc_next_reconnect(struct obd_import *imp) static timeout_t pinger_check_timeout(time64_t time) { - time64_t timeout = PING_INTERVAL; + timeout_t timeout = PING_INTERVAL; + timeout_t next_timeout; + time64_t now; + struct list_head *iter; + struct obd_import *imp; + + mutex_lock(&pinger_mutex); + now = ktime_get_seconds(); + /* Process imports to find a nearest next ping */ + list_for_each(iter, &pinger_imports) { + imp = list_entry(iter, struct obd_import, imp_pinger_chain); + if (!imp->imp_pingable || imp->imp_next_ping < now) + continue; + next_timeout = imp->imp_next_ping - now; + /* make sure imp_next_ping in the future from time */ + if (next_timeout > (now - time) && timeout > next_timeout) + timeout = next_timeout; + } + mutex_unlock(&pinger_mutex); - return time + timeout - ktime_get_seconds(); + return timeout - (now - time); } static bool ir_up; -- 1.8.3.1