From 7f60b2b5580f67ca55e53a78dbaf7d50b5b7ab47 Mon Sep 17 00:00:00 2001
From: Serguei Smirnov <ssmirnov@whamcloud.com>
Date: Thu, 10 Aug 2023 17:58:11 -0700
Subject: [PATCH] LU-17006 lnet: set up routes for going across subnets

Modify ksocklnd-config to set up route which features
default gateway for the subnet in case if default gateway
is defined, for example:
        ip route add default via <gw_for_eth0> dev eth0 table eth0
which results in a route similar to the following added to
the eth0 route table:
        default via <gw_for_eth0> dev eth0

If there's no gateway found for the eth0 subnet, keep the old
behaviour which results in the following added to eth0
route table:
        <eth0_subnet> dev eth0 proto kernel scope link src <eth0_ip>

This makes sure that MR traffic goes out the intended interface
as selected by LNet no matter whether going across subnets or not.

Test-Parameters: trivial testlist=sanity-lnet
Signed-off-by: Serguei Smirnov <ssmirnov@whamcloud.com>
Change-Id: I84a299c8b7eb4cdb4fc24408a1e42ad0283d9219
Reviewed-on: https://review.whamcloud.com/c/fs/lustre-release/+/51921
Tested-by: jenkins <devops@whamcloud.com>
Tested-by: Maloo <maloo@whamcloud.com>
Reviewed-by: Chris Horn <chris.horn@hpe.com>
Reviewed-by: Frank Sehr <fsehr@whamcloud.com>
Reviewed-by: Cyril Bordage <cbordage@whamcloud.com>
Reviewed-by: Oleg Drokin <green@whamcloud.com>
---
 lustre/scripts/ksocklnd-config | 27 ++++++++++++++++++++++++---
 1 file changed, 24 insertions(+), 3 deletions(-)

diff --git a/lustre/scripts/ksocklnd-config b/lustre/scripts/ksocklnd-config
index b716cfd..dee280b 100755
--- a/lustre/scripts/ksocklnd-config
+++ b/lustre/scripts/ksocklnd-config
@@ -131,6 +131,21 @@ do
 	fi
 done
 
+# generate list of available default gateways
+gwsline=$(/sbin/ip route | awk '/default/ { print $3 }')
+gateways=($gwsline)
+
+# select a gateway on the same subnet
+selectgw() {
+	for gw in "${gateways[@]}"; do
+		if [[ "$(netcalc "${1}" "${2}")" == "$(netcalc "${gw}" "${2}")" ]]; then
+			echo $gw
+			return
+		fi
+	done
+	echo "0.0.0.0"
+}
+
 # add the routing entries and rules
 for i in "${interfaces[@]}"
 do
@@ -139,10 +154,16 @@ do
 	cidrmask=($(/sbin/ip -o -4 addr list $i 2>&- | awk '{print $4}' | cut -d/ -f2))
 	# convert cidr mask to mask in dot format
 	dotmask=$(cidr2mask ${cidrmask[0]})
-	# apply mask to ip addr
-	net=$(netcalc ${addr[0]} $dotmask)
+	# find a gateway on the same subnet
+	gw=$(selectgw ${addr[0]} $dotmask)
 	# build and execute route commands
-	routecmd=(/sbin/ip route add ${net}/${cidrmask[0]} dev ${i} proto kernel scope link src ${addr[0]} table ${i})
+	if [[ $gw == "0.0.0.0" ]]; then
+		# gateway not found, assume local destinations
+		net=$(netcalc ${addr[0]} $dotmask)
+		routecmd=(/sbin/ip route add ${net}/${cidrmask[0]} dev ${i} proto kernel scope link src ${addr[0]} table ${i})
+	else
+		routecmd=(/sbin/ip route add default via ${gw} dev ${i} table ${i})
+	fi
 	ruledelcmd=(/sbin/ip rule del from ${addr[0]} table ${i} '&>/dev/null')
 	ruleaddcmd=(/sbin/ip rule add from ${addr[0]} table ${i})
 	eval ${routecmd[@]}
-- 
1.8.3.1