From 16dbb70a7d1a435acc1bd12edc826b55574f9f79 Mon Sep 17 00:00:00 2001 From: Chris Horn Date: Wed, 12 Jan 2022 19:19:21 +0000 Subject: [PATCH] LU-15446 lnet: Don't use pref NI for reserved portal Don't use the preferred NI when sending traffic on the LNet reserved portal. This allows local recovery pings to utilize any local NI as source in the case where we do not have a multi-rail peer entry for the local host. This is typically the case when MR is not being configured statically (i.e. when discovery is being used for MR configuration). lnet_get_best_ni() was modified to include health values of the NIs being compared in its debug output. Lustre-change: https://review.whamcloud.com/46078 Lustre-commit: a2815441381cb6cee8eb9865d9279541ea04828e HPE-bug-id: LUS-10658 Test-Parameters: trivial testlist=sanity-lnet Signed-off-by: Chris Horn Change-Id: I38f5760bf034f698b7f44ffa89aa91c4f5d4b9ea Reviewed-on: https://review.whamcloud.com/48312 Tested-by: jenkins Tested-by: Maloo Reviewed-by: Frank Sehr Reviewed-by: Andreas Dilger --- lnet/lnet/lib-move.c | 56 ++++++++++++++++++++++++++------------------- lustre/tests/sanity-lnet.sh | 54 +++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 86 insertions(+), 24 deletions(-) diff --git a/lnet/lnet/lib-move.c b/lnet/lnet/lib-move.c index 1047c30..b67036b 100644 --- a/lnet/lnet/lib-move.c +++ b/lnet/lnet/lib-move.c @@ -1684,6 +1684,19 @@ lnet_get_best_ni(struct lnet_net *local_net, struct lnet_ni *best_ni, return best_ni; } +static bool +lnet_reserved_msg(struct lnet_msg *msg) +{ + if (msg->msg_type == LNET_MSG_PUT) { + if (msg->msg_hdr.msg.put.ptl_index == LNET_RESERVED_PORTAL) + return true; + } else if (msg->msg_type == LNET_MSG_GET) { + if (msg->msg_hdr.msg.get.ptl_index == LNET_RESERVED_PORTAL) + return true; + } + return false; +} + /* * Traffic to the LNET_RESERVED_PORTAL may not trigger peer discovery, * because such traffic is required to perform discovery. We therefore @@ -1695,14 +1708,7 @@ lnet_get_best_ni(struct lnet_net *local_net, struct lnet_ni *best_ni, static bool lnet_msg_discovery(struct lnet_msg *msg) { - if (msg->msg_type == LNET_MSG_PUT) { - if (msg->msg_hdr.msg.put.ptl_index != LNET_RESERVED_PORTAL) - return true; - } else if (msg->msg_type == LNET_MSG_GET) { - if (msg->msg_hdr.msg.get.ptl_index != LNET_RESERVED_PORTAL) - return true; - } - return false; + return !(lnet_reserved_msg(msg) || lnet_msg_is_response(msg)); } #define SRC_SPEC 0x0001 @@ -2378,7 +2384,6 @@ static int lnet_select_preferred_best_ni(struct lnet_send_data *sd) { struct lnet_ni *best_ni = NULL; - struct lnet_peer_ni *best_lpni = sd->sd_best_lpni; /* * We must use a consistent source address when sending to a @@ -2389,25 +2394,28 @@ lnet_select_preferred_best_ni(struct lnet_send_data *sd) * * So we need to pick the NI the peer prefers for this * particular network. + * + * An exception is traffic on LNET_RESERVED_PORTAL. Internal LNet + * traffic doesn't care which source NI is used, and we don't actually + * want to restrict local recovery pings to a single source NI. */ + if (!lnet_reserved_msg(sd->sd_msg)) + best_ni = lnet_find_existing_preferred_best_ni(sd->sd_best_lpni, + sd->sd_cpt); - best_ni = lnet_find_existing_preferred_best_ni(sd->sd_best_lpni, - sd->sd_cpt); - - /* if best_ni is still not set just pick one */ - if (!best_ni) { - best_ni = - lnet_find_best_ni_on_spec_net(NULL, sd->sd_peer, + if (!best_ni) + best_ni = lnet_find_best_ni_on_spec_net(NULL, sd->sd_peer, sd->sd_best_lpni->lpni_peer_net, - sd->sd_msg, sd->sd_md_cpt, + sd->sd_msg, + sd->sd_md_cpt, true); - /* If there is no best_ni we don't have a route */ - if (!best_ni) { - CERROR("no path to %s from net %s\n", - libcfs_nid2str(best_lpni->lpni_nid), - libcfs_net2str(best_lpni->lpni_net->net_id)); - return -EHOSTUNREACH; - } + + /* If there is no best_ni we don't have a route */ + if (!best_ni) { + CERROR("no path to %s from net %s\n", + libcfs_nid2str(sd->sd_best_lpni->lpni_nid), + libcfs_net2str(sd->sd_best_lpni->lpni_net->net_id)); + return -EHOSTUNREACH; } sd->sd_best_ni = best_ni; diff --git a/lustre/tests/sanity-lnet.sh b/lustre/tests/sanity-lnet.sh index a274d25..6b8e0bc 100755 --- a/lustre/tests/sanity-lnet.sh +++ b/lustre/tests/sanity-lnet.sh @@ -92,6 +92,7 @@ load_lnet() { } do_lnetctl() { + $LCTL mark "$LNETCTL $@" echo "$LNETCTL $@" $LNETCTL "$@" } @@ -2001,6 +2002,59 @@ test_219() { } run_test 219 "Consolidate peer entries" +test_218() { + reinit_dlc || return $? + + [[ ${#INTERFACES[@]} -lt 2 ]] && + skip "Need two LNet interfaces" + + add_net "tcp" "${INTERFACES[0]}" || return $? + + local nid1=$($LCTL list_nids | head -n 1) + + do_lnetctl ping $nid1 || + error "ping failed" + + add_net "tcp" "${INTERFACES[1]}" || return $? + + local nid2=$($LCTL list_nids | tail --lines 1) + + do_lnetctl ping $nid2 || + error "ping failed" + + $LCTL net_drop_add -s $nid1 -d $nid1 -e local_error -r 1 + + do_lnetctl ping $nid1 && + error "ping should have failed" + + local health_recovered + local i + + for i in $(seq 1 5); do + health_recovered=$($LNETCTL net show -v 2 | + grep -c 'health value: 1000') + + if [[ $health_recovered -ne 2 ]]; then + echo "Wait 1 second for health to recover" + sleep 1 + else + break + fi + done + + health_recovered=$($LNETCTL net show -v 2 | + grep -c 'health value: 1000') + + $LCTL net_drop_del -a + + [[ $health_recovered -ne 2 ]] && + do_lnetctl net show -v 2 | egrep -e nid -e health && + error "Health hasn't recovered" + + return 0 +} +run_test 218 "Local recovery pings should exercise all available paths" + test_230() { # LU-12815 echo "Check valid values; Should succeed" -- 1.8.3.1