From a96e15188d9beb83266975375e47ad205e4241e8 Mon Sep 17 00:00:00 2001 From: Serguei Smirnov Date: Mon, 31 Aug 2020 18:35:52 -0400 Subject: [PATCH] LU-10973 lnet: LUTF UDSP test suite and routing test suite Added the UDSP suite and routing suite to the LUTF test cases. Updated some of the infrastructure scripts with methods needed for the new test cases. Test-Parameters: @lnet Signed-off-by: Serguei Smirnov Change-Id: Ibd74cea48982ccafc3b1d5034a409fd2df9e7b1c Reviewed-on: https://review.whamcloud.com/c/fs/lustre-release/+/39777 Tested-by: jenkins Tested-by: Maloo Reviewed-by: jsimmons Reviewed-by: Amir Shehata Reviewed-by: Oleg Drokin --- lustre/tests/lutf/Makefile.am | 2 + .../lutf/python/tests/suite_routing/callbacks.py | 4 + .../tests/suite_routing/test_routing_basic_01.py | 175 +++++++++++++++ .../tests/lutf/python/tests/suite_samples/skip.py | 3 + .../lutf/python/tests/suite_udsp/callbacks.py | 6 + .../python/tests/suite_udsp/test_udsp_basic_01.py | 45 ++++ .../python/tests/suite_udsp/test_udsp_basic_02.py | 48 ++++ .../python/tests/suite_udsp/test_udsp_basic_03.py | 58 +++++ .../python/tests/suite_udsp/test_udsp_basic_04.py | 48 ++++ .../python/tests/suite_udsp/test_udsp_basic_05.py | 48 ++++ .../python/tests/suite_udsp/test_udsp_basic_06.py | 48 ++++ .../tests/suite_udsp/test_udsp_basic_err_01.py | 59 +++++ .../tests/suite_udsp/test_udsp_basic_err_02.py | 93 ++++++++ .../tests/suite_udsp/test_udsp_multi_net_01.py | 129 +++++++++++ .../tests/suite_udsp/test_udsp_multi_net_02.py | 130 +++++++++++ .../tests/suite_udsp/test_udsp_multi_net_03.py | 160 ++++++++++++++ .../tests/suite_udsp/test_udsp_multi_net_04.py | 160 ++++++++++++++ .../tests/suite_udsp/test_udsp_multi_net_05.py | 148 +++++++++++++ .../tests/suite_udsp/test_udsp_multi_net_err_01.py | 148 +++++++++++++ .../tests/suite_udsp/test_udsp_multi_net_err_02.py | 191 ++++++++++++++++ .../tests/suite_udsp/test_udsp_routed_net_01.py | 243 +++++++++++++++++++++ .../tests/suite_udsp/test_udsp_single_net_01.py | 112 ++++++++++ .../tests/suite_udsp/test_udsp_single_net_02.py | 155 +++++++++++++ .../tests/suite_udsp/test_udsp_single_net_03.py | 185 ++++++++++++++++ .../tests/suite_udsp/test_udsp_single_net_04.py | 185 ++++++++++++++++ .../tests/suite_udsp/test_udsp_single_net_05.py | 158 ++++++++++++++ .../tests/suite_udsp/test_udsp_single_net_06.py | 136 ++++++++++++ .../tests/suite_udsp/test_udsp_single_net_07.py | 161 ++++++++++++++ 28 files changed, 3038 insertions(+) create mode 100755 lustre/tests/lutf/python/tests/suite_routing/callbacks.py create mode 100755 lustre/tests/lutf/python/tests/suite_routing/test_routing_basic_01.py create mode 100644 lustre/tests/lutf/python/tests/suite_samples/skip.py create mode 100755 lustre/tests/lutf/python/tests/suite_udsp/callbacks.py create mode 100755 lustre/tests/lutf/python/tests/suite_udsp/test_udsp_basic_01.py create mode 100755 lustre/tests/lutf/python/tests/suite_udsp/test_udsp_basic_02.py create mode 100755 lustre/tests/lutf/python/tests/suite_udsp/test_udsp_basic_03.py create mode 100755 lustre/tests/lutf/python/tests/suite_udsp/test_udsp_basic_04.py create mode 100755 lustre/tests/lutf/python/tests/suite_udsp/test_udsp_basic_05.py create mode 100755 lustre/tests/lutf/python/tests/suite_udsp/test_udsp_basic_06.py create mode 100755 lustre/tests/lutf/python/tests/suite_udsp/test_udsp_basic_err_01.py create mode 100755 lustre/tests/lutf/python/tests/suite_udsp/test_udsp_basic_err_02.py create mode 100755 lustre/tests/lutf/python/tests/suite_udsp/test_udsp_multi_net_01.py create mode 100755 lustre/tests/lutf/python/tests/suite_udsp/test_udsp_multi_net_02.py create mode 100755 lustre/tests/lutf/python/tests/suite_udsp/test_udsp_multi_net_03.py create mode 100755 lustre/tests/lutf/python/tests/suite_udsp/test_udsp_multi_net_04.py create mode 100755 lustre/tests/lutf/python/tests/suite_udsp/test_udsp_multi_net_05.py create mode 100755 lustre/tests/lutf/python/tests/suite_udsp/test_udsp_multi_net_err_01.py create mode 100755 lustre/tests/lutf/python/tests/suite_udsp/test_udsp_multi_net_err_02.py create mode 100755 lustre/tests/lutf/python/tests/suite_udsp/test_udsp_routed_net_01.py create mode 100755 lustre/tests/lutf/python/tests/suite_udsp/test_udsp_single_net_01.py create mode 100755 lustre/tests/lutf/python/tests/suite_udsp/test_udsp_single_net_02.py create mode 100755 lustre/tests/lutf/python/tests/suite_udsp/test_udsp_single_net_03.py create mode 100755 lustre/tests/lutf/python/tests/suite_udsp/test_udsp_single_net_04.py create mode 100755 lustre/tests/lutf/python/tests/suite_udsp/test_udsp_single_net_05.py create mode 100755 lustre/tests/lutf/python/tests/suite_udsp/test_udsp_single_net_06.py create mode 100755 lustre/tests/lutf/python/tests/suite_udsp/test_udsp_single_net_07.py diff --git a/lustre/tests/lutf/Makefile.am b/lustre/tests/lutf/Makefile.am index 8dd1190..eb2525d 100644 --- a/lustre/tests/lutf/Makefile.am +++ b/lustre/tests/lutf/Makefile.am @@ -20,6 +20,8 @@ nobase_noinst_SCRIPTS += $(wildcard python/tests/suite_dlc/*.py) nobase_noinst_SCRIPTS += $(wildcard python/tests/suite_samples/*.py) nobase_noinst_SCRIPTS += $(wildcard python/tests/suite_multi-rail/*.py) nobase_noinst_SCRIPTS += $(wildcard python/tests/suite_dynamic-discovery/*.py) +nobase_noinst_SCRIPTS += $(wildcard python/tests/suite_udsp/*.py) +nobase_noinst_SCRIPTS += $(wildcard python/tests/suite_routing/*.py) nobase_noinst_SCRIPTS += $(wildcard python/tests/suite_dynamic-nids/*.py) EXTRA_DIST=$(nobase_noinst_DATA) $(nobase_noinst_SCRIPTS) diff --git a/lustre/tests/lutf/python/tests/suite_routing/callbacks.py b/lustre/tests/lutf/python/tests/suite_routing/callbacks.py new file mode 100755 index 0000000..296cad6 --- /dev/null +++ b/lustre/tests/lutf/python/tests/suite_routing/callbacks.py @@ -0,0 +1,4 @@ +from lnet_cleanup import clean_lnet + +def lutf_clean_setup(): + clean_lnet() diff --git a/lustre/tests/lutf/python/tests/suite_routing/test_routing_basic_01.py b/lustre/tests/lutf/python/tests/suite_routing/test_routing_basic_01.py new file mode 100755 index 0000000..6f775eb --- /dev/null +++ b/lustre/tests/lutf/python/tests/suite_routing/test_routing_basic_01.py @@ -0,0 +1,175 @@ +""" +@PRIMARY: N/A +@PRIMARY_DESC: N/A +@SECONDARY: N/A +@DESIGN: N/A +@TESTCASE: 1. Allocate nodes: PeerA, PeerB and GW1 +2. Configure PeerA with "tcp" net, PeerB with "tcp1" net, GW1 with both +3. Configure GW1 as a router b/w tcp and tcp1 nets. +3. Add routes to PeerA and PeerB so they can reach each other. +4. Execute lnetctl ping several times +5. Retrieve stats on GW1 and verify that it was routing the pings. +6. Clean up. +""" + +import os +import yaml +import lnetconfig +import time +from lutf import agents, me +from lutf_basetest import * +from lnet import TheLNet +from lutf_exception import LUTFError +from lnet_helpers import LNetHelpers +from lustre_node import SimpleLustreNode +from lutf_cmd import lutf_exec_local_cmd +from utility_paths import LNETCTL + +MIN_NODES = 3 +MIN_IFS_PER_NODE = 1 +GW_MIN_IFS = 2 +PING_TIMES = 10 +PING_NID_NUM = 0 +LOCAL_NETS = ['tcp'] +REMOTE_NETS = ['tcp1'] +USE_NET_NUM = [0] +MAIN_NODE_ID = 0 +REMOTE_NODE_ID = 1 +GW_NODE_IDS = [2] + +class TestLustreTraffic: + def __init__(self, target=None): + self.lh = LNetHelpers(os.path.abspath(__file__), target=target) + self.sln = SimpleLustreNode(os.path.abspath(__file__), target=target) + +def run(): + la = agents.keys() + if len(la) < MIN_NODES: + return lutfrc(LUTF_TEST_SKIP, "Not enough agents to run the test") + nodes = [] + try: + # General config for all nodes + for i in range(0, MIN_NODES): + node = TestLustreTraffic(la[i]) + t = node.lh + if i in GW_NODE_IDS: + node_num_ifs = GW_MIN_IFS + print("Processing gw node init") + else: + node_num_ifs = MIN_IFS_PER_NODE + intfs = t.get_available_devs() + print(i, " intfs: ", intfs) + if len(intfs) < node_num_ifs: + return lutfrc(LUTF_TEST_FAIL, "Not enough interfaces") + print("Configure") + t.configure_lnet() + + if i == MAIN_NODE_ID: + # Peer A + net = LOCAL_NETS[0] + net_intfs_list = intfs[0:node_num_ifs] + t.configure_net(net, net_intfs_list) + elif i == REMOTE_NODE_ID: + # Peer B + net = REMOTE_NETS[0] + net_intfs_list = intfs[0:node_num_ifs] + t.configure_net(net, net_intfs_list) + else: + # GW1 + net = LOCAL_NETS[0] + net_intfs_list = intfs[0:node_num_ifs//2] + t.configure_net(net, net_intfs_list) + net = REMOTE_NETS[0] + net_intfs_list = intfs[node_num_ifs//2:node_num_ifs] + t.configure_net(net, net_intfs_list) + + print("Set discovery on") + t.set_discovery(1) + print("done") + nodes.append(node) + + main = nodes[MAIN_NODE_ID] + main_nids = main.lh.list_nids() + remote = nodes[REMOTE_NODE_ID] + remote_nids = remote.lh.list_nids() + gw1 = nodes[GW_NODE_IDS[0]] + gw1_nids = gw1.lh.list_nids() + + print("main nids: ", main_nids) + print("remote nids: ", remote_nids) + print("gw1 nids: ", gw1_nids) + + gw1.lh.api_set_routing(True) + + gw1_remote_nids = [x for x in gw1_nids if REMOTE_NETS[0] in x] + gw1_local_nids = [x for x in gw1_nids if x not in gw1_remote_nids] + + print("gw1 local/remote nids: ", gw1_local_nids, gw1_remote_nids) + + # discover the gateway from main and remote nodes + if len(main.lh.exec_discover_cmd(gw1_local_nids[0])) == 0: + return lutfrc(LUTF_TEST_FAIL, "unable to discover" , + target=gw1_local_nids[0]) + if len(remote.lh.exec_discover_cmd(gw1_remote_nids[0])) == 0: + return lutfrc(LUTF_TEST_FAIL, "unable to discover" , + target=gw1_remote_nids[0]) + + # setup the routing + rt1 = "--net " + REMOTE_NETS[0] + " --gateway " + gw1_local_nids[0] + rc = main.lh.exec_route_cmd(" add "+rt1) + rt1 = "--net " + LOCAL_NETS[0] + " --gateway " + gw1_remote_nids[0] + rc = remote.lh.exec_route_cmd(" add "+rt1) + + time.sleep(1) + + # let main and remote discover each other + try: + if len(main.lh.exec_discover_cmd(remote_nids[0])) == 0: + return lutfrc(LUTF_TEST_FAIL, "unable to discover" , + target=remote_nids[0]) + except Exception as e: + if len(main.lh.exec_discover_cmd(remote_nids[0])) == 0: + return lutfrc(LUTF_TEST_FAIL, "unable to discover" , + target=remote_nids[0]) + try: + if len(remote.lh.exec_discover_cmd(main_nids[0])) == 0: + return lutfrc(LUTF_TEST_FAIL, "unable to discover" , + target=main_nids[0]) + except Exception as e: + if len(remote.lh.exec_discover_cmd(main_nids[0])) == 0: + return lutfrc(LUTF_TEST_FAIL, "unable to discover" , + target=main_nids[0]) + + before_stats_gw1 = gw1.lh.get_stats() + print(before_stats_gw1) + + for i in range(0, PING_TIMES): + rc = main.lh.exec_ping(remote_nids[PING_NID_NUM]) + if not rc: + print("ping failed") + return lutfrc(LUTF_TEST_FAIL) + + after_stats_gw1 = gw1.lh.get_stats() + print(after_stats_gw1) + + route_count_before = before_stats_gw1['statistics']['route_count'] + route_count_after = after_stats_gw1['statistics']['route_count'] + + print(route_count_before, route_count_after) + + # Check stats: + # 1) expect the difference between "before" and "after" route counts to be + # same as or greater than the number of pings issued + if (route_count_after - route_count_before) < PING_TIMES: + print("routed count mismatch") + return lutfrc(LUTF_TEST_FAIL) + + for n in nodes: + lutf_exec_local_cmd(LNETCTL + " lnet unconfigure") + + return lutfrc(LUTF_TEST_PASS) + except Exception as e: + for n in nodes: + n.lh.uninit() + raise e + diff --git a/lustre/tests/lutf/python/tests/suite_samples/skip.py b/lustre/tests/lutf/python/tests/suite_samples/skip.py new file mode 100644 index 0000000..b6329c1 --- /dev/null +++ b/lustre/tests/lutf/python/tests/suite_samples/skip.py @@ -0,0 +1,3 @@ +skip_list = [] +expected_failures = ['sample_04', 'sample_04_1'] + diff --git a/lustre/tests/lutf/python/tests/suite_udsp/callbacks.py b/lustre/tests/lutf/python/tests/suite_udsp/callbacks.py new file mode 100755 index 0000000..d02f7e2 --- /dev/null +++ b/lustre/tests/lutf/python/tests/suite_udsp/callbacks.py @@ -0,0 +1,6 @@ +from lnet_cleanup import clean_lnet +from lustre_cleanup import clean_lustre + +def lutf_clean_setup(): + clean_lustre() + clean_lnet() diff --git a/lustre/tests/lutf/python/tests/suite_udsp/test_udsp_basic_01.py b/lustre/tests/lutf/python/tests/suite_udsp/test_udsp_basic_01.py new file mode 100755 index 0000000..c645779 --- /dev/null +++ b/lustre/tests/lutf/python/tests/suite_udsp/test_udsp_basic_01.py @@ -0,0 +1,45 @@ +""" +@PRIMARY: N/A +@PRIMARY_DESC: Check that udsp rule table is empty on a newly configured lnet +@SECONDARY: N/A +@DESIGN: N/A +@TESTCASE: +- check that udsp rule table is empty on a newly configured lnet +""" + +import os +import yaml +import lnetconfig +from lutf import agents, me +from lutf_basetest import * +from lnet import TheLNet +from lutf_exception import LUTFError +from lnet_helpers import LNetHelpers +from lutf_cmd import lutf_exec_local_cmd +from utility_paths import LNETCTL + +def check_udsp_empty(): + rc = lutf_exec_local_cmd(LNETCTL + " udsp show") + if 'idx' in rc[0].decode('utf-8'): + return False + else: + return True + +def run(): + la = agents.keys() + if len(la) < 1: + return lutfrc(LUTF_TEST_SKIP, "No agents to run test") + try: + t = LNetHelpers(target=la[0]) + t.configure_lnet() + if not t.check_udsp_present(): + return lutfrc(LUTF_TEST_SKIP, "UDSP feature is missing") + rc = check_udsp_empty(); + if not rc: + return lutfrc(LUTF_TEST_FAIL) + return lutfrc(LUTF_TEST_PASS) + except Exception as e: + t.uninit() + raise e + + diff --git a/lustre/tests/lutf/python/tests/suite_udsp/test_udsp_basic_02.py b/lustre/tests/lutf/python/tests/suite_udsp/test_udsp_basic_02.py new file mode 100755 index 0000000..8c15115f --- /dev/null +++ b/lustre/tests/lutf/python/tests/suite_udsp/test_udsp_basic_02.py @@ -0,0 +1,48 @@ +""" +@PRIMARY: N/A +@PRIMARY_DESC: Verify that UDSP rule prioritizing a source net can be added +@SECONDARY: N/A +@DESIGN: N/A +@TESTCASE: +- configure lnet +- add udsp rule to prioritize a source net +- verify that the rule has been added +""" + +import os +import yaml +import lnetconfig +from lutf import agents, me +from lutf_basetest import * +from lnet import TheLNet +from lutf_exception import LUTFError +from lnet_helpers import LNetHelpers +from lutf_cmd import lutf_exec_local_cmd + + +test_action_list = [{'udsp_cmd': "add --src tcp", + 'show_res': {'udsp': [{'idx': 0, 'src': 'tcp', 'dst': 'NA', 'rte': 'NA', 'action': {'priority': 0}}]}}] + +def run(): + la = agents.keys() + if len(la) < 1: + return lutfrc(LUTF_TEST_SKIP, "No agents to run test") + try: + t = LNetHelpers(target=la[0]) + t.configure_lnet() + if not t.check_udsp_present(): + return lutfrc(LUTF_TEST_SKIP, "UDSP feature is missing") + rc = t.check_udsp_empty() + if not rc: + return lutfrc(LUTF_TEST_FAIL) + rc = t.exec_udsp_cmd(test_action_list[0]['udsp_cmd']) + rc = t.check_udsp_expected(test_action_list[0]['show_res']) + if not rc: + return lutfrc(LUTF_TEST_FAIL) + rc = t.cleanup_udsp() + return lutfrc(LUTF_TEST_PASS) + except Exception as e: + t.uninit() + raise e + + diff --git a/lustre/tests/lutf/python/tests/suite_udsp/test_udsp_basic_03.py b/lustre/tests/lutf/python/tests/suite_udsp/test_udsp_basic_03.py new file mode 100755 index 0000000..f1a2d9c --- /dev/null +++ b/lustre/tests/lutf/python/tests/suite_udsp/test_udsp_basic_03.py @@ -0,0 +1,58 @@ +""" +@PRIMARY: N/A +@PRIMARY_DESC: Verify that UDSP rules prioritizing source net and nid can be added and deleted +@SECONDARY: N/A +@DESIGN: N/A +@TESTCASE: +- configure lnet +- add udsp rule to prioritize a source net +- verify that the rule has been added +- add udsp rule to prioritize a source nid +- verify that the rule has been added +- delete source net udsp rule +- verify that the rule has been deleted +""" + +import os +import yaml +import lnetconfig +from lutf import agents, me +from lutf_basetest import * +from lnet import TheLNet +from lutf_exception import LUTFError +from lnet_helpers import LNetHelpers +from lutf_cmd import lutf_exec_local_cmd + + +test_action_list = [{'udsp_cmd': "add --src tcp1", + 'show_res': {'udsp': [{'idx': 0, 'src': 'tcp1', 'dst': 'NA', 'rte': 'NA', 'action': {'priority': 0}}]}}, + {'udsp_cmd': "add --src 192.168.122.1@tcp1", + 'show_res': {'udsp': [{'idx': 0, 'src': 'tcp1', 'dst': 'NA', 'rte': 'NA', 'action': {'priority': 0}}, + {'idx': 1, 'src': '192.168.122.1@tcp1', 'dst': 'NA', 'rte': 'NA', 'action': {'priority': 0}}]}}, + {'udsp_cmd': "del --idx=0", + 'show_res': {'udsp': [{'idx': 0, 'src': '192.168.122.1@tcp1', 'dst': 'NA', 'rte': 'NA', 'action': {'priority': 0}}]}}] + +def run(): + la = agents.keys() + if len(la) < 1: + return lutfrc(LUTF_TEST_SKIP, "No agents to run test") + try: + t = LNetHelpers(target=la[0]) + t.configure_lnet() + if not t.check_udsp_present(): + return lutfrc(LUTF_TEST_SKIP, "UDSP feature is missing") + rc = t.check_udsp_empty() + if not rc: + return lutfrc(LUTF_TEST_FAIL) + for test_action in test_action_list: + rc = t.exec_udsp_cmd(test_action['udsp_cmd']) + rc = t.check_udsp_expected(test_action['show_res']) + if not rc: + return lutfrc(LUTF_TEST_FAIL) + rc = t.cleanup_udsp() + return lutfrc(LUTF_TEST_PASS) + except Exception as e: + t.uninit() + raise e + + diff --git a/lustre/tests/lutf/python/tests/suite_udsp/test_udsp_basic_04.py b/lustre/tests/lutf/python/tests/suite_udsp/test_udsp_basic_04.py new file mode 100755 index 0000000..03ef606 --- /dev/null +++ b/lustre/tests/lutf/python/tests/suite_udsp/test_udsp_basic_04.py @@ -0,0 +1,48 @@ +""" +@PRIMARY: N/A +@PRIMARY_DESC: Verify that UDSP rule prioritizing a destination nid can be added +@SECONDARY: N/A +@DESIGN: N/A +@TESTCASE: +- configure LNet +- add udsp rule that prioritizes a destination nid +- verify that the rule has been added +""" + +import os +import yaml +import lnetconfig +from lutf import agents, me +from lutf_basetest import * +from lnet import TheLNet +from lutf_exception import LUTFError +from lnet_helpers import LNetHelpers +from lutf_cmd import lutf_exec_local_cmd + + +test_action_list = [{'udsp_cmd': "add --dst 192.168.122.101@tcp2", + 'show_res': {'udsp': [{'idx': 0, 'src': 'NA', 'dst': '192.168.122.101@tcp2', 'rte': 'NA', 'action': {'priority': 0}}]}}] + +def run(): + la = agents.keys() + if len(la) < 1: + return lutfrc(LUTF_TEST_SKIP, "No agents to run test") + try: + t = LNetHelpers(target=la[0]) + t.configure_lnet() + if not t.check_udsp_present(): + return lutfrc(LUTF_TEST_SKIP, "UDSP feature is missing") + rc = t.check_udsp_empty() + if not rc: + return lutfrc(LUTF_TEST_FAIL) + rc = t.exec_udsp_cmd(test_action_list[0]['udsp_cmd']) + rc = t.check_udsp_expected(test_action_list[0]['show_res']) + if not rc: + return lutfrc(LUTF_TEST_FAIL) + rc = t.cleanup_udsp() + return lutfrc(LUTF_TEST_PASS) + except Exception as e: + t.uninit() + raise e + + diff --git a/lustre/tests/lutf/python/tests/suite_udsp/test_udsp_basic_05.py b/lustre/tests/lutf/python/tests/suite_udsp/test_udsp_basic_05.py new file mode 100755 index 0000000..7f51dd7 --- /dev/null +++ b/lustre/tests/lutf/python/tests/suite_udsp/test_udsp_basic_05.py @@ -0,0 +1,48 @@ +""" +@PRIMARY: N/A +@PRIMARY_DESC: Verify that UDSP rule designating src-dst nid pair can be added +@SECONDARY: N/A +@DESIGN: N/A +@TESTCASE: +- configure LNet +- add udsp rule designating a source-destination nid pair +- verify that the rule has been added +""" + +import os +import yaml +import lnetconfig +from lutf import agents, me +from lutf_basetest import * +from lnet import TheLNet +from lutf_exception import LUTFError +from lnet_helpers import LNetHelpers +from lutf_cmd import lutf_exec_local_cmd + + +test_action_list = [{'udsp_cmd': "add --src 192.168.122.1@tcp1 --dst 192.168.122.101@tcp1", + 'show_res': {'udsp': [{'idx': 0, 'src': '192.168.122.1@tcp1', 'dst': '192.168.122.101@tcp1', 'rte': 'NA'}]}}] + +def run(): + la = agents.keys() + if len(la) < 1: + return lutfrc(LUTF_TEST_SKIP, "No agents to run test") + try: + t = LNetHelpers(target=la[0]) + t.configure_lnet() + if not t.check_udsp_present(): + return lutfrc(LUTF_TEST_SKIP, "UDSP feature is missing") + rc = t.check_udsp_empty() + if not rc: + return lutfrc(LUTF_TEST_FAIL) + rc = t.exec_udsp_cmd(test_action_list[0]['udsp_cmd']) + rc = t.check_udsp_expected(test_action_list[0]['show_res']) + if not rc: + return lutfrc(LUTF_TEST_FAIL) + rc = t.cleanup_udsp() + return lutfrc(LUTF_TEST_PASS) + except Exception as e: + t.uninit() + raise e + + diff --git a/lustre/tests/lutf/python/tests/suite_udsp/test_udsp_basic_06.py b/lustre/tests/lutf/python/tests/suite_udsp/test_udsp_basic_06.py new file mode 100755 index 0000000..6d2c70f --- /dev/null +++ b/lustre/tests/lutf/python/tests/suite_udsp/test_udsp_basic_06.py @@ -0,0 +1,48 @@ +""" +@PRIMARY: N/A +@PRIMARY_DESC: Verify that UDSP rule designating a dst nid and router pair can be added +@SECONDARY: N/A +@DESIGN: N/A +@TESTCASE: +- configure LNet +- add udsp rule designating a destination nid and router pair +- verify that the rule has been added +""" + +import os +import yaml +import lnetconfig +from lutf import agents, me +from lutf_basetest import * +from lnet import TheLNet +from lutf_exception import LUTFError +from lnet_helpers import LNetHelpers +from lutf_cmd import lutf_exec_local_cmd + + +test_action_list = [{'udsp_cmd': "add --rte 192.168.122.50@tcp1 --dst 192.168.122.102@tcp2", + 'show_res': {'udsp': [{'idx': 0, 'src': 'NA', 'dst': '192.168.122.102@tcp2', 'rte': '192.168.122.50@tcp1'}]}}] + +def run(): + la = agents.keys() + if len(la) < 1: + return lutfrc(LUTF_TEST_SKIP, "No agents to run test") + try: + t = LNetHelpers(target=la[0]) + t.configure_lnet() + if not t.check_udsp_present(): + return lutfrc(LUTF_TEST_SKIP, "UDSP feature is missing") + rc = t.check_udsp_empty() + if not rc: + return lutfrc(LUTF_TEST_FAIL) + rc = t.exec_udsp_cmd(test_action_list[0]['udsp_cmd']) + rc = t.check_udsp_expected(test_action_list[0]['show_res']) + if not rc: + return lutfrc(LUTF_TEST_FAIL) + rc = t.cleanup_udsp() + return lutfrc(LUTF_TEST_PASS) + except Exception as e: + t.uninit() + raise e + + diff --git a/lustre/tests/lutf/python/tests/suite_udsp/test_udsp_basic_err_01.py b/lustre/tests/lutf/python/tests/suite_udsp/test_udsp_basic_err_01.py new file mode 100755 index 0000000..4c1f660 --- /dev/null +++ b/lustre/tests/lutf/python/tests/suite_udsp/test_udsp_basic_err_01.py @@ -0,0 +1,59 @@ +""" +@PRIMARY: N/A +@PRIMARY_DESC: Verify that using incorrect idx parameter value with UDSP lnetctl resuts in an error +@SECONDARY: N/A +@DESIGN: N/A +@TESTCASE: +- configure LNet +- add udsp rule prioritizing local network +- attempt to delete the rule providing incorrect idx parameter +- verify that the rule is not deleted +""" + +import os +import yaml +import lnetconfig +from lutf import agents, me +from lutf_basetest import * +from lnet import TheLNet +from lutf_exception import LUTFError +from lnet_helpers import LNetHelpers +from lutf_cmd import lutf_exec_local_cmd + + +test_action_list = [{'udsp_cmd': "add --src tcp1", + 'show_res': {'udsp': [{'idx': 0, 'src': 'tcp1', 'dst': 'NA', 'rte': 'NA', 'action': {'priority': 0}}]}}, + {'udsp_cmd': "del --idx=20", + 'show_res': {'udsp': [{'idx': 0, 'src': 'tcp1', 'dst': 'NA', 'rte': 'NA', 'action': {'priority': 0}}]}}, + {'udsp_cmd': "del --idx=-1", + 'show_res': {'udsp': [{'idx': 0, 'src': 'tcp1', 'dst': 'NA', 'rte': 'NA', 'action': {'priority': 0}}]}}, + {'udsp_cmd': "del --idx=abcd", + 'show_res': {'udsp': [{'idx': 0, 'src': 'tcp1', 'dst': 'NA', 'rte': 'NA', 'action': {'priority': 0}}]}}] + +def run(): + la = agents.keys() + if len(la) < 1: + return lutfrc(LUTF_TEST_SKIP, "No agents to run test") + try: + t = LNetHelpers(target=la[0]) + t.configure_lnet() + if not t.check_udsp_present(): + return lutfrc(LUTF_TEST_SKIP, "UDSP feature is missing") + rc = t.check_udsp_empty() + if not rc: + return lutfrc(LUTF_TEST_FAIL) + count = 0 + for test_action in test_action_list: + count+=1 + print("Test action: ", count) + rc = t.exec_udsp_cmd(test_action['udsp_cmd']) + rc = t.check_udsp_expected(test_action['show_res']) + if not rc: + return lutfrc(LUTF_TEST_FAIL) + rc = t.cleanup_udsp() + return lutfrc(LUTF_TEST_PASS) + except Exception as e: + t.uninit() + raise e + + diff --git a/lustre/tests/lutf/python/tests/suite_udsp/test_udsp_basic_err_02.py b/lustre/tests/lutf/python/tests/suite_udsp/test_udsp_basic_err_02.py new file mode 100755 index 0000000..e64b222 --- /dev/null +++ b/lustre/tests/lutf/python/tests/suite_udsp/test_udsp_basic_err_02.py @@ -0,0 +1,93 @@ +""" +@PRIMARY: N/A +@PRIMARY_DESC: Verify that providing incorrect parameters to UDSP lnetctl results in an error +@SECONDARY: N/A +@DESIGN: N/A +@TESTCASE: +- configure LNet +- attempt to add udsp rule providing incorrect parameters +- verify that attempts fail with an error +""" + +import os +import yaml +import lnetconfig +from lutf import agents, me +from lutf_basetest import * +from lnet import TheLNet +from lutf_exception import LUTFError +from lnet_helpers import LNetHelpers +from lutf_cmd import lutf_exec_local_cmd + + +test_action_list = [{'udsp_cmd': "add --src abcd", + 'fail_res': "ailed to parse src"}, + {'udsp_cmd': "add --src tcppp", + 'fail_res': "ailed to parse src"}, + {'udsp_cmd': "add --src t", + 'fail_res': "ailed to parse src"}, + {'udsp_cmd': "add --src o2ibbb", + 'fail_res': "ailed to parse src"}, + {'udsp_cmd': "add --src o2i", + 'fail_res': "ailed to parse src"}, + {'udsp_cmd': "add --src 10000000000000000000000000000001", + 'fail_res': "ailed to parse src"}, + {'udsp_cmd': "add --dst abcd", + 'fail_res': "ailed to parse dst"}, + {'udsp_cmd': "add --dst tcppp", + 'fail_res': "ailed to parse dst"}, + {'udsp_cmd': "add --dst t", + 'fail_res': "ailed to parse dst"}, + {'udsp_cmd': "add --dst o2ibbb", + 'fail_res': "ailed to parse dst"}, + {'udsp_cmd': "add --dst o2i", + 'fail_res': "ailed to parse dst"}, + {'udsp_cmd': "add --dst 10000000000000000000000000000001", + 'fail_res': "ailed to parse dst"}, + {'udsp_cmd': "add --rte abcd --dst tcp", + 'fail_res': "ailed to parse rte"}, + {'udsp_cmd': "add --rte tcppp --dst tcp", + 'fail_res': "ailed to parse rte"}, + {'udsp_cmd': "add --rte t --dst tcp", + 'fail_res': "ailed to parse rte"}, + {'udsp_cmd': "add --rte o2ibbb --dst tcp", + 'fail_res': "ailed to parse rte"}, + {'udsp_cmd': "add --rte o2i --dst tcp", + 'fail_res': "ailed to parse rte"}, + {'udsp_cmd': "add --rte 10000000000000000000000000000001 --dst tcp", + 'fail_res': "ailed to parse rte"}] + +def run(): + la = agents.keys() + if len(la) < 1: + return lutfrc(LUTF_TEST_SKIP, "No agents to run test") + try: + t = LNetHelpers(target=la[0]) + t.configure_lnet() + if not t.check_udsp_present(): + return lutfrc(LUTF_TEST_SKIP, "UDSP feature is missing") + rc = t.check_udsp_empty() + if not rc: + return lutfrc(LUTF_TEST_FAIL) + count = 0 + for test_action in test_action_list: + count+=1 + print("Test action: ", count) + """ + Expect the command to fail with an exception. + Look for the specific error message. + """ + try: + rc = t.exec_udsp_cmd(test_action['udsp_cmd']) + return lutfrc(LUTF_TEST_FAIL) + except Exception as e: + if test_action['fail_res'] not in e.msg: + return lutfrc(LUTF_TEST_FAIL) + + rc = t.cleanup_udsp() + return lutfrc(LUTF_TEST_PASS) + except Exception as e: + t.uninit() + raise e + + diff --git a/lustre/tests/lutf/python/tests/suite_udsp/test_udsp_multi_net_01.py b/lustre/tests/lutf/python/tests/suite_udsp/test_udsp_multi_net_01.py new file mode 100755 index 0000000..e50d27e --- /dev/null +++ b/lustre/tests/lutf/python/tests/suite_udsp/test_udsp_multi_net_01.py @@ -0,0 +1,129 @@ +""" +@PRIMARY: N/A +@PRIMARY_DESC: Verify functionality of UDSP rule that prioritizes a source net +@SECONDARY: N/A +@DESIGN: N/A +@TESTCASE: +- configure two networks, multiple nids per network +- add udsp rule that gives one of the networks highest priority +- generate traffic +- verify that the network with the highest priority was used +""" + +import os +import yaml +import lnetconfig +from lutf import agents, me +from lutf_basetest import * +from lnet import TheLNet +from lutf_exception import LUTFError +from lnet_helpers import LNetHelpers +from lustre_node import SimpleLustreNode + +MIN_NODES = 2 +MIN_IFS_PER_NODE = 4 +PING_TIMES = 10 +PING_NID_NUM = 0 +LOCAL_NETS = ['tcp', 'tcp1'] +USE_NET_NUM = [0] + +class TestLustreTraffic: + def __init__(self, target=None): + self.lh = LNetHelpers(os.path.abspath(__file__), target=target) + self.sln = SimpleLustreNode(os.path.abspath(__file__), target=target) + +def getStatNet(stats_dict, net_list, net_num, nid_stat_str): + stat_val = 0 + for x in stats_dict: + if (x['net type'] == net_list[net_num]): + for y in x['local NI(s)']: + stat_val += y['statistics'][nid_stat_str] + return stat_val + + +def run(): + la = agents.keys() + if len(la) < MIN_NODES: + return lutfrc(LUTF_TEST_SKIP, "Not enough agents to run the test") + nodes = [] + try: + for i in range(0, MIN_NODES): + node = TestLustreTraffic(la[i]) + t = node.lh + intfs = t.get_available_devs() + if len(intfs) < MIN_IFS_PER_NODE: + return lutfrc(LUTF_TEST_SKIP, "Not enough interfaces") + t.configure_lnet() + if not t.check_udsp_present(): + return lutfrc(LUTF_TEST_SKIP, "UDSP feature is missing") + for j, net in enumerate(LOCAL_NETS): + half = len(intfs)//len(LOCAL_NETS) + net_intfs_list = intfs[half*(j):half*(j+1)] + t.configure_net(net, net_intfs_list) + t.set_discovery(1) + nodes.append(node) + + main = nodes[1] + main_nids = main.lh.list_nids() + agent = nodes[0] + agent_nids = agent.lh.list_nids() + + # discover all the peers from main + if len(main.lh.discover(agent_nids[0])) == 0: + return lutfrc(LUTF_TEST_FAIL, "unable to discover" , + target=agent_nids[0]) + + rc = main.lh.check_udsp_empty() + if not rc: + return lutfrc(LUTF_TEST_FAIL, "UDSP list not empty") + + for net_num in USE_NET_NUM: + rc = main.lh.exec_udsp_cmd(" add --src "+LOCAL_NETS[net_num]) + + before_stats_main = main.lh.get_net_stats() + print(before_stats_main) + + for i in range(0, PING_TIMES): + rc = main.lh.exec_ping(agent_nids[PING_NID_NUM]) + if not rc: + return lutfrc(LUTF_TEST_FAIL, "ping failed") + + after_stats_main = main.lh.get_net_stats() + print(after_stats_main) + + send_count_before = {} + send_count_after = {} + total_send_count_before = 0 + total_send_count_after = 0 + for net_num in USE_NET_NUM: + send_count_before[net_num] = getStatNet(before_stats_main, LOCAL_NETS, net_num, 'send_count') + total_send_count_before += send_count_before[net_num] + send_count_after[net_num] = getStatNet(after_stats_main, LOCAL_NETS, net_num, 'send_count') + total_send_count_after += send_count_after[net_num] + + print(send_count_before, send_count_after) + + + # Check stats: + # 1) expect the total send_count to be no less than the number of pings issued + # 2) expect the send count on the preferred net to increase by no less than the + # number of pings issued + if (total_send_count_after - total_send_count_before) < PING_TIMES: + return lutfrc(LUTF_TEST_FAIL, "total send count mismatch") + + for net_num in USE_NET_NUM: + if abs(send_count_after[net_num] - send_count_before[net_num]) < PING_TIMES: + print("send count increase on network ", LOCAL_NETS[net_num], + " insufficient. Expected ", PING_TIMES, " got", + abs(send_count_after[net_num] - send_count_before[net_num])) + return lutfrc(LUTF_TEST_FAIL, "send count increase insufficient") + + for n in nodes: + n.lh.unconfigure_lnet() + + return lutfrc(LUTF_TEST_PASS) + except Exception as e: + for n in nodes: + n.lh.uninit() + raise e + diff --git a/lustre/tests/lutf/python/tests/suite_udsp/test_udsp_multi_net_02.py b/lustre/tests/lutf/python/tests/suite_udsp/test_udsp_multi_net_02.py new file mode 100755 index 0000000..ed770b1 --- /dev/null +++ b/lustre/tests/lutf/python/tests/suite_udsp/test_udsp_multi_net_02.py @@ -0,0 +1,130 @@ +""" +@PRIMARY: N/A +@PRIMARY_DESC: Verify that UDSP rule prioritizing a source net works correctly if deleted and re-applied +@SECONDARY: N/A +@DESIGN: N/A +@TESTCASE: +- configure two networks, multiple nids per network +- add udsp rule that gives one of the networks highest priority +- generate traffic +- verify that the network with the highest priority was used +- delete the rule and repeat for the second network +""" + +import os +import yaml +import lnetconfig +from lutf import agents, me +from lutf_basetest import * +from lnet import TheLNet +from lutf_exception import LUTFError +from lnet_helpers import LNetHelpers +from lustre_node import SimpleLustreNode + +MIN_NODES = 2 +MIN_IFS_PER_NODE = 4 +PING_TIMES = 10 +PING_NID_NUM = 0 +LOCAL_NETS = ['tcp', 'tcp1'] +USE_NET_NUM = [0, 1] + +class TestLustreTraffic: + def __init__(self, target=None): + self.lh = LNetHelpers(os.path.abspath(__file__), target=target) + self.sln = SimpleLustreNode(os.path.abspath(__file__), target=target) + +def getStatNet(stats_dict, net_list, net_num, nid_stat_str): + stat_val = 0 + for x in stats_dict: + if (x['net type'] == net_list[net_num]): + for y in x['local NI(s)']: + stat_val += y['statistics'][nid_stat_str] + return stat_val + + +def run(): + la = agents.keys() + if len(la) < MIN_NODES: + return lutfrc(LUTF_TEST_SKIP, "Not enough agents to run the test") + nodes = [] + try: + for i in range(0, MIN_NODES): + node = TestLustreTraffic(la[i]) + t = node.lh + intfs = t.get_available_devs() + if len(intfs) < MIN_IFS_PER_NODE: + return lutfrc(LUTF_TEST_SKIP, "Not enough interfaces") + t.configure_lnet() + if not t.check_udsp_present(): + return lutfrc(LUTF_TEST_SKIP, "UDSP feature is missing") + for j, net in enumerate(LOCAL_NETS): + half = len(intfs)//len(LOCAL_NETS) + net_intfs_list = intfs[half*(j):half*(j+1)] + t.configure_net(net, net_intfs_list) + t.set_discovery(1) + nodes.append(node) + + main = nodes[1] + main_nids = main.lh.list_nids() + agent = nodes[0] + agent_nids = agent.lh.list_nids() + + # discover all the peers from main + if len(main.lh.discover(agent_nids[0])) == 0: + return lutfrc(LUTF_TEST_FAIL, "unable to discover" , + target=agent_nids[0]) + + for prio_net_num in USE_NET_NUM: + + rc = main.lh.check_udsp_empty() + if not rc: + return lutfrc(LUTF_TEST_FAIL, "UDSP list not empty") + + rc = main.lh.exec_udsp_cmd(" add --src "+LOCAL_NETS[prio_net_num]) + + before_stats_main = main.lh.get_net_stats() + #print(before_stats_main) + + for i in range(0, PING_TIMES): + rc = main.lh.exec_ping(agent_nids[PING_NID_NUM]) + if not rc: + return lutfrc(LUTF_TEST_FAIL, "ping failed") + + after_stats_main = main.lh.get_net_stats() + #print(after_stats_main) + + send_count_before = {} + send_count_after = {} + total_send_count_before = 0 + total_send_count_after = 0 + + send_count_before[prio_net_num] = getStatNet(before_stats_main, LOCAL_NETS, prio_net_num, 'send_count') + total_send_count_before += send_count_before[prio_net_num] + send_count_after[prio_net_num] = getStatNet(after_stats_main, LOCAL_NETS, prio_net_num, 'send_count') + total_send_count_after += send_count_after[prio_net_num] + + #print(send_count_before, send_count_after) + + # Check stats: + # 1) expect the total send_count to be no less than the number of pings issued + # 2) expect the send count on the preferred net to increase by no less than the + # number of pings issued + if (total_send_count_after - total_send_count_before) < PING_TIMES: + return lutfrc(LUTF_TEST_FAIL, "total send count mismatch") + + if abs(send_count_after[prio_net_num] - send_count_before[prio_net_num]) < PING_TIMES: + print("send count increase on network ", LOCAL_NETS[prio_net_num], + " insufficient. Expected ", PING_TIMES, " got", + abs(send_count_after[prio_net_num] - send_count_before[prio_net_num])) + return lutfrc(LUTF_TEST_FAIL, "send count increase insufficient") + + rc = main.lh.exec_udsp_cmd(" del --idx 0") + for n in nodes: + n.lh.unconfigure_lnet() + + return lutfrc(LUTF_TEST_PASS) + except Exception as e: + for n in nodes: + n.lh.uninit() + raise e + diff --git a/lustre/tests/lutf/python/tests/suite_udsp/test_udsp_multi_net_03.py b/lustre/tests/lutf/python/tests/suite_udsp/test_udsp_multi_net_03.py new file mode 100755 index 0000000..8b17ecf --- /dev/null +++ b/lustre/tests/lutf/python/tests/suite_udsp/test_udsp_multi_net_03.py @@ -0,0 +1,160 @@ +""" +@PRIMARY: N/A +@PRIMARY_DESC: Verify that UDSP rules prioritizing source net and source nid work correctly in combination +@SECONDARY: N/A +@DESIGN: N/A +@TESTCASE: +- configure two networks, multiple nids per network +- add udsp rule that gives one of the networks highest priority +- add udsp rule that gives the highest priority to one of the nids on the prioritized network +- generate traffic +- verify that the prioritized nid on the network with the highest priority was used +""" + +import os +import yaml +import lnetconfig +from lutf import agents, me +from lutf_basetest import * +from lnet import TheLNet +from lutf_exception import LUTFError +from lnet_helpers import LNetHelpers +from lustre_node import SimpleLustreNode + +MIN_NODES = 2 +MIN_IFS_PER_NODE = 4 +PING_TIMES = 10 +PING_NID_NUM = 0 +LOCAL_NETS = ['tcp', 'tcp1'] +USE_NET_NUM = [0] +USE_NID_NUM = 0 + +class TestLustreTraffic: + def __init__(self, target=None): + self.lh = LNetHelpers(os.path.abspath(__file__), target=target) + self.sln = SimpleLustreNode(os.path.abspath(__file__), target=target) + + +def getStatNIDStr(stats_dict, nid_str, nid_stat_str): + for x in stats_dict: + for y in x['local NI(s)']: + if y['nid'] == nid_str: + return y['statistics'][nid_stat_str] + return -1 + +def getStatNet(stats_dict, net_list, net_num, nid_stat_str): + stat_val = 0 + for x in stats_dict: + if (x['net type'] == net_list[net_num]): + for y in x['local NI(s)']: + stat_val += y['statistics'][nid_stat_str] + return stat_val + +def getNetNIDbyIdx(stats_dict, net_name, nid_idx): + nid_str = "" + for x in stats_dict: + if (x['net type'] == net_name): + nid_str = x['local NI(s)'][nid_idx]['nid'] + break + return nid_str + +def run(): + la = agents.keys() + if len(la) < MIN_NODES: + return lutfrc(LUTF_TEST_SKIP, "Not enough agents to run the test") + nodes = [] + try: + for i in range(0, MIN_NODES): + node = TestLustreTraffic(la[i]) + t = node.lh + intfs = t.get_available_devs() + if len(intfs) < MIN_IFS_PER_NODE: + return lutfrc(LUTF_TEST_SKIP, "Not enough interfaces") + t.configure_lnet() + if not t.check_udsp_present(): + return lutfrc(LUTF_TEST_SKIP, "UDSP feature is missing") + for j, net in enumerate(LOCAL_NETS): + half = len(intfs)//len(LOCAL_NETS) + net_intfs_list = intfs[half*(j):half*(j+1)] + t.configure_net(net, net_intfs_list) + t.set_discovery(1) + nodes.append(node) + + main = nodes[1] + main_nids = main.lh.list_nids() + agent = nodes[0] + agent_nids = agent.lh.list_nids() + + # discover all the peers from main + if len(main.lh.discover(agent_nids[0])) == 0: + return lutfrc(LUTF_TEST_FAIL, "unable to discover" , + target=agent_nids[0]) + + rc = main.lh.check_udsp_empty() + if not rc: + return lutfrc(LUTF_TEST_FAIL, "UDSP list not empty") + + for net_num in USE_NET_NUM: + rc = main.lh.exec_udsp_cmd(" add --src "+LOCAL_NETS[net_num]) + before_stats_main = main.lh.get_net_stats() + nid_str = getNetNIDbyIdx(before_stats_main, LOCAL_NETS[net_num], USE_NID_NUM) + rc = main.lh.exec_udsp_cmd(" add --src "+nid_str) + + before_stats_main = main.lh.get_net_stats() + #print(before_stats_main) + + for i in range(0, PING_TIMES): + rc = main.lh.exec_ping(agent_nids[PING_NID_NUM]) + if not rc: + return lutfrc(LUTF_TEST_FAIL, "ping failed") + + after_stats_main = main.lh.get_net_stats() + #print(after_stats_main) + + send_count_before = {} + send_count_after = {} + total_send_count_before = 0 + total_send_count_after = 0 + send_count_nid_before = 0 + send_count_nid_after = 0 + for net_num in USE_NET_NUM: + send_count_before[net_num] = getStatNet(before_stats_main, LOCAL_NETS, net_num, 'send_count') + total_send_count_before += send_count_before[net_num] + send_count_after[net_num] = getStatNet(after_stats_main, LOCAL_NETS, net_num, 'send_count') + total_send_count_after += send_count_after[net_num] + + send_count_nid_before = getStatNIDStr(before_stats_main, nid_str, 'send_count') + send_count_nid_after = getStatNIDStr(after_stats_main, nid_str, 'send_count') + #print(send_count_before, send_count_after) + #print(send_count_nid_before, send_count_nid_after) + + # Check stats: + # 1) expect the total send_count to be no less than the number of pings issued + # 2) expect the send count on the preferred net to increase by no less than the + # number of pings issued + # 3) expect the preferred nid on the preferred net to be used + if (total_send_count_after - total_send_count_before) < PING_TIMES: + return lutfrc(LUTF_TEST_FAIL, "total send count mismatch") + + for net_num in USE_NET_NUM: + if abs(send_count_after[net_num] - send_count_before[net_num]) < PING_TIMES: + print("send count increase on network ", LOCAL_NETS[net_num], + " insufficient. Expected ", PING_TIMES, " got ", + abs(send_count_after[net_num] - send_count_before[net_num])) + return lutfrc(LUTF_TEST_FAIL, "send count increase insufficient") + + if (send_count_nid_after - send_count_nid_before) < PING_TIMES: + print("send count increase on the preferred nid insufficient. Expected ", + PING_TIMES, "got ", (send_count_nid_after - send_count_nid_before)) + return lutfrc(LUTF_TEST_FAIL, "send count increase on the preferred nid insufficient") + + + for n in nodes: + n.lh.unconfigure_lnet() + + return lutfrc(LUTF_TEST_PASS) + except Exception as e: + for n in nodes: + n.lh.uninit() + raise e + diff --git a/lustre/tests/lutf/python/tests/suite_udsp/test_udsp_multi_net_04.py b/lustre/tests/lutf/python/tests/suite_udsp/test_udsp_multi_net_04.py new file mode 100755 index 0000000..6d5312a --- /dev/null +++ b/lustre/tests/lutf/python/tests/suite_udsp/test_udsp_multi_net_04.py @@ -0,0 +1,160 @@ +""" +@PRIMARY: N/A +@PRIMARY_DESC: Verify that UDSP rules prioritizing source net and source nid work correctly in combination +@SECONDARY: N/A +@DESIGN: N/A +@TESTCASE: +- configure two networks, multiple nids per network +- add udsp rule that gives one of the networks highest priority +- add udsp rule that gives the highest priority to one of the nids on the lower priority network +- generate traffic +- verify that the nids on the network with the highest priority were used +""" + +import os +import yaml +import lnetconfig +from lutf import agents, me +from lutf_basetest import * +from lnet import TheLNet +from lutf_exception import LUTFError +from lnet_helpers import LNetHelpers +from lustre_node import SimpleLustreNode + +MIN_NODES = 2 +MIN_IFS_PER_NODE = 4 +PING_TIMES = 10 +PING_NID_NUM = 0 +LOCAL_NETS = ['tcp', 'tcp1'] +USE_NET_NUM = [0] +USE_NID_NUM = 0 + +class TestLustreTraffic: + def __init__(self, target=None): + self.lh = LNetHelpers(os.path.abspath(__file__), target=target) + self.sln = SimpleLustreNode(os.path.abspath(__file__), target=target) + + +def getStatNIDStr(stats_dict, nid_str, nid_stat_str): + for x in stats_dict: + for y in x['local NI(s)']: + if y['nid'] == nid_str: + return y['statistics'][nid_stat_str] + return -1 + +def getStatNet(stats_dict, net_list, net_num, nid_stat_str): + stat_val = 0 + for x in stats_dict: + if (x['net type'] == net_list[net_num]): + for y in x['local NI(s)']: + stat_val += y['statistics'][nid_stat_str] + return stat_val + +def getNetNIDbyIdx(stats_dict, net_name, nid_idx): + nid_str = "" + for x in stats_dict: + if (x['net type'] == net_name): + nid_str = x['local NI(s)'][nid_idx]['nid'] + break + return nid_str + +def run(): + la = agents.keys() + if len(la) < MIN_NODES: + return lutfrc(LUTF_TEST_SKIP, "Not enough agents to run the test") + nodes = [] + try: + for i in range(0, MIN_NODES): + node = TestLustreTraffic(la[i]) + t = node.lh + intfs = t.get_available_devs() + if len(intfs) < MIN_IFS_PER_NODE: + return lutfrc(LUTF_TEST_SKIP, "Not enough interfaces") + t.configure_lnet() + if not t.check_udsp_present(): + return lutfrc(LUTF_TEST_SKIP, "UDSP feature is missing") + for j, net in enumerate(LOCAL_NETS): + half = len(intfs)//len(LOCAL_NETS) + net_intfs_list = intfs[half*(j):half*(j+1)] + t.configure_net(net, net_intfs_list) + t.set_discovery(1) + nodes.append(node) + + main = nodes[1] + main_nids = main.lh.list_nids() + agent = nodes[0] + agent_nids = agent.lh.list_nids() + + # discover all the peers from main + if len(main.lh.discover(agent_nids[0])) == 0: + return lutfrc(LUTF_TEST_FAIL, "unable to discover" , + target=agent_nids[0]) + + rc = main.lh.check_udsp_empty() + if not rc: + return lutfrc(LUTF_TEST_FAIL) + + for net_num in USE_NET_NUM: + rc = main.lh.exec_udsp_cmd(" add --src "+LOCAL_NETS[net_num]) + before_stats_main = main.lh.get_net_stats() + nid_str = getNetNIDbyIdx(before_stats_main, LOCAL_NETS[(net_num+1) % (len(LOCAL_NETS))], USE_NID_NUM) + rc = main.lh.exec_udsp_cmd(" add --src "+nid_str) + + before_stats_main = main.lh.get_net_stats() + #print(before_stats_main) + + for i in range(0, PING_TIMES): + rc = main.lh.exec_ping(agent_nids[PING_NID_NUM]) + if not rc: + return lutfrc(LUTF_TEST_FAILi, "ping failed") + + after_stats_main = main.lh.get_net_stats() + #print(after_stats_main) + + send_count_before = {} + send_count_after = {} + total_send_count_before = 0 + total_send_count_after = 0 + send_count_nid_before = 0 + send_count_nid_after = 0 + for net_num in USE_NET_NUM: + send_count_before[net_num] = getStatNet(before_stats_main, LOCAL_NETS, net_num, 'send_count') + total_send_count_before += send_count_before[net_num] + send_count_after[net_num] = getStatNet(after_stats_main, LOCAL_NETS, net_num, 'send_count') + total_send_count_after += send_count_after[net_num] + + send_count_nid_before = getStatNIDStr(before_stats_main, nid_str, 'send_count') + send_count_nid_after = getStatNIDStr(after_stats_main, nid_str, 'send_count') + #print(send_count_before, send_count_after) + #print(send_count_nid_before, send_count_nid_after) + + # Check stats: + # 1) expect the total send_count to be no less than the number of pings issued + # 2) expect the send count on the preferred net to increase by no less than the + # number of pings issued + # 3) expect the preferred nid on the non-preferred net not to be used + if (total_send_count_after - total_send_count_before) < PING_TIMES: + return lutfrc(LUTF_TEST_FAIL, "total send count mismatch") + + for net_num in USE_NET_NUM: + if abs(send_count_after[net_num] - send_count_before[net_num]) < PING_TIMES: + print("send count increase on network ", LOCAL_NETS[net_num], + " insufficient. Expected ", PING_TIMES, " got ", + abs(send_count_after[net_num] - send_count_before[net_num])) + return lutfrc(LUTF_TEST_FAIL, "send count increase insufficient") + + if (send_count_nid_after - send_count_nid_before) >= PING_TIMES//2: + print("send count increase on the preferred nid is unexpected. Expected none, ", + "got ", (send_count_nid_after - send_count_nid_before)) + return lutfrc(LUTF_TEST_FAIL, "send count increase on the preferred nid is unexpected.") + + + for n in nodes: + n.lh.unconfigure_lnet() + + return lutfrc(LUTF_TEST_PASS) + except Exception as e: + for n in nodes: + n.lh.uninit() + raise e + diff --git a/lustre/tests/lutf/python/tests/suite_udsp/test_udsp_multi_net_05.py b/lustre/tests/lutf/python/tests/suite_udsp/test_udsp_multi_net_05.py new file mode 100755 index 0000000..2e24adc --- /dev/null +++ b/lustre/tests/lutf/python/tests/suite_udsp/test_udsp_multi_net_05.py @@ -0,0 +1,148 @@ +""" +@PRIMARY: N/A +@PRIMARY_DESC: Verify that UDSP rules prioritizing source net on two peers work correctly in combination +@SECONDARY: N/A +@DESIGN: N/A +@TESTCASE: +- configure two networks, multiple nids per network +- add udsp rule that gives one of the networks highest priority on the local node +- add udsp rule that gives priority to the other network on the remote node +- generate traffic +- verify that the network with the highest priority was used +""" + +import os +import yaml +import lnetconfig +from lutf import agents, me +from lutf_basetest import * +from lnet import TheLNet +from lutf_exception import LUTFError +from lnet_helpers import LNetHelpers +from lustre_node import SimpleLustreNode + +MIN_NODES = 2 +MIN_IFS_PER_NODE = 4 +PING_TIMES = 10 +PING_NID_NUM = 0 +LOCAL_NETS = ['tcp', 'tcp1'] +USE_NET_NUM = [0] + +class TestLustreTraffic: + def __init__(self, target=None): + self.lh = LNetHelpers(os.path.abspath(__file__), target=target) + self.sln = SimpleLustreNode(os.path.abspath(__file__), target=target) + + +def getStatNIDStr(stats_dict, nid_str, nid_stat_str): + for x in stats_dict: + for y in x['local NI(s)']: + if y['nid'] == nid_str: + return y['statistics'][nid_stat_str] + return -1 + +def getStatNet(stats_dict, net_list, net_num, nid_stat_str): + stat_val = 0 + for x in stats_dict: + if (x['net type'] == net_list[net_num]): + for y in x['local NI(s)']: + stat_val += y['statistics'][nid_stat_str] + return stat_val + +def getNetNIDbyIdx(stats_dict, net_name, nid_idx): + nid_str = "" + for x in stats_dict: + if (x['net type'] == net_name): + nid_str = x['local NI(s)'][nid_idx]['nid'] + break + return nid_str + +def run(): + la = agents.keys() + if len(la) < MIN_NODES: + return lutfrc(LUTF_TEST_SKIP, "Not enough agents to run the test") + nodes = [] + try: + for i in range(0, MIN_NODES): + node = TestLustreTraffic(la[i]) + t = node.lh + intfs = t.get_available_devs() + if len(intfs) < MIN_IFS_PER_NODE: + return lutfrc(LUTF_TEST_SKIP, "Not enough interfaces") + t.configure_lnet() + if not t.check_udsp_present(): + return lutfrc(LUTF_TEST_SKIP, "UDSP feature is missing") + for j, net in enumerate(LOCAL_NETS): + half = len(intfs)//len(LOCAL_NETS) + net_intfs_list = intfs[half*(j):half*(j+1)] + t.configure_net(net, net_intfs_list) + t.set_discovery(1) + nodes.append(node) + + main = nodes[1] + main_nids = main.lh.list_nids() + agent = nodes[0] + agent_nids = agent.lh.list_nids() + + # discover all the peers from main + if len(main.lh.discover(agent_nids[0])) == 0: + return lutfrc(LUTF_TEST_FAIL, "unable to discover" , + target=agent_nids[0]) + + rc = main.lh.check_udsp_empty() + if not rc: + return lutfrc(LUTF_TEST_FAIL, "UDSP list not empty") + + for net_num in USE_NET_NUM: + rc = main.lh.exec_udsp_cmd(" add --src "+LOCAL_NETS[net_num]) + # Prioritize the other net on the agent + rc = agent.lh.exec_udsp_cmd(" add --src "+LOCAL_NETS[(net_num+1) % (len(LOCAL_NETS))]) + + before_stats_main = main.lh.get_net_stats() + #print(before_stats_main) + + for i in range(0, PING_TIMES): + rc = main.lh.exec_ping(agent_nids[PING_NID_NUM]) + if not rc: + return lutfrc(LUTF_TEST_FAIL, "ping failed") + + after_stats_main = main.lh.get_net_stats() + #print(after_stats_main) + + send_count_before = {} + send_count_after = {} + total_send_count_before = 0 + total_send_count_after = 0 + send_count_nid_before = 0 + send_count_nid_after = 0 + for net_num in USE_NET_NUM: + send_count_before[net_num] = getStatNet(before_stats_main, LOCAL_NETS, net_num, 'send_count') + total_send_count_before += send_count_before[net_num] + send_count_after[net_num] = getStatNet(after_stats_main, LOCAL_NETS, net_num, 'send_count') + total_send_count_after += send_count_after[net_num] + + #print(send_count_before, send_count_after) + + # Check stats: + # 1) expect the total send_count to be no less than the number of pings issued + # 2) expect the send count on the preferred net to increase by no less than the + # number of pings issued + if (total_send_count_after - total_send_count_before) < PING_TIMES: + return lutfrc(LUTF_TEST_FAIL, "total send count mismatch") + + for net_num in USE_NET_NUM: + if abs(send_count_after[net_num] - send_count_before[net_num]) < PING_TIMES: + print("send count increase on network ", LOCAL_NETS[net_num], + " insufficient. Expected ", PING_TIMES, " got ", + abs(send_count_after[net_num] - send_count_before[net_num])) + return lutfrc(LUTF_TEST_FAIL, "send count increase insufficient.") + + for n in nodes: + n.lh.unconfigure_lnet() + + return lutfrc(LUTF_TEST_PASS) + except Exception as e: + for n in nodes: + n.lh.uninit() + raise e + diff --git a/lustre/tests/lutf/python/tests/suite_udsp/test_udsp_multi_net_err_01.py b/lustre/tests/lutf/python/tests/suite_udsp/test_udsp_multi_net_err_01.py new file mode 100755 index 0000000..039dca1 --- /dev/null +++ b/lustre/tests/lutf/python/tests/suite_udsp/test_udsp_multi_net_err_01.py @@ -0,0 +1,148 @@ +""" +@PRIMARY: N/A +@PRIMARY_DESC: Verify that adding a UDSP rule that prioritizes an unexisting network doesn't affect traffic distribution +@SECONDARY: N/A +@DESIGN: N/A +@TESTCASE: +- configure two networks, multiple nids per network +- add udsp rule that gives highest priority to an unexisting network +- generate traffic +- verify that traffic is evenly distributed across both networks +""" + +import os +import yaml +import lnetconfig +from lutf import agents, me +from lutf_basetest import * +from lnet import TheLNet +from lutf_exception import LUTFError +from lnet_helpers import LNetHelpers +from lustre_node import SimpleLustreNode + +MIN_NODES = 2 +MIN_IFS_PER_NODE = 4 +PING_TIMES = 10 +PING_NID_NUM = 0 +LOCAL_NETS = ['tcp', 'tcp1'] +USE_NET_NUM = [0, 1] +UNEX_NET = "tcp2" +EXP_UDSP_SHOW = {'udsp': [{'idx': 0, 'src': 'tcp2', 'dst': 'NA', 'rte': 'NA', 'action': {'priority': 0}}]} + +class TestLustreTraffic: + def __init__(self, target=None): + self.lh = LNetHelpers(os.path.abspath(__file__), target=target) + self.sln = SimpleLustreNode(os.path.abspath(__file__), target=target) + + +def getStatNIDStr(stats_dict, nid_str, nid_stat_str): + for x in stats_dict: + for y in x['local NI(s)']: + if y['nid'] == nid_str: + return y['statistics'][nid_stat_str] + return -1 + +def getStatNet(stats_dict, net_list, net_num, nid_stat_str): + stat_val = 0 + for x in stats_dict: + if (x['net type'] == net_list[net_num]): + for y in x['local NI(s)']: + stat_val += y['statistics'][nid_stat_str] + return stat_val + +def getNetNIDbyIdx(stats_dict, net_name, nid_idx): + nid_str = "" + for x in stats_dict: + if (x['net type'] == net_name): + nid_str = x['local NI(s)'][nid_idx]['nid'] + break + return nid_str + +def run(): + la = agents.keys() + if len(la) < MIN_NODES: + return lutfrc(LUTF_TEST_SKIP, "Not enough agents to run the test") + nodes = [] + try: + for i in range(0, MIN_NODES): + node = TestLustreTraffic(la[i]) + t = node.lh + intfs = t.get_available_devs() + if len(intfs) < MIN_IFS_PER_NODE: + return lutfrc(LUTF_TEST_SKIP, "Not enough interfaces") + t.configure_lnet() + if not t.check_udsp_present(): + return lutfrc(LUTF_TEST_SKIP, "UDSP feature is missing") + for j, net in enumerate(LOCAL_NETS): + half = len(intfs)//len(LOCAL_NETS) + net_intfs_list = intfs[half*(j):half*(j+1)] + t.configure_net(net, net_intfs_list) + t.set_discovery(1) + nodes.append(node) + + main = nodes[1] + main_nids = main.lh.list_nids() + agent = nodes[0] + agent_nids = agent.lh.list_nids() + + # discover all the peers from main + if len(main.lh.discover(agent_nids[0])) == 0: + return lutfrc(LUTF_TEST_FAIL, "unable to discover" , + target=agent_nids[0]) + + rc = main.lh.check_udsp_empty() + if not rc: + return lutfrc(LUTF_TEST_FAIL, "UDSP list not empty") + + # Prioritize unexisting net + rc = main.lh.exec_udsp_cmd(" add --src "+UNEX_NET) + # Verify the rule got added + rc = main.lh.check_udsp_expected(EXP_UDSP_SHOW) + if not rc: + return lutfrc(LUTF_TEST_FAIL) + + before_stats_main = main.lh.get_net_stats() + #print(before_stats_main) + + for i in range(0, PING_TIMES): + rc = main.lh.exec_ping(agent_nids[PING_NID_NUM]) + if not rc: + return lutfrc(LUTF_TEST_FAIL, "ping failed") + + after_stats_main = main.lh.get_net_stats() + #print(after_stats_main) + + send_count_before = {} + send_count_after = {} + total_send_count_before = 0 + total_send_count_after = 0 + for net_num in USE_NET_NUM: + send_count_before[net_num] = getStatNet(before_stats_main, LOCAL_NETS, net_num, 'send_count') + total_send_count_before += send_count_before[net_num] + send_count_after[net_num] = getStatNet(after_stats_main, LOCAL_NETS, net_num, 'send_count') + total_send_count_after += send_count_after[net_num] + + #print(send_count_before, send_count_after) + + # Check stats: + # 1) expect the total send_count to be no less than the number of pings issued + # 2) expect the send count to be even across nets (no preference) + if (total_send_count_after - total_send_count_before) < PING_TIMES: + return lutfrc(LUTF_TEST_FAIL, "total send count mismatch") + + for net_num in USE_NET_NUM: + if abs(send_count_after[net_num] - send_count_before[net_num]) > PING_TIMES//2: + print("send count increase on network ", LOCAL_NETS[net_num], + " excessive. Expected no more than", PING_TIMES//2, " got ", + abs(send_count_after[net_num] - send_count_before[net_num])) + return lutfrc(LUTF_TEST_FAIL, "send count increase excessive.") + + for n in nodes: + n.lh.unconfigure_lnet() + + return lutfrc(LUTF_TEST_PASS) + except Exception as e: + for n in nodes: + n.lh.uninit() + raise e + diff --git a/lustre/tests/lutf/python/tests/suite_udsp/test_udsp_multi_net_err_02.py b/lustre/tests/lutf/python/tests/suite_udsp/test_udsp_multi_net_err_02.py new file mode 100755 index 0000000..72c454d --- /dev/null +++ b/lustre/tests/lutf/python/tests/suite_udsp/test_udsp_multi_net_err_02.py @@ -0,0 +1,191 @@ +""" +@PRIMARY: N/A +@PRIMARY_DESC: Verify that deleting prioritized local net results no issues +@SECONDARY: N/A +@DESIGN: N/A +@TESTCASE: +- configure two networks, multiple nids per network +- add udsp rule that gives highest priority to one of the networks +- generate traffic +- verify that the prioritized network is used +- delete the prioritized network +- generate traffic +- verify that the remaining network is used +""" + +import os +import yaml +import lnetconfig +from lutf import agents, me +from lutf_basetest import * +from lnet import TheLNet +from lutf_exception import LUTFError +from lnet_helpers import LNetHelpers +from lustre_node import SimpleLustreNode + +MIN_NODES = 2 +MIN_IFS_PER_NODE = 4 +PING_TIMES = 10 +PING_NID_NUM = 0 +LOCAL_NETS = ['tcp', 'tcp1'] +USE_NET_NUM = [1] +PRIO_NET = "tcp1" +EXP_UDSP_SHOW = {'udsp': [{'idx': 0, 'src': 'tcp1', 'dst': 'NA', 'rte': 'NA', 'action': {'priority': 0}}]} + +class TestLustreTraffic: + def __init__(self, target=None): + self.lh = LNetHelpers(os.path.abspath(__file__), target=target) + self.sln = SimpleLustreNode(os.path.abspath(__file__), target=target) + + +def getStatNIDStr(stats_dict, nid_str, nid_stat_str): + for x in stats_dict: + for y in x['local NI(s)']: + if y['nid'] == nid_str: + return y['statistics'][nid_stat_str] + return -1 + +def getStatNet(stats_dict, net_list, net_num, nid_stat_str): + stat_val = 0 + for x in stats_dict: + if (x['net type'] == net_list[net_num]): + for y in x['local NI(s)']: + stat_val += y['statistics'][nid_stat_str] + return stat_val + +def getNetNIDbyIdx(stats_dict, net_name, nid_idx): + nid_str = "" + for x in stats_dict: + if (x['net type'] == net_name): + nid_str = x['local NI(s)'][nid_idx]['nid'] + break + return nid_str + +def run(): + la = agents.keys() + if len(la) < MIN_NODES: + return lutfrc(LUTF_TEST_SKIP, "Not enough agents to run the test") + nodes = [] + try: + for i in range(0, MIN_NODES): + node = TestLustreTraffic(la[i]) + t = node.lh + intfs = t.get_available_devs() + if len(intfs) < MIN_IFS_PER_NODE: + return lutfrc(LUTF_TEST_SKIP, "Not enough interfaces") + t.configure_lnet() + if not t.check_udsp_present(): + return lutfrc(LUTF_TEST_SKIP, "UDSP feature is missing") + for j, net in enumerate(LOCAL_NETS): + half = len(intfs)//len(LOCAL_NETS) + net_intfs_list = intfs[half*(j):half*(j+1)] + t.configure_net(net, net_intfs_list) + t.set_discovery(1) + nodes.append(node) + + main = nodes[1] + main_nids = main.lh.list_nids() + agent = nodes[0] + agent_nids = agent.lh.list_nids() + + # discover all the peers from main + if len(main.lh.discover(agent_nids[0])) == 0: + return lutfrc(LUTF_TEST_FAIL, "unable to discover" , + target=agent_nids[0]) + + rc = main.lh.check_udsp_empty() + if not rc: + return lutfrc(LUTF_TEST_FAIL, "UDSP list not empty") + + # Prioritize one of the nets + rc = main.lh.exec_udsp_cmd(" add --src "+PRIO_NET) + # Verify the rule got added + rc = main.lh.check_udsp_expected(EXP_UDSP_SHOW) + if not rc: + return lutfrc(LUTF_TEST_FAIL, "adding UDSP rule failed") + + before_stats_main = main.lh.get_net_stats() + #print(before_stats_main) + + for i in range(0, PING_TIMES): + rc = main.lh.exec_ping(agent_nids[PING_NID_NUM]) + if not rc: + return lutfrc(LUTF_TEST_FAIL, "ping failed") + + after_stats_main = main.lh.get_net_stats() + #print(after_stats_main) + + send_count_before = {} + send_count_after = {} + total_send_count_before = 0 + total_send_count_after = 0 + for net_num in USE_NET_NUM: + send_count_before[net_num] = getStatNet(before_stats_main, LOCAL_NETS, net_num, 'send_count') + total_send_count_before += send_count_before[net_num] + send_count_after[net_num] = getStatNet(after_stats_main, LOCAL_NETS, net_num, 'send_count') + total_send_count_after += send_count_after[net_num] + + #print(send_count_before, send_count_after) + + # Check stats: + # 1) expect the total send_count to be no less than the number of pings issued + # 2) expect the send count to increase by the number of pings issued on the preferred net + if (total_send_count_after - total_send_count_before) < PING_TIMES: + return lutfrc(LUTF_TEST_FAIL, "total send count mismatch") + + for net_num in USE_NET_NUM: + if abs(send_count_after[net_num] - send_count_before[net_num]) < PING_TIMES: + print("send count increase on network ", LOCAL_NETS[net_num], + " insufficient. Expected ", PING_TIMES, " got ", + abs(send_count_after[net_num] - send_count_before[net_num])) + return lutfrc(LUTF_TEST_FAIL, "send count increase insufficient") + + main.lh.unconfigure_net(PRIO_NET) + + before_stats_main = main.lh.get_net_stats() + #print(before_stats_main) + + for i in range(0, PING_TIMES): + rc = main.lh.exec_ping(agent_nids[PING_NID_NUM]) + if not rc: + return lutfrc(LUTF_TEST_FAIL, "ping failed") + + after_stats_main = main.lh.get_net_stats() + #print(after_stats_main) + + # Use remaining net + USE_NET_NUM[0] = 0 + + send_count_before = {} + send_count_after = {} + total_send_count_before = 0 + total_send_count_after = 0 + for net_num in USE_NET_NUM: + send_count_before[net_num] = getStatNet(before_stats_main, LOCAL_NETS, net_num, 'send_count') + total_send_count_before += send_count_before[net_num] + send_count_after[net_num] = getStatNet(after_stats_main, LOCAL_NETS, net_num, 'send_count') + total_send_count_after += send_count_after[net_num] + + #print(send_count_before, send_count_after) + + # Check stats: + # 1) expect the total send_count to be no less than the number of pings issued + # 2) expect the send count to increase by the number of pings issued on the preferred net + if (total_send_count_after - total_send_count_before) < PING_TIMES: + return lutfrc(LUTF_TEST_FAIL, "total send count mismatch") + + for net_num in USE_NET_NUM: + if abs(send_count_after[net_num] - send_count_before[net_num]) < PING_TIMES: + print("send count increase on network ", LOCAL_NETS[net_num], + " insufficient. Expected ", PING_TIMES, " got ", + abs(send_count_after[net_num] - send_count_before[net_num])) + return lutfrc(LUTF_TEST_FAIL, "send count increase insufficient") + for n in nodes: + n.lh.unconfigure_lnet() + + return lutfrc(LUTF_TEST_PASS) + except Exception as e: + for n in nodes: + n.lh.uninit() + raise e + diff --git a/lustre/tests/lutf/python/tests/suite_udsp/test_udsp_routed_net_01.py b/lustre/tests/lutf/python/tests/suite_udsp/test_udsp_routed_net_01.py new file mode 100755 index 0000000..5f067a5 --- /dev/null +++ b/lustre/tests/lutf/python/tests/suite_udsp/test_udsp_routed_net_01.py @@ -0,0 +1,243 @@ +""" +@PRIMARY: N/A +@PRIMARY_DESC: Verify that deleting prioritized local net results no issues +@SECONDARY: N/A +@DESIGN: N/A +@TESTCASE: +- configure single local network with two nids +- configure two gws each providing access to a remote network +- configure remote peer to have access to the remote network with two nids +- add udsp rule that designates a pair of a router and remote peer nid as preferred +- add udsp rule that gives the same remote peer nid highest priority +- generate traffic +- verify that the preferred remote peer nid an router were used +""" + +import os +import yaml +import lnetconfig +import time +from lutf import agents, me +from lutf_basetest import * +from lnet import TheLNet +from lutf_exception import LUTFError +from lnet_helpers import LNetHelpers +from lustre_node import SimpleLustreNode + +MIN_NODES = 4 +MIN_IFS_PER_NODE = 2 +GW_MIN_IFS = 4 +PING_TIMES = 10 +PING_NID_NUM = 0 +LOCAL_NETS = ['tcp'] +REMOTE_NETS = ['tcp1'] +USE_NET_NUM = [0] +MAIN_NODE_ID = 1 +REMOTE_NODE_ID = 3 +GW_NODE_IDS = [2, 0] + +class TestLustreTraffic: + def __init__(self, target=None): + self.lh = LNetHelpers(os.path.abspath(__file__), target=target) + self.sln = SimpleLustreNode(os.path.abspath(__file__), target=target) + + +def getStatNIDStr(stats_dict, nid_str, nid_stat_str): + for x in stats_dict: + for y in x['local NI(s)']: + if y['nid'] == nid_str: + return y['statistics'][nid_stat_str] + return -1 + +def getStatNet(stats_dict, net_list, net_num, nid_stat_str): + stat_val = 0 + for x in stats_dict: + if (x['net type'] == net_list[net_num]): + for y in x['local NI(s)']: + stat_val += y['statistics'][nid_stat_str] + return stat_val + +def getNetNIDbyIdx(stats_dict, net_name, nid_idx): + nid_str = "" + for x in stats_dict: + if (x['net type'] == net_name): + nid_str = x['local NI(s)'][nid_idx]['nid'] + break + return nid_str + +def run(): + la = agents.keys() + #if len(la) < MIN_NODES: + # return lutfrc(LUTF_TEST_SKIP, "Not enough agents to run the test") + nodes = [] + try: + # General config for all nodes + for i in range(0, MIN_NODES): + node = TestLustreTraffic(la[i]) + t = node.lh + if i in GW_NODE_IDS: + node_num_ifs = GW_MIN_IFS + else: + node_num_ifs = MIN_IFS_PER_NODE + intfs = t.get_available_devs() + if len(intfs) < node_num_ifs: + return lutfrc(LUTF_TEST_SKIP, "Not enough interfaces") + t.configure_lnet() + if not t.check_udsp_present(): + return lutfrc(LUTF_TEST_SKIP, "UDSP feature is missing") + + if i == MAIN_NODE_ID: + # Peer A + net = LOCAL_NETS[0] + net_intfs_list = intfs[0:node_num_ifs] + t.configure_net(net, net_intfs_list) + elif i == REMOTE_NODE_ID: + # Peer B + net = REMOTE_NETS[0] + net_intfs_list = intfs[0:node_num_ifs] + t.configure_net(net, net_intfs_list) + else: + # GW1 and GW2 + net = LOCAL_NETS[0] + net_intfs_list = intfs[0:node_num_ifs//2] + t.configure_net(net, net_intfs_list) + net = REMOTE_NETS[0] + net_intfs_list = intfs[node_num_ifs//2:node_num_ifs] + t.configure_net(net, net_intfs_list) + + t.set_discovery(1) + nodes.append(node) + + main = nodes[MAIN_NODE_ID] + main_nids = main.lh.list_nids() + remote = nodes[REMOTE_NODE_ID] + remote_nids = remote.lh.list_nids() + gw1 = nodes[GW_NODE_IDS[0]] + gw2 = nodes[GW_NODE_IDS[1]] + gw1_nids = gw1.lh.list_nids() + gw2_nids = gw2.lh.list_nids() + + #print("main nids: ", main_nids) + #print("remote nids: ", remote_nids) + #print("gw1 nids: ", gw1_nids) + #print("gw2 nids: ", gw2_nids) + + gw1.lh.api_set_routing(True) + gw2.lh.api_set_routing(True) + + gw1_remote_nids = [x for x in gw1_nids if REMOTE_NETS[0] in x] + gw1_local_nids = [x for x in gw1_nids if x not in gw1_remote_nids] + + gw2_remote_nids = [x for x in gw2_nids if REMOTE_NETS[0] in x] + gw2_local_nids = [x for x in gw2_nids if x not in gw2_remote_nids] + + #print("gw1 local/remote nids: ", gw1_local_nids, gw1_remote_nids) + #print("gw2 local/remote nids: ", gw2_local_nids, gw2_remote_nids) + + # discover the gateways from main and remote + if len(main.lh.exec_discover_cmd(gw1_local_nids[0])) == 0: + return lutfrc(LUTF_TEST_FAIL, "unable to discover" , + target=gw1_local_nids[0]) + if len(main.lh.exec_discover_cmd(gw2_local_nids[0])) == 0: + return lutfrc(LUTF_TEST_FAIL, "unable to discover" , + target=gw2_local_nids[0]) + if len(remote.lh.exec_discover_cmd(gw1_remote_nids[0])) == 0: + return lutfrc(LUTF_TEST_FAIL, "unable to discover" , + target=gw1_remote_nids[0]) + if len(remote.lh.exec_discover_cmd(gw2_remote_nids[0])) == 0: + return lutfrc(LUTF_TEST_FAIL, "unable to discover" , + target=gw2_remote_nids[0]) + + # setup the routing + rt1 = "--net " + REMOTE_NETS[0] + " --gateway " + gw1_local_nids[0] + rt2 = "--net " + REMOTE_NETS[0] + " --gateway " + gw2_local_nids[0] + rc = main.lh.exec_route_cmd(" add "+rt1) + rc = main.lh.exec_route_cmd(" add "+rt2) + rt1 = "--net " + LOCAL_NETS[0] + " --gateway " + gw1_remote_nids[0] + rt2 = "--net " + LOCAL_NETS[0] + " --gateway " + gw2_remote_nids[0] + rc = remote.lh.exec_route_cmd(" add "+rt1) + rc = remote.lh.exec_route_cmd(" add "+rt2) + + # commit configuration + #gw1.sln.commit() + #gw1.lh.api_set_routing(True) + #time.sleep(1) + #gw2.sln.commit() + #gw2.lh.api_set_routing(True) + #main.sln.commit() + #remote.sln.commit() + time.sleep(1) + + rc = main.lh.exec_ping(remote_nids[0]) + if not rc: + return lutfrc(LUTF_TEST_FAIL, "ping failed") + + # let main and remote discover each other + try: + if len(main.lh.exec_discover_cmd(remote_nids[0])) == 0: + return lutfrc(LUTF_TEST_FAIL, "unable to discover" , + target=remote_nids[0]) + except Exception as e: + if len(main.lh.exec_discover_cmd(remote_nids[0])) == 0: + return lutfrc(LUTF_TEST_FAIL, "unable to discover" , + target=remote_nids[0]) + try: + if len(remote.lh.exec_discover_cmd(main_nids[0])) == 0: + return lutfrc(LUTF_TEST_FAIL, "unable to discover" , + target=main_nids[0]) + except Exception as e: + if len(remote.lh.exec_discover_cmd(main_nids[0])) == 0: + return lutfrc(LUTF_TEST_FAIL, "unable to discover" , + target=main_nids[0]) + + + rc = main.lh.check_udsp_empty() + if not rc: + return lutfrc(LUTF_TEST_FAIL, "UDSP list not empty") + + for net_num in USE_NET_NUM: + # Add UDSP rule on main node to prioritize a nid on remote node + rc = main.lh.exec_udsp_cmd(" add --dst "+remote_nids[0]) + # Prioritize gw1 for use with the prioritized remote node + rc = main.lh.exec_udsp_cmd(" add --rte "+\ + gw1_local_nids[0]+\ + " --dst "+remote_nids[0]) + + before_stats_gw1 = gw1.lh.get_stats() + #print(before_stats_gw1) + + for i in range(0, PING_TIMES): + rc = main.lh.exec_ping(remote_nids[PING_NID_NUM]) + if not rc: + return lutfrc(LUTF_TEST_FAIL, "ping failed") + + after_stats_gw1 = gw1.lh.get_stats() + #print(after_stats_gw1) + + route_count_before = before_stats_gw1['statistics']['route_count'] + route_count_after = after_stats_gw1['statistics']['route_count'] + + #print(route_count_before, route_count_after) + + # Check stats: + # 1) expect the difference between "before" and "after" route counts to be + # same as or greater than the number of pings issued + if (route_count_after - route_count_before) < PING_TIMES: + return lutfrc(LUTF_TEST_FAIL, "routed count mismatch") + + for i in range(0, MIN_NODES): + if i == MAIN_NODE_ID or i == REMOTE_NODE_ID: + nodes[i].lh.unconfigure_lnet() + + for i in range(0, MIN_NODES): + if i != MAIN_NODE_ID and i != REMOTE_NODE_ID: + nodes[i].lh.unconfigure_lnet() + #for n in nodes: + # n.lh.unconfigure_lnet() + + return lutfrc(LUTF_TEST_PASS) + except Exception as e: + for n in nodes: + n.lh.uninit() + raise e + diff --git a/lustre/tests/lutf/python/tests/suite_udsp/test_udsp_single_net_01.py b/lustre/tests/lutf/python/tests/suite_udsp/test_udsp_single_net_01.py new file mode 100755 index 0000000..3783c56 --- /dev/null +++ b/lustre/tests/lutf/python/tests/suite_udsp/test_udsp_single_net_01.py @@ -0,0 +1,112 @@ +""" +@PRIMARY: N/A +@PRIMARY_DESC: Verify that local nid can be prioritized for sending via UDSP +@SECONDARY: N/A +@DESIGN: N/A +@TESTCASE: +- configure one lnet and more than one nid +- turn on LNet discovery +- run lnetctl discover +- verify that udsp list is empty +- add udsp rule that gives one of the local nids higher priority +- get lnet stats (lnetctl net show -v 4) +- generate traffic by running lnetctl ping multiple times +- get lnet stats again +- verify that all sends were done using the prioritized nid +""" + +import os +import yaml +import lnetconfig +from lutf import agents, me +from lutf_basetest import * +from lnet import TheLNet +from lutf_exception import LUTFError +from lnet_helpers import LNetHelpers +from lustre_node import SimpleLustreNode + +MIN_NODES = 2 +MIN_IFS_PER_NODE = 2 +PING_TIMES = 10 +PING_NID_NUM = 0 +LOCAL_NET = 'tcp' +USE_NID_NUM = 0 + +class TestLustreTraffic: + def __init__(self, target=None): + self.lh = LNetHelpers(os.path.abspath(__file__), target=target) + self.sln = SimpleLustreNode(os.path.abspath(__file__), target=target) + +def run(): + la = agents.keys() + if len(la) < MIN_NODES: + return lutfrc(LUTF_TEST_SKIP, "Not enough agents to run the test") + nodes = [] + try: + for i in range(0, MIN_NODES): + node = TestLustreTraffic(la[i]) + t = node.lh + intfs = t.get_available_devs() + if len(intfs) < MIN_IFS_PER_NODE: + return lutfrc(LUTF_TEST_SKIP, "Not enough interfaces") + if not t.check_udsp_present(): + return lutfrc(LUTF_TEST_SKIP, "UDSP feature is missing") + t.configure_lnet() + t.configure_net(LOCAL_NET, intfs) + t.set_discovery(1) + nodes.append(node) + + + main = nodes[1] + main_nids = main.lh.list_nids() + agent = nodes[0] + agent_nids = agent.lh.list_nids() + + rc = main.lh.check_udsp_empty() + if not rc: + print("UDSP list not empty") + return lutfrc(LUTF_TEST_FAIL) + rc = main.lh.exec_udsp_cmd(" add --src "+main_nids[USE_NID_NUM]) + #rc = main.lh.exec_udsp_cmd(" + + before_stats_main = main.sln.get_lnet().get_net_stats() + print(before_stats_main) + + for i in range(0, PING_TIMES): + rc = main.lh.exec_ping(agent_nids[PING_NID_NUM]) + if not rc: + print("ping failed") + return lutfrc(LUTF_TEST_FAIL) + + after_stats_main = main.sln.get_lnet().get_net_stats() + print(after_stats_main) + + send_count_before = -1 + send_count_after = -1 + for x in before_stats_main: + if (x['net type'] == LOCAL_NET): + for y in x['local NI(s)']: + if y['nid'] == main_nids[USE_NID_NUM]: + send_count_before = y['statistics']['send_count'] + for x in after_stats_main: + if (x['net type'] == LOCAL_NET): + for y in x['local NI(s)']: + if y['nid'] == main_nids[USE_NID_NUM]: + send_count_after = y['statistics']['send_count'] + if send_count_before < 0 or send_count_after < 0: + print("failed to parse net stats") + return lutfrc(LUTF_TEST_FAIL) + + if send_count_after - send_count_before < PING_TIMES: + print("Unexpected number of sends: ", send_count_after - send_count_before) + return lutfrc(LUTF_TEST_FAIL) + + for n in nodes: + n.lh.unconfigure_lnet() + + return lutfrc(LUTF_TEST_PASS) + except Exception as e: + for n in nodes: + n.lh.uninit() + raise e + diff --git a/lustre/tests/lutf/python/tests/suite_udsp/test_udsp_single_net_02.py b/lustre/tests/lutf/python/tests/suite_udsp/test_udsp_single_net_02.py new file mode 100755 index 0000000..b3226fe --- /dev/null +++ b/lustre/tests/lutf/python/tests/suite_udsp/test_udsp_single_net_02.py @@ -0,0 +1,155 @@ +""" +@PRIMARY: N/A +@PRIMARY_DESC: N/A +@PRIMARY_DESC: Verify that multiple local nids can be prioritized for sending via UDSP +@SECONDARY: N/A +@DESIGN: N/A +@TESTCASE: +- configure one lnet and more than two nids +- turn on LNet discovery +- run lnetctl discover +- verify that udsp list is empty +- add udsp rule that gives two of the local nids higher priority +- get lnet stats (lnetctl net show -v 4) +- generate traffic by running lnetctl ping multiple times +- get lnet stats again +- verify that all sends were done using the prioritized nids +""" + +import os +import yaml +import lnetconfig +from lutf import agents, me +from lutf_basetest import * +from lnet import TheLNet +from lutf_exception import LUTFError +from lnet_helpers import LNetHelpers +from lustre_node import SimpleLustreNode + +MIN_NODES = 2 +MIN_IFS_PER_NODE = 3 +PING_TIMES = 10 +PING_NID_NUM = 0 +LOCAL_NET = 'tcp' +USE_NID_NUM = [0, 1] + +class TestLustreTraffic: + def __init__(self, target=None): + self.lh = LNetHelpers(os.path.abspath(__file__), target=target) + self.sln = SimpleLustreNode(os.path.abspath(__file__), target=target) + +def getStatNID(stats_dict, nid_list, nid_num, nid_stat_str): + for x in stats_dict: + if (x['net type'] == LOCAL_NET): + for y in x['local NI(s)']: + if y['nid'] == nid_list[nid_num]: + return y['statistics'][nid_stat_str] + return -1 + + +def run(): + la = agents.keys() + if len(la) < MIN_NODES: + return lutfrc(LUTF_TEST_SKIP, "Not enough agents to run the test") + nodes = [] + try: + for i in range(0, MIN_NODES): + node = TestLustreTraffic(la[i]) + t = node.lh + intfs = t.get_available_devs() + if len(intfs) < MIN_IFS_PER_NODE: + return lutfrc(LUTF_TEST_SKIP, "Not enough interfaces") + if not t.check_udsp_present(): + return lutfrc(LUTF_TEST_SKIP, "UDSP feature is missing") + t.configure_lnet() + t.configure_net(LOCAL_NET, intfs) + t.set_discovery(1) + nodes.append(node) + + main = nodes[1] + main_nids = main.lh.list_nids() + agent = nodes[0] + agent_nids = agent.lh.list_nids() + print("nids: ", main_nids) + + # discover all the peers from main + if len(main.lh.discover(agent_nids[0])) == 0: + return lutfrc(LUTF_TEST_FAIL, "unable to discover" , + target=agent_nids[0]) + + rc = main.lh.check_udsp_empty() + if not rc: + print("UDSP list not empty") + return lutfrc(LUTF_TEST_FAIL) + + for nid_num in USE_NID_NUM: + rc = main.lh.exec_udsp_cmd(" add --src "+main_nids[nid_num]) + + before_stats_main = main.sln.get_lnet().get_net_stats() + print(before_stats_main) + + for i in range(0, PING_TIMES): + rc = main.lh.exec_ping(agent_nids[PING_NID_NUM]) + if not rc: + print("ping failed") + return lutfrc(LUTF_TEST_FAIL) + + after_stats_main = main.sln.get_lnet().get_net_stats() + print(after_stats_main) + + send_count_before = {} + send_count_after = {} + total_send_count_before = 0 + total_send_count_after = 0 + for nid_num in USE_NID_NUM: + #print({nid_num: getStatNID(before_stats_main, main_nids, nid_num, 'send_count')}) + #print({nid_num: getStatNID(after_stats_main, main_nids, nid_num, 'send_count')}) + send_count_before[nid_num] = getStatNID(before_stats_main, main_nids, nid_num, 'send_count') + total_send_count_before += send_count_before[nid_num] + send_count_after[nid_num] = getStatNID(after_stats_main, main_nids, nid_num, 'send_count') + total_send_count_after += send_count_after[nid_num] + + """ + for x in before_stats_main: + if (x['net type'] == LOCAL_NET): + for y in x['local NI(s)']: + if y['nid'] == main_nids[USE_NID_NUM]: + send_count_before = y['statistics']['send_count'] + for x in after_stats_main: + if (x['net type'] == LOCAL_NET): + for y in x['local NI(s)']: + if y['nid'] == main_nids[USE_NID_NUM]: + send_count_after = y['statistics']['send_count'] + if send_count_before < 0 or send_count_after < 0: + print("failed to parse net stats") + return lutfrc(LUTF_TEST_FAIL) + + if send_count_after - send_count_before < PING_TIMES: + print("Unexpected number of sends: ", send_count_after - send_count_before) + return lutfrc(LUTF_TEST_FAIL) + """ + + print(send_count_before, send_count_after) + + # Check stats: + # 1) expect send counts on both interfaces to increase + # 2) expect the total send_count to be no less than the number of pings issued + # 3) expect the send counts on the two interfaces used to be close (diff < 2) + if (total_send_count_after - total_send_count_before) < PING_TIMES: + print("total send count mismatch") + return lutfrc(LUTF_TEST_FAIL) + + if abs(send_count_after[0] - send_count_after[1]) > 1: + print("uneven tx traffic distribution across interfaces") + return lutfrc(LUTF_TEST_FAIL) + + + for n in nodes: + n.lh.unconfigure_lnet() + + return lutfrc(LUTF_TEST_PASS) + except Exception as e: + for n in nodes: + n.lh.uninit() + raise e + diff --git a/lustre/tests/lutf/python/tests/suite_udsp/test_udsp_single_net_03.py b/lustre/tests/lutf/python/tests/suite_udsp/test_udsp_single_net_03.py new file mode 100755 index 0000000..a184e07 --- /dev/null +++ b/lustre/tests/lutf/python/tests/suite_udsp/test_udsp_single_net_03.py @@ -0,0 +1,185 @@ +""" +@PRIMARY: N/A +@PRIMARY_DESC: Verify that local nid can be deprioritized for sending via UDSP +@SECONDARY: N/A +@DESIGN: N/A +@TESTCASE: +- configure one lnet and more than two nids +- turn on LNet discovery +- run lnetctl discover +- verify that udsp list is empty +- add udsp rule that gives two of the local nids higher priority +- get lnet stats (lnetctl net show -v 4) +- generate traffic by running lnetctl ping multiple times +- get lnet stats again +- verify that all sends were done using the prioritized nids +- delete udsp rule for one of the nids +- generate traffic by running lnetctl ping multiple times +- get lnet stats again +- verify that all sends were done using the remaining prioritized nid +""" + +import os +import yaml +import lnetconfig +from lutf import agents, me +from lutf_basetest import * +from lnet import TheLNet +from lutf_exception import LUTFError +from lnet_helpers import LNetHelpers +from lustre_node import SimpleLustreNode + +MIN_NODES = 2 +MIN_IFS_PER_NODE = 3 +PING_TIMES = 10 +PING_NID_NUM = 0 +LOCAL_NET = 'tcp' +NID_NUM_A = 0 +NID_NUM_B = 1 +USE_NID_NUM = [NID_NUM_A, NID_NUM_B] + +class TestLustreTraffic: + def __init__(self, target=None): + self.lh = LNetHelpers(os.path.abspath(__file__), target=target) + self.sln = SimpleLustreNode(os.path.abspath(__file__), target=target) + +def getStatNID(stats_dict, nid_list, nid_num, nid_stat_str): + for x in stats_dict: + if (x['net type'] == LOCAL_NET): + for y in x['local NI(s)']: + if y['nid'] == nid_list[nid_num]: + return y['statistics'][nid_stat_str] + return -1 + + +def run(): + la = agents.keys() + if len(la) < MIN_NODES: + return lutfrc(LUTF_TEST_SKIP, "Not enough agents to run the test") + nodes = [] + try: + for i in range(0, MIN_NODES): + node = TestLustreTraffic(la[i]) + t = node.lh + intfs = t.get_available_devs() + if len(intfs) < MIN_IFS_PER_NODE: + return lutfrc(LUTF_TEST_SKIP, "Not enough interfaces") + if not t.check_udsp_present(): + return lutfrc(LUTF_TEST_SKIP, "UDSP feature is missing") + t.configure_lnet() + t.configure_net(LOCAL_NET, intfs) + t.set_discovery(1) + nodes.append(node) + + main = nodes[1] + main_nids = main.lh.list_nids() + agent = nodes[0] + agent_nids = agent.lh.list_nids() + + # discover all the peers from main + if len(main.lh.discover(agent_nids[0])) == 0: + return lutfrc(LUTF_TEST_FAIL, "unable to discover" , + target=agent_nids[0]) + + rc = main.lh.check_udsp_empty() + if not rc: + print("UDSP list not empty") + return lutfrc(LUTF_TEST_FAIL) + + for nid_num in USE_NID_NUM: + rc = main.lh.exec_udsp_cmd(" add --src "+main_nids[nid_num]) + + before_stats_main = main.sln.get_lnet().get_net_stats() + print(before_stats_main) + + for i in range(0, PING_TIMES): + rc = main.lh.exec_ping(agent_nids[PING_NID_NUM]) + if not rc: + print("ping failed") + return lutfrc(LUTF_TEST_FAIL) + + after_stats_main = main.sln.get_lnet().get_net_stats() + print(after_stats_main) + + send_count_before = {} + send_count_after = {} + total_send_count_before = 0 + total_send_count_after = 0 + for nid_num in USE_NID_NUM: + #print({nid_num: getStatNID(before_stats_main, main_nids, nid_num, 'send_count')}) + #print({nid_num: getStatNID(after_stats_main, main_nids, nid_num, 'send_count')}) + send_count_before[nid_num] = getStatNID(before_stats_main, main_nids, nid_num, 'send_count') + total_send_count_before += send_count_before[nid_num] + send_count_after[nid_num] = getStatNID(after_stats_main, main_nids, nid_num, 'send_count') + total_send_count_after += send_count_after[nid_num] + + print(send_count_before, send_count_after) + + # Check stats: + # 1) expect send counts on both interfaces to increase + # 2) expect the total send_count to be no less than the number of pings issued + # 3) expect the send counts on the two interfaces used to be close (diff < 2) + if (total_send_count_after - total_send_count_before) < PING_TIMES: + print("total send count mismatch") + return lutfrc(LUTF_TEST_FAIL) + + if abs(send_count_after[NID_NUM_A] - send_count_after[NID_NUM_B]) > 1: + print("uneven tx traffic distribution across interfaces") + return lutfrc(LUTF_TEST_FAIL) + + # Delete the UDSP for the first of the nids + rc = main.lh.exec_udsp_cmd(" del --idx 0 ") + + # Generate traffic and collect stats + before_stats_main = main.sln.get_lnet().get_net_stats() + print(before_stats_main) + + for i in range(0, PING_TIMES): + rc = main.lh.exec_ping(agent_nids[PING_NID_NUM]) + if not rc: + print("ping failed") + return lutfrc(LUTF_TEST_FAIL) + + after_stats_main = main.sln.get_lnet().get_net_stats() + print(after_stats_main) + + send_count_before = {} + send_count_after = {} + total_send_count_before = 0 + total_send_count_after = 0 + for nid_num in USE_NID_NUM: + #print({nid_num: getStatNID(before_stats_main, main_nids, nid_num, 'send_count')}) + #print({nid_num: getStatNID(after_stats_main, main_nids, nid_num, 'send_count')}) + send_count_before[nid_num] = getStatNID(before_stats_main, main_nids, nid_num, 'send_count') + total_send_count_before += send_count_before[nid_num] + send_count_after[nid_num] = getStatNID(after_stats_main, main_nids, nid_num, 'send_count') + total_send_count_after += send_count_after[nid_num] + + print(send_count_before, send_count_after) + + # Check stats: + # 1) expect send count on second interface to increase + # 2) expect the total send_count to be no less than the number of pings issued + # 3) expect the send count on the preferred interface to increase by no less than the + # number of pings issued + # 4) expect the send count on the preferred interface to be greater than on the other interface + # by at least the number of pings less one + if (total_send_count_after - total_send_count_before) < PING_TIMES: + print("total send count mismatch") + return lutfrc(LUTF_TEST_FAIL) + + if abs(send_count_after[NID_NUM_B] - send_count_before[NID_NUM_B]) < PING_TIMES: + print("send count increase on preferred interface ", NID_NUM_B, + " insufficient. Expected ", PING_TIMES, " got", + abs(send_count_after[NID_NUM_B] - send_count_before[NID_NUM_B])) + return lutfrc(LUTF_TEST_FAIL) + + for n in nodes: + n.lh.unconfigure_lnet() + + return lutfrc(LUTF_TEST_PASS) + except Exception as e: + for n in nodes: + n.lh.uninit() + raise e + diff --git a/lustre/tests/lutf/python/tests/suite_udsp/test_udsp_single_net_04.py b/lustre/tests/lutf/python/tests/suite_udsp/test_udsp_single_net_04.py new file mode 100755 index 0000000..29e27fc --- /dev/null +++ b/lustre/tests/lutf/python/tests/suite_udsp/test_udsp_single_net_04.py @@ -0,0 +1,185 @@ +""" +@PRIMARY: N/A +@PRIMARY_DESC: Verify that multiple local nids can be deprioritized for sending via UDSP +@SECONDARY: N/A +@DESIGN: N/A +@TESTCASE: +- configure one lnet and more than two nids +- turn on LNet discovery +- run lnetctl discover +- verify that udsp list is empty +- add udsp rule that gives two of the local nids higher priority +- get lnet stats (lnetctl net show -v 4) +- generate traffic by running lnetctl ping multiple times +- get lnet stats again +- verify that all sends were done using the prioritized nids +- delete udsp rules for both prioritized nids +- generate traffic by running lnetctl ping multiple times +- get lnet stats again +- verify that all sends were spread evenly across local nids +""" + +import os +import yaml +import lnetconfig +from lutf import agents, me +from lutf_basetest import * +from lnet import TheLNet +from lutf_exception import LUTFError +from lnet_helpers import LNetHelpers +from lustre_node import SimpleLustreNode + +MIN_NODES = 2 +MIN_IFS_PER_NODE = 3 +PING_TIMES = 10 +PING_NID_NUM = 0 +LOCAL_NET = 'tcp' +NID_NUM_A = 0 +NID_NUM_B = 1 +USE_NID_NUM = [NID_NUM_A, NID_NUM_B] + +class TestLustreTraffic: + def __init__(self, target=None): + self.lh = LNetHelpers(os.path.abspath(__file__), target=target) + self.sln = SimpleLustreNode(os.path.abspath(__file__), target=target) + +def getStatNID(stats_dict, nid_list, nid_num, nid_stat_str): + for x in stats_dict: + if (x['net type'] == LOCAL_NET): + for y in x['local NI(s)']: + if y['nid'] == nid_list[nid_num]: + return y['statistics'][nid_stat_str] + return -1 + + +def run(): + la = agents.keys() + if len(la) < MIN_NODES: + return lutfrc(LUTF_TEST_SKIP, "Not enough agents to run the test") + nodes = [] + try: + for i in range(0, MIN_NODES): + node = TestLustreTraffic(la[i]) + t = node.lh + intfs = t.get_available_devs() + if len(intfs) < MIN_IFS_PER_NODE: + return lutfrc(LUTF_TEST_SKIP, "Not enough interfaces") + if not t.check_udsp_present(): + return lutfrc(LUTF_TEST_SKIP, "UDSP feature is missing") + t.configure_lnet() + t.configure_net(LOCAL_NET, intfs) + t.set_discovery(1) + nodes.append(node) + + main = nodes[1] + main_nids = main.lh.list_nids() + agent = nodes[0] + agent_nids = agent.lh.list_nids() + print("nids: ", main_nids) + + # discover all the peers from main + if len(main.lh.discover(agent_nids[0])) == 0: + return lutfrc(LUTF_TEST_FAIL, "unable to discover" , + target=agent_nids[0]) + + rc = main.lh.check_udsp_empty() + if not rc: + print("UDSP list not empty") + return lutfrc(LUTF_TEST_FAIL) + + for nid_num in USE_NID_NUM: + rc = main.lh.exec_udsp_cmd(" add --src "+main_nids[nid_num]) + + before_stats_main = main.sln.get_lnet().get_net_stats() + print(before_stats_main) + + for i in range(0, PING_TIMES): + rc = main.lh.exec_ping(agent_nids[PING_NID_NUM]) + if not rc: + print("ping failed") + return lutfrc(LUTF_TEST_FAIL) + + after_stats_main = main.sln.get_lnet().get_net_stats() + print(after_stats_main) + + send_count_before = {} + send_count_after = {} + total_send_count_before = 0 + total_send_count_after = 0 + for nid_num in USE_NID_NUM: + #print({nid_num: getStatNID(before_stats_main, main_nids, nid_num, 'send_count')}) + #print({nid_num: getStatNID(after_stats_main, main_nids, nid_num, 'send_count')}) + send_count_before[nid_num] = getStatNID(before_stats_main, main_nids, nid_num, 'send_count') + total_send_count_before += send_count_before[nid_num] + send_count_after[nid_num] = getStatNID(after_stats_main, main_nids, nid_num, 'send_count') + total_send_count_after += send_count_after[nid_num] + + print(send_count_before, send_count_after) + + # Check stats: + # 1) expect send counts on both interfaces to increase + # 2) expect the total send_count to be no less than the number of pings issued + # 3) expect the send counts on the two interfaces used to be close (diff < 2) + if (total_send_count_after - total_send_count_before) < PING_TIMES: + print("total send count mismatch") + return lutfrc(LUTF_TEST_FAIL) + + if abs(send_count_after[NID_NUM_A] - send_count_after[NID_NUM_B]) > 1: + print("uneven tx traffic distribution across interfaces") + return lutfrc(LUTF_TEST_FAIL) + + # Delete the UDSP for both nids + rc = main.lh.exec_udsp_cmd(" del --idx 0 ") + rc = main.lh.exec_udsp_cmd(" del --idx 0 ") + + # Generate traffic and collect stats + before_stats_main = main.sln.get_lnet().get_net_stats() + print(before_stats_main) + + for i in range(0, PING_TIMES): + rc = main.lh.exec_ping(agent_nids[PING_NID_NUM]) + if not rc: + print("ping failed") + return lutfrc(LUTF_TEST_FAIL) + + after_stats_main = main.sln.get_lnet().get_net_stats() + print(after_stats_main) + + send_count_before = {} + send_count_after = {} + total_send_count_before = 0 + total_send_count_after = 0 + nid_num = 0 + for nid in main_nids: + #print({nid_num: getStatNID(before_stats_main, main_nids, nid_num, 'send_count')}) + #print({nid_num: getStatNID(after_stats_main, main_nids, nid_num, 'send_count')}) + send_count_before[nid_num] = getStatNID(before_stats_main, main_nids, nid_num, 'send_count') + total_send_count_before += send_count_before[nid_num] + send_count_after[nid_num] = getStatNID(after_stats_main, main_nids, nid_num, 'send_count') + total_send_count_after += send_count_after[nid_num] + nid_num+=1 + + print(send_count_before, send_count_after) + + # Check stats: + # 1) expect send count on all interfaces to increase + # 2) expect the total send_count to increase by no less than the number of pings issued + # 3) expect the send counts on all interfaces to be close + if (total_send_count_after - total_send_count_before) < PING_TIMES: + print("total send count mismatch") + return lutfrc(LUTF_TEST_FAIL) + + if abs(send_count_after[NID_NUM_A] - send_count_after[NID_NUM_B]) > 1 or \ + abs(send_count_after[NID_NUM_A] - send_count_after[NID_NUM_B+1]) > 1: + print("uneven tx traffic distribution across interfaces") + return lutfrc(LUTF_TEST_FAIL) + + for n in nodes: + n.lh.unconfigure_lnet() + + return lutfrc(LUTF_TEST_PASS) + except Exception as e: + for n in nodes: + n.lh.uninit() + raise e + diff --git a/lustre/tests/lutf/python/tests/suite_udsp/test_udsp_single_net_05.py b/lustre/tests/lutf/python/tests/suite_udsp/test_udsp_single_net_05.py new file mode 100755 index 0000000..31c95a5 --- /dev/null +++ b/lustre/tests/lutf/python/tests/suite_udsp/test_udsp_single_net_05.py @@ -0,0 +1,158 @@ +""" +@PRIMARY: N/A +@PRIMARY_DESC: Verify that destination nid can be prioritized for receiving via UDSP +@SECONDARY: N/A +@DESIGN: N/A +@TESTCASE: +- configure one lnet and more than one nid +- turn on LNet discovery +- run lnetctl discover +- verify that udsp list is empty +- add udsp rule that gives one of the peer nids higher priority +- get lnet stats (lnetctl net show -v 4) +- generate traffic by running lnetctl ping multiple times +- get lnet stats again +- verify that the prioritized nid received all pings +""" + +import os +import yaml +import lnetconfig +from lutf import agents, me +from lutf_basetest import * +from lnet import TheLNet +from lutf_exception import LUTFError +from lnet_helpers import LNetHelpers +from lustre_node import SimpleLustreNode + +MIN_IFS_PER_NODE = 3 +PING_TIMES = 10 +PING_NID_NUM = 0 +LOCAL_NET = 'tcp' +USE_NID_NUM = [0] +MIN_NODES = 2 +LOCAL_DEBUG = 1 + +class TestLustreTraffic: + def __init__(self, target=None): + self.lh = LNetHelpers(os.path.abspath(__file__), target=target) + self.sln = SimpleLustreNode(os.path.abspath(__file__), target=target) + +def getNetStatNID(stats_dict, nid_list, nid_num, nid_stat_str): + for x in stats_dict: + if (x['net type'] == LOCAL_NET): + for y in x['local NI(s)']: + if y['nid'] == nid_list[nid_num]: + return y['statistics'][nid_stat_str] + return -1 + +def udsp_print(*args): + if LOCAL_DEBUG: + print(args) + +def run(): + la = agents.keys() + if len(la) < MIN_NODES: + return lutfrc(LUTF_TEST_SKIP, "Not enough agents to run the test") + nodes = [] + try: + for i in range(0, MIN_NODES): + node = TestLustreTraffic(la[i]) + t = node.lh + intfs = t.get_available_devs() + if len(intfs) < MIN_IFS_PER_NODE: + return lutfrc(LUTF_TEST_SKIP, "Not enough interfaces") + if not t.check_udsp_present(): + return lutfrc(LUTF_TEST_SKIP, "UDSP feature is missing") + t.configure_lnet() + t.configure_net(LOCAL_NET, intfs) + t.set_discovery(1) + nodes.append(node) + + main = nodes[1] + main_nids = main.lh.list_nids() + agent = nodes[0] + agent_nids = agent.lh.list_nids() + + # discover all the peers from main + if len(main.lh.discover(agent_nids[0])) == 0: + return lutfrc(LUTF_TEST_FAIL, "unable to discover" , + target=agent_nids[0]) + + rc = main.lh.check_udsp_empty() + if not rc: + print("UDSP list not empty") + return lutfrc(LUTF_TEST_FAIL) + + for nid_num in USE_NID_NUM: + rc = main.lh.exec_udsp_cmd(" add --dst "+agent_nids[nid_num]) + + before_stats_agent = agent.lh.get_net_stats() + udsp_print(before_stats_agent) + + for i in range(0, PING_TIMES): + rc = main.lh.exec_ping(agent_nids[PING_NID_NUM]) + if not rc: + udsp_print("ping failed") + return lutfrc(LUTF_TEST_FAIL) + + after_stats_agent = agent.lh.get_net_stats() + udsp_print(after_stats_agent) + + recv_count_before = {} + recv_count_after = {} + total_recv_count_before = 0 + total_recv_count_after = 0 + nid_num = 0 + for nid in agent_nids: + #print({nid_num: getStatNID(before_stats_main, main_nids, nid_num, 'send_count')}) + #print({nid_num: getStatNID(after_stats_main, main_nids, nid_num, 'send_count')}) + recv_count_before[nid_num] = getNetStatNID(before_stats_agent, agent_nids, nid_num, 'recv_count') + total_recv_count_before += recv_count_before[nid_num] + recv_count_after[nid_num] = getNetStatNID(after_stats_agent, agent_nids, nid_num, 'recv_count') + total_recv_count_after += recv_count_after[nid_num] + nid_num += 1 + + """ + for x in before_stats_main: + if (x['net type'] == LOCAL_NET): + for y in x['local NI(s)']: + if y['nid'] == main_nids[USE_NID_NUM]: + send_count_before = y['statistics']['send_count'] + for x in after_stats_main: + if (x['net type'] == LOCAL_NET): + for y in x['local NI(s)']: + if y['nid'] == main_nids[USE_NID_NUM]: + send_count_after = y['statistics']['send_count'] + if send_count_before < 0 or send_count_after < 0: + print("failed to parse net stats") + return lutfrc(LUTF_TEST_FAIL) + + if send_count_after - send_count_before < PING_TIMES: + print("Unexpected number of sends: ", send_count_after - send_count_before) + return lutfrc(LUTF_TEST_FAIL) + """ + + udsp_print(recv_count_before, recv_count_after) + + # Check + # 1) expect the total recv_count on the agent to be no less than the number of pings issued + # 2) expect the recv count on the preferred peer interface to increase by the number of pings issues + if (total_recv_count_after - total_recv_count_before) < PING_TIMES: + udsp_print("total recv count mismatch") + return lutfrc(LUTF_TEST_FAIL) + + if abs(recv_count_after[0] - recv_count_before[0]) < PING_TIMES: + udsp_print("less than expected traffic on the prioritized peer nid") + return lutfrc(LUTF_TEST_FAIL) + + + for n in nodes: + n.lh.unconfigure_lnet() + + return lutfrc(LUTF_TEST_PASS) + except Exception as e: + for n in nodes: + n.lh.uninit() + raise e + diff --git a/lustre/tests/lutf/python/tests/suite_udsp/test_udsp_single_net_06.py b/lustre/tests/lutf/python/tests/suite_udsp/test_udsp_single_net_06.py new file mode 100755 index 0000000..a94bf64 --- /dev/null +++ b/lustre/tests/lutf/python/tests/suite_udsp/test_udsp_single_net_06.py @@ -0,0 +1,136 @@ +""" +@PRIMARY: N/A +@PRIMARY_DESC: Verify UDSP nid pair rule in combination with local nid prioritization rule +@SECONDARY: N/A +@DESIGN: N/A +@TESTCASE: +- configure one lnet and more than one nid +- turn on LNet discovery +- run lnetctl discover +- verify that udsp list is empty +- add udsp rule that gives one of the local nids higher priority +- add udsp rule that pairs the prioritized local nid with a certain peer nid +- get lnet stats (lnetctl net show -v 4) +- generate traffic by running lnetctl ping multiple times +- get lnet stats again +- verify that the paired destination nid received all pings +""" + +import os +import yaml +import lnetconfig +from lutf import agents, me +from lutf_basetest import * +from lnet import TheLNet +from lutf_exception import LUTFError +from lnet_helpers import LNetHelpers +from lustre_node import SimpleLustreNode + +MIN_IFS_PER_NODE = 3 +PING_TIMES = 10 +PING_NID_NUM = 0 +LOCAL_NET = 'tcp' +USE_NID_NUM = [0] +MIN_NODES = 2 +LOCAL_DEBUG = 1 + +class TestLustreTraffic: + def __init__(self, target=None): + self.lh = LNetHelpers(os.path.abspath(__file__), target=target) + self.sln = SimpleLustreNode(os.path.abspath(__file__), target=target) + +def getNetStatNID(stats_dict, nid_list, nid_num, nid_stat_str): + for x in stats_dict: + if (x['net type'] == LOCAL_NET): + for y in x['local NI(s)']: + if y['nid'] == nid_list[nid_num]: + return y['statistics'][nid_stat_str] + return -1 + +def udsp_print(*args): + if LOCAL_DEBUG: + print(args) + +def run(): + la = agents.keys() + if len(la) < MIN_NODES: + return lutfrc(LUTF_TEST_SKIP, "Not enough agents to run the test") + nodes = [] + try: + for i in range(0, MIN_NODES): + node = TestLustreTraffic(la[i]) + t = node.lh + intfs = t.get_available_devs() + if len(intfs) < MIN_IFS_PER_NODE: + return lutfrc(LUTF_TEST_SKIP, "Not enough interfaces") + if not t.check_udsp_present(): + return lutfrc(LUTF_TEST_SKIP, "UDSP feature is missing") + t.configure_lnet() + t.configure_net(LOCAL_NET, intfs) + t.set_discovery(1) + nodes.append(node) + + main = nodes[1] + main_nids = main.lh.list_nids() + agent = nodes[0] + agent_nids = agent.lh.list_nids() + + # discover all the peers from main + if len(main.lh.discover(agent_nids[0])) == 0: + return lutfrc(LUTF_TEST_FAIL, "unable to discover" , + target=agent_nids[0]) + + rc = main.lh.check_udsp_empty() + if not rc: + udsp_print("UDSP list not empty") + return lutfrc(LUTF_TEST_FAIL) + + for nid_num in USE_NID_NUM: + rc = main.lh.exec_udsp_cmd(" add --src "+main_nids[nid_num]) + rc = main.lh.exec_udsp_cmd(" add --src "+main_nids[nid_num]+" --dst "+agent_nids[nid_num]) + before_stats_agent = agent.lh.get_net_stats() + udsp_print(before_stats_agent) + + for i in range(0, PING_TIMES): + rc = main.lh.exec_ping(agent_nids[PING_NID_NUM]) + if not rc: + udsp_print("ping failed") + return lutfrc(LUTF_TEST_FAIL) + + after_stats_agent = agent.lh.get_net_stats() + udsp_print(after_stats_agent) + + recv_count_before = {} + recv_count_after = {} + total_recv_count_before = 0 + total_recv_count_after = 0 + nid_num = 0 + for nid in agent_nids: + recv_count_before[nid_num] = getNetStatNID(before_stats_agent, agent_nids, nid_num, 'recv_count') + total_recv_count_before += recv_count_before[nid_num] + recv_count_after[nid_num] = getNetStatNID(after_stats_agent, agent_nids, nid_num, 'recv_count') + total_recv_count_after += recv_count_after[nid_num] + nid_num += 1 + + udsp_print(recv_count_before, recv_count_after) + + # Check + # 1) expect the total recv_count on the agent to be no less than the number of pings issued + # 2) expect the recv count on the preferred peer interface to increase by the number of pings issues + if (total_recv_count_after - total_recv_count_before) < PING_TIMES: + udsp_print("total recv count mismatch") + return lutfrc(LUTF_TEST_FAIL) + + if abs(recv_count_after[0] - recv_count_before[0]) < PING_TIMES: + udsp_print("less than expected traffic on the prioritized peer nid") + return lutfrc(LUTF_TEST_FAIL) + + for n in nodes: + n.lh.unconfigure_lnet() + + return lutfrc(LUTF_TEST_PASS) + except Exception as e: + for n in nodes: + n.lh.uninit() + raise e + diff --git a/lustre/tests/lutf/python/tests/suite_udsp/test_udsp_single_net_07.py b/lustre/tests/lutf/python/tests/suite_udsp/test_udsp_single_net_07.py new file mode 100755 index 0000000..b564979 --- /dev/null +++ b/lustre/tests/lutf/python/tests/suite_udsp/test_udsp_single_net_07.py @@ -0,0 +1,161 @@ +""" +@PRIMARY: N/A +@PRIMARY_DESC: Verify multiple nid pairing UDSP rule +@SECONDARY: N/A +@DESIGN: N/A +@TESTCASE: +- configure one lnet and more than two nids +- turn on LNet discovery +- run lnetctl discover +- verify that udsp list is empty +- add udsp rule that pairs all local nids with one of the peer nids +- get lnet stats (lnetctl net show -v 4) +- generate traffic by running lnetctl ping multiple times +- get lnet stats again +- verify that the paired peer nid received all pings +""" + +import os +import yaml +import lnetconfig +from lutf import agents, me +from lutf_basetest import * +from lnet import TheLNet +from lutf_exception import LUTFError +from lnet_helpers import LNetHelpers +from lustre_node import SimpleLustreNode + +MIN_IFS_PER_NODE = 3 +PING_TIMES = 10 +PING_NID_NUM = 0 +LOCAL_NET = 'tcp' +USE_NID_NUM = [0, 1, 2] +MIN_NODES = 2 +LOCAL_DEBUG = 1 + +class TestLustreTraffic: + def __init__(self, target=None): + self.lh = LNetHelpers(os.path.abspath(__file__), target=target) + self.sln = SimpleLustreNode(os.path.abspath(__file__), target=target) + +def getNetStatNID(stats_dict, nid_list, nid_num, nid_stat_str): + for x in stats_dict: + if (x['net type'] == LOCAL_NET): + for y in x['local NI(s)']: + if y['nid'] == nid_list[nid_num]: + return y['statistics'][nid_stat_str] + return -1 + +def udsp_print(*args): + if LOCAL_DEBUG: + print(args) + +def run(): + la = agents.keys() + if len(la) < MIN_NODES: + return lutfrc(LUTF_TEST_SKIP, "Not enough agents to run the test") + nodes = [] + try: + for i in range(0, MIN_NODES): + node = TestLustreTraffic(la[i]) + t = node.lh + intfs = t.get_available_devs() + if len(intfs) < MIN_IFS_PER_NODE: + return lutfrc(LUTF_TEST_SKIP, "Not enough interfaces") + if not t.check_udsp_present(): + return lutfrc(LUTF_TEST_SKIP, "UDSP feature is missing") + t.configure_lnet() + t.configure_net(LOCAL_NET, intfs) + t.set_discovery(1) + nodes.append(node) + + main = nodes[1] + main_nids = main.lh.list_nids() + agent = nodes[0] + agent_nids = agent.lh.list_nids() + udsp_print("nids: ", main_nids) + + # discover all the peers from main + if len(main.lh.discover(agent_nids[0])) == 0: + return lutfrc(LUTF_TEST_FAIL, "unable to discover" , + target=agent_nids[0]) + + rc = main.lh.check_udsp_empty() + if not rc: + udsp_print("UDSP list not empty") + return lutfrc(LUTF_TEST_FAIL) + + cmd_str = " add --src *.*.*.*@" + LOCAL_NET + " --dst "+agent_nids[PING_NID_NUM] + + rc = main.lh.exec_udsp_cmd(cmd_str) + + rc = main.lh.exec_udsp_cmd(" show") + print(rc) + + before_stats_agent = agent.lh.get_net_stats() + udsp_print(before_stats_agent) + + for i in range(0, PING_TIMES): + rc = main.lh.exec_ping(agent_nids[PING_NID_NUM]) + if not rc: + udsp_print("ping failed") + return lutfrc(LUTF_TEST_FAIL) + + after_stats_agent = agent.lh.get_net_stats() + udsp_print(after_stats_agent) + + recv_count_before = {} + recv_count_after = {} + total_recv_count_before = 0 + total_recv_count_after = 0 + nid_num = 0 + for nid in agent_nids: + recv_count_before[nid_num] = getNetStatNID(before_stats_agent, agent_nids, nid_num, 'recv_count') + total_recv_count_before += recv_count_before[nid_num] + recv_count_after[nid_num] = getNetStatNID(after_stats_agent, agent_nids, nid_num, 'recv_count') + total_recv_count_after += recv_count_after[nid_num] + nid_num += 1 + + """ + for x in before_stats_main: + if (x['net type'] == LOCAL_NET): + for y in x['local NI(s)']: + if y['nid'] == main_nids[USE_NID_NUM]: + send_count_before = y['statistics']['send_count'] + for x in after_stats_main: + if (x['net type'] == LOCAL_NET): + for y in x['local NI(s)']: + if y['nid'] == main_nids[USE_NID_NUM]: + send_count_after = y['statistics']['send_count'] + if send_count_before < 0 or send_count_after < 0: + print("failed to parse net stats") + return lutfrc(LUTF_TEST_FAIL) + + if send_count_after - send_count_before < PING_TIMES: + print("Unexpected number of sends: ", send_count_after - send_count_before) + return lutfrc(LUTF_TEST_FAIL) + """ + + udsp_print(recv_count_before, recv_count_after) + + # Check + # 1) expect the total recv_count on the agent to be no less than the number of pings issued + # 2) expect the recv count on the preferred peer interface to increase by the number of pings issues + if (total_recv_count_after - total_recv_count_before) < PING_TIMES: + udsp_print("total recv count mismatch") + return lutfrc(LUTF_TEST_FAIL) + + if abs(recv_count_after[PING_NID_NUM] - recv_count_before[PING_NID_NUM]) < PING_TIMES: + udsp_print("less than expected traffic on the prioritized peer nid") + return lutfrc(LUTF_TEST_FAIL) + + + for n in nodes: + n.lh.unconfigure_lnet() + + return lutfrc(LUTF_TEST_PASS) + except Exception as e: + for n in nodes: + n.lh.uninit() + raise e + -- 1.8.3.1