Added the UDSP suite and routing suite to the LUTF test cases.
Updated some of the infrastructure scripts with methods needed
for the new test cases.
Test-Parameters: @lnet
Signed-off-by: Serguei Smirnov <ssmirnov@whamcloud.com>
Change-Id: Ibd74cea48982ccafc3b1d5034a409fd2df9e7b1c
Reviewed-on: https://review.whamcloud.com/c/fs/lustre-release/+/39777
Tested-by: jenkins <devops@whamcloud.com>
Tested-by: Maloo <maloo@whamcloud.com>
Reviewed-by: jsimmons <jsimmons@infradead.org>
Reviewed-by: Amir Shehata <ashehata@whamcloud.com>
Reviewed-by: Oleg Drokin <green@whamcloud.com>
nobase_noinst_SCRIPTS += $(wildcard python/tests/suite_samples/*.py)
nobase_noinst_SCRIPTS += $(wildcard python/tests/suite_multi-rail/*.py)
nobase_noinst_SCRIPTS += $(wildcard python/tests/suite_dynamic-discovery/*.py)
+nobase_noinst_SCRIPTS += $(wildcard python/tests/suite_udsp/*.py)
+nobase_noinst_SCRIPTS += $(wildcard python/tests/suite_routing/*.py)
nobase_noinst_SCRIPTS += $(wildcard python/tests/suite_dynamic-nids/*.py)
EXTRA_DIST=$(nobase_noinst_DATA) $(nobase_noinst_SCRIPTS)
--- /dev/null
+from lnet_cleanup import clean_lnet
+
+def lutf_clean_setup():
+ clean_lnet()
--- /dev/null
+"""
+@PRIMARY: N/A
+@PRIMARY_DESC: N/A
+@SECONDARY: N/A
+@DESIGN: N/A
+@TESTCASE: 1. Allocate nodes: PeerA, PeerB and GW1
+2. Configure PeerA with "tcp" net, PeerB with "tcp1" net, GW1 with both
+3. Configure GW1 as a router b/w tcp and tcp1 nets.
+3. Add routes to PeerA and PeerB so they can reach each other.
+4. Execute lnetctl ping several times
+5. Retrieve stats on GW1 and verify that it was routing the pings.
+6. Clean up.
+"""
+
+import os
+import yaml
+import lnetconfig
+import time
+from lutf import agents, me
+from lutf_basetest import *
+from lnet import TheLNet
+from lutf_exception import LUTFError
+from lnet_helpers import LNetHelpers
+from lustre_node import SimpleLustreNode
+from lutf_cmd import lutf_exec_local_cmd
+from utility_paths import LNETCTL
+
+MIN_NODES = 3
+MIN_IFS_PER_NODE = 1
+GW_MIN_IFS = 2
+PING_TIMES = 10
+PING_NID_NUM = 0
+LOCAL_NETS = ['tcp']
+REMOTE_NETS = ['tcp1']
+USE_NET_NUM = [0]
+MAIN_NODE_ID = 0
+REMOTE_NODE_ID = 1
+GW_NODE_IDS = [2]
+
+class TestLustreTraffic:
+ def __init__(self, target=None):
+ self.lh = LNetHelpers(os.path.abspath(__file__), target=target)
+ self.sln = SimpleLustreNode(os.path.abspath(__file__), target=target)
+
+def run():
+ la = agents.keys()
+ if len(la) < MIN_NODES:
+ return lutfrc(LUTF_TEST_SKIP, "Not enough agents to run the test")
+ nodes = []
+ try:
+ # General config for all nodes
+ for i in range(0, MIN_NODES):
+ node = TestLustreTraffic(la[i])
+ t = node.lh
+ if i in GW_NODE_IDS:
+ node_num_ifs = GW_MIN_IFS
+ print("Processing gw node init")
+ else:
+ node_num_ifs = MIN_IFS_PER_NODE
+ intfs = t.get_available_devs()
+ print(i, " intfs: ", intfs)
+ if len(intfs) < node_num_ifs:
+ return lutfrc(LUTF_TEST_FAIL, "Not enough interfaces")
+ print("Configure")
+ t.configure_lnet()
+
+ if i == MAIN_NODE_ID:
+ # Peer A
+ net = LOCAL_NETS[0]
+ net_intfs_list = intfs[0:node_num_ifs]
+ t.configure_net(net, net_intfs_list)
+ elif i == REMOTE_NODE_ID:
+ # Peer B
+ net = REMOTE_NETS[0]
+ net_intfs_list = intfs[0:node_num_ifs]
+ t.configure_net(net, net_intfs_list)
+ else:
+ # GW1
+ net = LOCAL_NETS[0]
+ net_intfs_list = intfs[0:node_num_ifs//2]
+ t.configure_net(net, net_intfs_list)
+ net = REMOTE_NETS[0]
+ net_intfs_list = intfs[node_num_ifs//2:node_num_ifs]
+ t.configure_net(net, net_intfs_list)
+
+ print("Set discovery on")
+ t.set_discovery(1)
+ print("done")
+ nodes.append(node)
+
+ main = nodes[MAIN_NODE_ID]
+ main_nids = main.lh.list_nids()
+ remote = nodes[REMOTE_NODE_ID]
+ remote_nids = remote.lh.list_nids()
+ gw1 = nodes[GW_NODE_IDS[0]]
+ gw1_nids = gw1.lh.list_nids()
+
+ print("main nids: ", main_nids)
+ print("remote nids: ", remote_nids)
+ print("gw1 nids: ", gw1_nids)
+
+ gw1.lh.api_set_routing(True)
+
+ gw1_remote_nids = [x for x in gw1_nids if REMOTE_NETS[0] in x]
+ gw1_local_nids = [x for x in gw1_nids if x not in gw1_remote_nids]
+
+ print("gw1 local/remote nids: ", gw1_local_nids, gw1_remote_nids)
+
+ # discover the gateway from main and remote nodes
+ if len(main.lh.exec_discover_cmd(gw1_local_nids[0])) == 0:
+ return lutfrc(LUTF_TEST_FAIL, "unable to discover" ,
+ target=gw1_local_nids[0])
+ if len(remote.lh.exec_discover_cmd(gw1_remote_nids[0])) == 0:
+ return lutfrc(LUTF_TEST_FAIL, "unable to discover" ,
+ target=gw1_remote_nids[0])
+
+ # setup the routing
+ rt1 = "--net " + REMOTE_NETS[0] + " --gateway " + gw1_local_nids[0]
+ rc = main.lh.exec_route_cmd(" add "+rt1)
+ rt1 = "--net " + LOCAL_NETS[0] + " --gateway " + gw1_remote_nids[0]
+ rc = remote.lh.exec_route_cmd(" add "+rt1)
+
+ time.sleep(1)
+
+ # let main and remote discover each other
+ try:
+ if len(main.lh.exec_discover_cmd(remote_nids[0])) == 0:
+ return lutfrc(LUTF_TEST_FAIL, "unable to discover" ,
+ target=remote_nids[0])
+ except Exception as e:
+ if len(main.lh.exec_discover_cmd(remote_nids[0])) == 0:
+ return lutfrc(LUTF_TEST_FAIL, "unable to discover" ,
+ target=remote_nids[0])
+ try:
+ if len(remote.lh.exec_discover_cmd(main_nids[0])) == 0:
+ return lutfrc(LUTF_TEST_FAIL, "unable to discover" ,
+ target=main_nids[0])
+ except Exception as e:
+ if len(remote.lh.exec_discover_cmd(main_nids[0])) == 0:
+ return lutfrc(LUTF_TEST_FAIL, "unable to discover" ,
+ target=main_nids[0])
+
+ before_stats_gw1 = gw1.lh.get_stats()
+ print(before_stats_gw1)
+
+ for i in range(0, PING_TIMES):
+ rc = main.lh.exec_ping(remote_nids[PING_NID_NUM])
+ if not rc:
+ print("ping failed")
+ return lutfrc(LUTF_TEST_FAIL)
+
+ after_stats_gw1 = gw1.lh.get_stats()
+ print(after_stats_gw1)
+
+ route_count_before = before_stats_gw1['statistics']['route_count']
+ route_count_after = after_stats_gw1['statistics']['route_count']
+
+ print(route_count_before, route_count_after)
+
+ # Check stats:
+ # 1) expect the difference between "before" and "after" route counts to be
+ # same as or greater than the number of pings issued
+ if (route_count_after - route_count_before) < PING_TIMES:
+ print("routed count mismatch")
+ return lutfrc(LUTF_TEST_FAIL)
+
+ for n in nodes:
+ lutf_exec_local_cmd(LNETCTL + " lnet unconfigure")
+
+ return lutfrc(LUTF_TEST_PASS)
+ except Exception as e:
+ for n in nodes:
+ n.lh.uninit()
+ raise e
+
--- /dev/null
+skip_list = []
+expected_failures = ['sample_04', 'sample_04_1']
+
--- /dev/null
+from lnet_cleanup import clean_lnet
+from lustre_cleanup import clean_lustre
+
+def lutf_clean_setup():
+ clean_lustre()
+ clean_lnet()
--- /dev/null
+"""
+@PRIMARY: N/A
+@PRIMARY_DESC: Check that udsp rule table is empty on a newly configured lnet
+@SECONDARY: N/A
+@DESIGN: N/A
+@TESTCASE:
+- check that udsp rule table is empty on a newly configured lnet
+"""
+
+import os
+import yaml
+import lnetconfig
+from lutf import agents, me
+from lutf_basetest import *
+from lnet import TheLNet
+from lutf_exception import LUTFError
+from lnet_helpers import LNetHelpers
+from lutf_cmd import lutf_exec_local_cmd
+from utility_paths import LNETCTL
+
+def check_udsp_empty():
+ rc = lutf_exec_local_cmd(LNETCTL + " udsp show")
+ if 'idx' in rc[0].decode('utf-8'):
+ return False
+ else:
+ return True
+
+def run():
+ la = agents.keys()
+ if len(la) < 1:
+ return lutfrc(LUTF_TEST_SKIP, "No agents to run test")
+ try:
+ t = LNetHelpers(target=la[0])
+ t.configure_lnet()
+ if not t.check_udsp_present():
+ return lutfrc(LUTF_TEST_SKIP, "UDSP feature is missing")
+ rc = check_udsp_empty();
+ if not rc:
+ return lutfrc(LUTF_TEST_FAIL)
+ return lutfrc(LUTF_TEST_PASS)
+ except Exception as e:
+ t.uninit()
+ raise e
+
+
--- /dev/null
+"""
+@PRIMARY: N/A
+@PRIMARY_DESC: Verify that UDSP rule prioritizing a source net can be added
+@SECONDARY: N/A
+@DESIGN: N/A
+@TESTCASE:
+- configure lnet
+- add udsp rule to prioritize a source net
+- verify that the rule has been added
+"""
+
+import os
+import yaml
+import lnetconfig
+from lutf import agents, me
+from lutf_basetest import *
+from lnet import TheLNet
+from lutf_exception import LUTFError
+from lnet_helpers import LNetHelpers
+from lutf_cmd import lutf_exec_local_cmd
+
+
+test_action_list = [{'udsp_cmd': "add --src tcp",
+ 'show_res': {'udsp': [{'idx': 0, 'src': 'tcp', 'dst': 'NA', 'rte': 'NA', 'action': {'priority': 0}}]}}]
+
+def run():
+ la = agents.keys()
+ if len(la) < 1:
+ return lutfrc(LUTF_TEST_SKIP, "No agents to run test")
+ try:
+ t = LNetHelpers(target=la[0])
+ t.configure_lnet()
+ if not t.check_udsp_present():
+ return lutfrc(LUTF_TEST_SKIP, "UDSP feature is missing")
+ rc = t.check_udsp_empty()
+ if not rc:
+ return lutfrc(LUTF_TEST_FAIL)
+ rc = t.exec_udsp_cmd(test_action_list[0]['udsp_cmd'])
+ rc = t.check_udsp_expected(test_action_list[0]['show_res'])
+ if not rc:
+ return lutfrc(LUTF_TEST_FAIL)
+ rc = t.cleanup_udsp()
+ return lutfrc(LUTF_TEST_PASS)
+ except Exception as e:
+ t.uninit()
+ raise e
+
+
--- /dev/null
+"""
+@PRIMARY: N/A
+@PRIMARY_DESC: Verify that UDSP rules prioritizing source net and nid can be added and deleted
+@SECONDARY: N/A
+@DESIGN: N/A
+@TESTCASE:
+- configure lnet
+- add udsp rule to prioritize a source net
+- verify that the rule has been added
+- add udsp rule to prioritize a source nid
+- verify that the rule has been added
+- delete source net udsp rule
+- verify that the rule has been deleted
+"""
+
+import os
+import yaml
+import lnetconfig
+from lutf import agents, me
+from lutf_basetest import *
+from lnet import TheLNet
+from lutf_exception import LUTFError
+from lnet_helpers import LNetHelpers
+from lutf_cmd import lutf_exec_local_cmd
+
+
+test_action_list = [{'udsp_cmd': "add --src tcp1",
+ 'show_res': {'udsp': [{'idx': 0, 'src': 'tcp1', 'dst': 'NA', 'rte': 'NA', 'action': {'priority': 0}}]}},
+ {'udsp_cmd': "add --src 192.168.122.1@tcp1",
+ 'show_res': {'udsp': [{'idx': 0, 'src': 'tcp1', 'dst': 'NA', 'rte': 'NA', 'action': {'priority': 0}},
+ {'idx': 1, 'src': '192.168.122.1@tcp1', 'dst': 'NA', 'rte': 'NA', 'action': {'priority': 0}}]}},
+ {'udsp_cmd': "del --idx=0",
+ 'show_res': {'udsp': [{'idx': 0, 'src': '192.168.122.1@tcp1', 'dst': 'NA', 'rte': 'NA', 'action': {'priority': 0}}]}}]
+
+def run():
+ la = agents.keys()
+ if len(la) < 1:
+ return lutfrc(LUTF_TEST_SKIP, "No agents to run test")
+ try:
+ t = LNetHelpers(target=la[0])
+ t.configure_lnet()
+ if not t.check_udsp_present():
+ return lutfrc(LUTF_TEST_SKIP, "UDSP feature is missing")
+ rc = t.check_udsp_empty()
+ if not rc:
+ return lutfrc(LUTF_TEST_FAIL)
+ for test_action in test_action_list:
+ rc = t.exec_udsp_cmd(test_action['udsp_cmd'])
+ rc = t.check_udsp_expected(test_action['show_res'])
+ if not rc:
+ return lutfrc(LUTF_TEST_FAIL)
+ rc = t.cleanup_udsp()
+ return lutfrc(LUTF_TEST_PASS)
+ except Exception as e:
+ t.uninit()
+ raise e
+
+
--- /dev/null
+"""
+@PRIMARY: N/A
+@PRIMARY_DESC: Verify that UDSP rule prioritizing a destination nid can be added
+@SECONDARY: N/A
+@DESIGN: N/A
+@TESTCASE:
+- configure LNet
+- add udsp rule that prioritizes a destination nid
+- verify that the rule has been added
+"""
+
+import os
+import yaml
+import lnetconfig
+from lutf import agents, me
+from lutf_basetest import *
+from lnet import TheLNet
+from lutf_exception import LUTFError
+from lnet_helpers import LNetHelpers
+from lutf_cmd import lutf_exec_local_cmd
+
+
+test_action_list = [{'udsp_cmd': "add --dst 192.168.122.101@tcp2",
+ 'show_res': {'udsp': [{'idx': 0, 'src': 'NA', 'dst': '192.168.122.101@tcp2', 'rte': 'NA', 'action': {'priority': 0}}]}}]
+
+def run():
+ la = agents.keys()
+ if len(la) < 1:
+ return lutfrc(LUTF_TEST_SKIP, "No agents to run test")
+ try:
+ t = LNetHelpers(target=la[0])
+ t.configure_lnet()
+ if not t.check_udsp_present():
+ return lutfrc(LUTF_TEST_SKIP, "UDSP feature is missing")
+ rc = t.check_udsp_empty()
+ if not rc:
+ return lutfrc(LUTF_TEST_FAIL)
+ rc = t.exec_udsp_cmd(test_action_list[0]['udsp_cmd'])
+ rc = t.check_udsp_expected(test_action_list[0]['show_res'])
+ if not rc:
+ return lutfrc(LUTF_TEST_FAIL)
+ rc = t.cleanup_udsp()
+ return lutfrc(LUTF_TEST_PASS)
+ except Exception as e:
+ t.uninit()
+ raise e
+
+
--- /dev/null
+"""
+@PRIMARY: N/A
+@PRIMARY_DESC: Verify that UDSP rule designating src-dst nid pair can be added
+@SECONDARY: N/A
+@DESIGN: N/A
+@TESTCASE:
+- configure LNet
+- add udsp rule designating a source-destination nid pair
+- verify that the rule has been added
+"""
+
+import os
+import yaml
+import lnetconfig
+from lutf import agents, me
+from lutf_basetest import *
+from lnet import TheLNet
+from lutf_exception import LUTFError
+from lnet_helpers import LNetHelpers
+from lutf_cmd import lutf_exec_local_cmd
+
+
+test_action_list = [{'udsp_cmd': "add --src 192.168.122.1@tcp1 --dst 192.168.122.101@tcp1",
+ 'show_res': {'udsp': [{'idx': 0, 'src': '192.168.122.1@tcp1', 'dst': '192.168.122.101@tcp1', 'rte': 'NA'}]}}]
+
+def run():
+ la = agents.keys()
+ if len(la) < 1:
+ return lutfrc(LUTF_TEST_SKIP, "No agents to run test")
+ try:
+ t = LNetHelpers(target=la[0])
+ t.configure_lnet()
+ if not t.check_udsp_present():
+ return lutfrc(LUTF_TEST_SKIP, "UDSP feature is missing")
+ rc = t.check_udsp_empty()
+ if not rc:
+ return lutfrc(LUTF_TEST_FAIL)
+ rc = t.exec_udsp_cmd(test_action_list[0]['udsp_cmd'])
+ rc = t.check_udsp_expected(test_action_list[0]['show_res'])
+ if not rc:
+ return lutfrc(LUTF_TEST_FAIL)
+ rc = t.cleanup_udsp()
+ return lutfrc(LUTF_TEST_PASS)
+ except Exception as e:
+ t.uninit()
+ raise e
+
+
--- /dev/null
+"""
+@PRIMARY: N/A
+@PRIMARY_DESC: Verify that UDSP rule designating a dst nid and router pair can be added
+@SECONDARY: N/A
+@DESIGN: N/A
+@TESTCASE:
+- configure LNet
+- add udsp rule designating a destination nid and router pair
+- verify that the rule has been added
+"""
+
+import os
+import yaml
+import lnetconfig
+from lutf import agents, me
+from lutf_basetest import *
+from lnet import TheLNet
+from lutf_exception import LUTFError
+from lnet_helpers import LNetHelpers
+from lutf_cmd import lutf_exec_local_cmd
+
+
+test_action_list = [{'udsp_cmd': "add --rte 192.168.122.50@tcp1 --dst 192.168.122.102@tcp2",
+ 'show_res': {'udsp': [{'idx': 0, 'src': 'NA', 'dst': '192.168.122.102@tcp2', 'rte': '192.168.122.50@tcp1'}]}}]
+
+def run():
+ la = agents.keys()
+ if len(la) < 1:
+ return lutfrc(LUTF_TEST_SKIP, "No agents to run test")
+ try:
+ t = LNetHelpers(target=la[0])
+ t.configure_lnet()
+ if not t.check_udsp_present():
+ return lutfrc(LUTF_TEST_SKIP, "UDSP feature is missing")
+ rc = t.check_udsp_empty()
+ if not rc:
+ return lutfrc(LUTF_TEST_FAIL)
+ rc = t.exec_udsp_cmd(test_action_list[0]['udsp_cmd'])
+ rc = t.check_udsp_expected(test_action_list[0]['show_res'])
+ if not rc:
+ return lutfrc(LUTF_TEST_FAIL)
+ rc = t.cleanup_udsp()
+ return lutfrc(LUTF_TEST_PASS)
+ except Exception as e:
+ t.uninit()
+ raise e
+
+
--- /dev/null
+"""
+@PRIMARY: N/A
+@PRIMARY_DESC: Verify that using incorrect idx parameter value with UDSP lnetctl resuts in an error
+@SECONDARY: N/A
+@DESIGN: N/A
+@TESTCASE:
+- configure LNet
+- add udsp rule prioritizing local network
+- attempt to delete the rule providing incorrect idx parameter
+- verify that the rule is not deleted
+"""
+
+import os
+import yaml
+import lnetconfig
+from lutf import agents, me
+from lutf_basetest import *
+from lnet import TheLNet
+from lutf_exception import LUTFError
+from lnet_helpers import LNetHelpers
+from lutf_cmd import lutf_exec_local_cmd
+
+
+test_action_list = [{'udsp_cmd': "add --src tcp1",
+ 'show_res': {'udsp': [{'idx': 0, 'src': 'tcp1', 'dst': 'NA', 'rte': 'NA', 'action': {'priority': 0}}]}},
+ {'udsp_cmd': "del --idx=20",
+ 'show_res': {'udsp': [{'idx': 0, 'src': 'tcp1', 'dst': 'NA', 'rte': 'NA', 'action': {'priority': 0}}]}},
+ {'udsp_cmd': "del --idx=-1",
+ 'show_res': {'udsp': [{'idx': 0, 'src': 'tcp1', 'dst': 'NA', 'rte': 'NA', 'action': {'priority': 0}}]}},
+ {'udsp_cmd': "del --idx=abcd",
+ 'show_res': {'udsp': [{'idx': 0, 'src': 'tcp1', 'dst': 'NA', 'rte': 'NA', 'action': {'priority': 0}}]}}]
+
+def run():
+ la = agents.keys()
+ if len(la) < 1:
+ return lutfrc(LUTF_TEST_SKIP, "No agents to run test")
+ try:
+ t = LNetHelpers(target=la[0])
+ t.configure_lnet()
+ if not t.check_udsp_present():
+ return lutfrc(LUTF_TEST_SKIP, "UDSP feature is missing")
+ rc = t.check_udsp_empty()
+ if not rc:
+ return lutfrc(LUTF_TEST_FAIL)
+ count = 0
+ for test_action in test_action_list:
+ count+=1
+ print("Test action: ", count)
+ rc = t.exec_udsp_cmd(test_action['udsp_cmd'])
+ rc = t.check_udsp_expected(test_action['show_res'])
+ if not rc:
+ return lutfrc(LUTF_TEST_FAIL)
+ rc = t.cleanup_udsp()
+ return lutfrc(LUTF_TEST_PASS)
+ except Exception as e:
+ t.uninit()
+ raise e
+
+
--- /dev/null
+"""
+@PRIMARY: N/A
+@PRIMARY_DESC: Verify that providing incorrect parameters to UDSP lnetctl results in an error
+@SECONDARY: N/A
+@DESIGN: N/A
+@TESTCASE:
+- configure LNet
+- attempt to add udsp rule providing incorrect parameters
+- verify that attempts fail with an error
+"""
+
+import os
+import yaml
+import lnetconfig
+from lutf import agents, me
+from lutf_basetest import *
+from lnet import TheLNet
+from lutf_exception import LUTFError
+from lnet_helpers import LNetHelpers
+from lutf_cmd import lutf_exec_local_cmd
+
+
+test_action_list = [{'udsp_cmd': "add --src abcd",
+ 'fail_res': "ailed to parse src"},
+ {'udsp_cmd': "add --src tcppp",
+ 'fail_res': "ailed to parse src"},
+ {'udsp_cmd': "add --src t",
+ 'fail_res': "ailed to parse src"},
+ {'udsp_cmd': "add --src o2ibbb",
+ 'fail_res': "ailed to parse src"},
+ {'udsp_cmd': "add --src o2i",
+ 'fail_res': "ailed to parse src"},
+ {'udsp_cmd': "add --src 10000000000000000000000000000001",
+ 'fail_res': "ailed to parse src"},
+ {'udsp_cmd': "add --dst abcd",
+ 'fail_res': "ailed to parse dst"},
+ {'udsp_cmd': "add --dst tcppp",
+ 'fail_res': "ailed to parse dst"},
+ {'udsp_cmd': "add --dst t",
+ 'fail_res': "ailed to parse dst"},
+ {'udsp_cmd': "add --dst o2ibbb",
+ 'fail_res': "ailed to parse dst"},
+ {'udsp_cmd': "add --dst o2i",
+ 'fail_res': "ailed to parse dst"},
+ {'udsp_cmd': "add --dst 10000000000000000000000000000001",
+ 'fail_res': "ailed to parse dst"},
+ {'udsp_cmd': "add --rte abcd --dst tcp",
+ 'fail_res': "ailed to parse rte"},
+ {'udsp_cmd': "add --rte tcppp --dst tcp",
+ 'fail_res': "ailed to parse rte"},
+ {'udsp_cmd': "add --rte t --dst tcp",
+ 'fail_res': "ailed to parse rte"},
+ {'udsp_cmd': "add --rte o2ibbb --dst tcp",
+ 'fail_res': "ailed to parse rte"},
+ {'udsp_cmd': "add --rte o2i --dst tcp",
+ 'fail_res': "ailed to parse rte"},
+ {'udsp_cmd': "add --rte 10000000000000000000000000000001 --dst tcp",
+ 'fail_res': "ailed to parse rte"}]
+
+def run():
+ la = agents.keys()
+ if len(la) < 1:
+ return lutfrc(LUTF_TEST_SKIP, "No agents to run test")
+ try:
+ t = LNetHelpers(target=la[0])
+ t.configure_lnet()
+ if not t.check_udsp_present():
+ return lutfrc(LUTF_TEST_SKIP, "UDSP feature is missing")
+ rc = t.check_udsp_empty()
+ if not rc:
+ return lutfrc(LUTF_TEST_FAIL)
+ count = 0
+ for test_action in test_action_list:
+ count+=1
+ print("Test action: ", count)
+ """
+ Expect the command to fail with an exception.
+ Look for the specific error message.
+ """
+ try:
+ rc = t.exec_udsp_cmd(test_action['udsp_cmd'])
+ return lutfrc(LUTF_TEST_FAIL)
+ except Exception as e:
+ if test_action['fail_res'] not in e.msg:
+ return lutfrc(LUTF_TEST_FAIL)
+
+ rc = t.cleanup_udsp()
+ return lutfrc(LUTF_TEST_PASS)
+ except Exception as e:
+ t.uninit()
+ raise e
+
+
--- /dev/null
+"""
+@PRIMARY: N/A
+@PRIMARY_DESC: Verify functionality of UDSP rule that prioritizes a source net
+@SECONDARY: N/A
+@DESIGN: N/A
+@TESTCASE:
+- configure two networks, multiple nids per network
+- add udsp rule that gives one of the networks highest priority
+- generate traffic
+- verify that the network with the highest priority was used
+"""
+
+import os
+import yaml
+import lnetconfig
+from lutf import agents, me
+from lutf_basetest import *
+from lnet import TheLNet
+from lutf_exception import LUTFError
+from lnet_helpers import LNetHelpers
+from lustre_node import SimpleLustreNode
+
+MIN_NODES = 2
+MIN_IFS_PER_NODE = 4
+PING_TIMES = 10
+PING_NID_NUM = 0
+LOCAL_NETS = ['tcp', 'tcp1']
+USE_NET_NUM = [0]
+
+class TestLustreTraffic:
+ def __init__(self, target=None):
+ self.lh = LNetHelpers(os.path.abspath(__file__), target=target)
+ self.sln = SimpleLustreNode(os.path.abspath(__file__), target=target)
+
+def getStatNet(stats_dict, net_list, net_num, nid_stat_str):
+ stat_val = 0
+ for x in stats_dict:
+ if (x['net type'] == net_list[net_num]):
+ for y in x['local NI(s)']:
+ stat_val += y['statistics'][nid_stat_str]
+ return stat_val
+
+
+def run():
+ la = agents.keys()
+ if len(la) < MIN_NODES:
+ return lutfrc(LUTF_TEST_SKIP, "Not enough agents to run the test")
+ nodes = []
+ try:
+ for i in range(0, MIN_NODES):
+ node = TestLustreTraffic(la[i])
+ t = node.lh
+ intfs = t.get_available_devs()
+ if len(intfs) < MIN_IFS_PER_NODE:
+ return lutfrc(LUTF_TEST_SKIP, "Not enough interfaces")
+ t.configure_lnet()
+ if not t.check_udsp_present():
+ return lutfrc(LUTF_TEST_SKIP, "UDSP feature is missing")
+ for j, net in enumerate(LOCAL_NETS):
+ half = len(intfs)//len(LOCAL_NETS)
+ net_intfs_list = intfs[half*(j):half*(j+1)]
+ t.configure_net(net, net_intfs_list)
+ t.set_discovery(1)
+ nodes.append(node)
+
+ main = nodes[1]
+ main_nids = main.lh.list_nids()
+ agent = nodes[0]
+ agent_nids = agent.lh.list_nids()
+
+ # discover all the peers from main
+ if len(main.lh.discover(agent_nids[0])) == 0:
+ return lutfrc(LUTF_TEST_FAIL, "unable to discover" ,
+ target=agent_nids[0])
+
+ rc = main.lh.check_udsp_empty()
+ if not rc:
+ return lutfrc(LUTF_TEST_FAIL, "UDSP list not empty")
+
+ for net_num in USE_NET_NUM:
+ rc = main.lh.exec_udsp_cmd(" add --src "+LOCAL_NETS[net_num])
+
+ before_stats_main = main.lh.get_net_stats()
+ print(before_stats_main)
+
+ for i in range(0, PING_TIMES):
+ rc = main.lh.exec_ping(agent_nids[PING_NID_NUM])
+ if not rc:
+ return lutfrc(LUTF_TEST_FAIL, "ping failed")
+
+ after_stats_main = main.lh.get_net_stats()
+ print(after_stats_main)
+
+ send_count_before = {}
+ send_count_after = {}
+ total_send_count_before = 0
+ total_send_count_after = 0
+ for net_num in USE_NET_NUM:
+ send_count_before[net_num] = getStatNet(before_stats_main, LOCAL_NETS, net_num, 'send_count')
+ total_send_count_before += send_count_before[net_num]
+ send_count_after[net_num] = getStatNet(after_stats_main, LOCAL_NETS, net_num, 'send_count')
+ total_send_count_after += send_count_after[net_num]
+
+ print(send_count_before, send_count_after)
+
+
+ # Check stats:
+ # 1) expect the total send_count to be no less than the number of pings issued
+ # 2) expect the send count on the preferred net to increase by no less than the
+ # number of pings issued
+ if (total_send_count_after - total_send_count_before) < PING_TIMES:
+ return lutfrc(LUTF_TEST_FAIL, "total send count mismatch")
+
+ for net_num in USE_NET_NUM:
+ if abs(send_count_after[net_num] - send_count_before[net_num]) < PING_TIMES:
+ print("send count increase on network ", LOCAL_NETS[net_num],
+ " insufficient. Expected ", PING_TIMES, " got",
+ abs(send_count_after[net_num] - send_count_before[net_num]))
+ return lutfrc(LUTF_TEST_FAIL, "send count increase insufficient")
+
+ for n in nodes:
+ n.lh.unconfigure_lnet()
+
+ return lutfrc(LUTF_TEST_PASS)
+ except Exception as e:
+ for n in nodes:
+ n.lh.uninit()
+ raise e
+
--- /dev/null
+"""
+@PRIMARY: N/A
+@PRIMARY_DESC: Verify that UDSP rule prioritizing a source net works correctly if deleted and re-applied
+@SECONDARY: N/A
+@DESIGN: N/A
+@TESTCASE:
+- configure two networks, multiple nids per network
+- add udsp rule that gives one of the networks highest priority
+- generate traffic
+- verify that the network with the highest priority was used
+- delete the rule and repeat for the second network
+"""
+
+import os
+import yaml
+import lnetconfig
+from lutf import agents, me
+from lutf_basetest import *
+from lnet import TheLNet
+from lutf_exception import LUTFError
+from lnet_helpers import LNetHelpers
+from lustre_node import SimpleLustreNode
+
+MIN_NODES = 2
+MIN_IFS_PER_NODE = 4
+PING_TIMES = 10
+PING_NID_NUM = 0
+LOCAL_NETS = ['tcp', 'tcp1']
+USE_NET_NUM = [0, 1]
+
+class TestLustreTraffic:
+ def __init__(self, target=None):
+ self.lh = LNetHelpers(os.path.abspath(__file__), target=target)
+ self.sln = SimpleLustreNode(os.path.abspath(__file__), target=target)
+
+def getStatNet(stats_dict, net_list, net_num, nid_stat_str):
+ stat_val = 0
+ for x in stats_dict:
+ if (x['net type'] == net_list[net_num]):
+ for y in x['local NI(s)']:
+ stat_val += y['statistics'][nid_stat_str]
+ return stat_val
+
+
+def run():
+ la = agents.keys()
+ if len(la) < MIN_NODES:
+ return lutfrc(LUTF_TEST_SKIP, "Not enough agents to run the test")
+ nodes = []
+ try:
+ for i in range(0, MIN_NODES):
+ node = TestLustreTraffic(la[i])
+ t = node.lh
+ intfs = t.get_available_devs()
+ if len(intfs) < MIN_IFS_PER_NODE:
+ return lutfrc(LUTF_TEST_SKIP, "Not enough interfaces")
+ t.configure_lnet()
+ if not t.check_udsp_present():
+ return lutfrc(LUTF_TEST_SKIP, "UDSP feature is missing")
+ for j, net in enumerate(LOCAL_NETS):
+ half = len(intfs)//len(LOCAL_NETS)
+ net_intfs_list = intfs[half*(j):half*(j+1)]
+ t.configure_net(net, net_intfs_list)
+ t.set_discovery(1)
+ nodes.append(node)
+
+ main = nodes[1]
+ main_nids = main.lh.list_nids()
+ agent = nodes[0]
+ agent_nids = agent.lh.list_nids()
+
+ # discover all the peers from main
+ if len(main.lh.discover(agent_nids[0])) == 0:
+ return lutfrc(LUTF_TEST_FAIL, "unable to discover" ,
+ target=agent_nids[0])
+
+ for prio_net_num in USE_NET_NUM:
+
+ rc = main.lh.check_udsp_empty()
+ if not rc:
+ return lutfrc(LUTF_TEST_FAIL, "UDSP list not empty")
+
+ rc = main.lh.exec_udsp_cmd(" add --src "+LOCAL_NETS[prio_net_num])
+
+ before_stats_main = main.lh.get_net_stats()
+ #print(before_stats_main)
+
+ for i in range(0, PING_TIMES):
+ rc = main.lh.exec_ping(agent_nids[PING_NID_NUM])
+ if not rc:
+ return lutfrc(LUTF_TEST_FAIL, "ping failed")
+
+ after_stats_main = main.lh.get_net_stats()
+ #print(after_stats_main)
+
+ send_count_before = {}
+ send_count_after = {}
+ total_send_count_before = 0
+ total_send_count_after = 0
+
+ send_count_before[prio_net_num] = getStatNet(before_stats_main, LOCAL_NETS, prio_net_num, 'send_count')
+ total_send_count_before += send_count_before[prio_net_num]
+ send_count_after[prio_net_num] = getStatNet(after_stats_main, LOCAL_NETS, prio_net_num, 'send_count')
+ total_send_count_after += send_count_after[prio_net_num]
+
+ #print(send_count_before, send_count_after)
+
+ # Check stats:
+ # 1) expect the total send_count to be no less than the number of pings issued
+ # 2) expect the send count on the preferred net to increase by no less than the
+ # number of pings issued
+ if (total_send_count_after - total_send_count_before) < PING_TIMES:
+ return lutfrc(LUTF_TEST_FAIL, "total send count mismatch")
+
+ if abs(send_count_after[prio_net_num] - send_count_before[prio_net_num]) < PING_TIMES:
+ print("send count increase on network ", LOCAL_NETS[prio_net_num],
+ " insufficient. Expected ", PING_TIMES, " got",
+ abs(send_count_after[prio_net_num] - send_count_before[prio_net_num]))
+ return lutfrc(LUTF_TEST_FAIL, "send count increase insufficient")
+
+ rc = main.lh.exec_udsp_cmd(" del --idx 0")
+ for n in nodes:
+ n.lh.unconfigure_lnet()
+
+ return lutfrc(LUTF_TEST_PASS)
+ except Exception as e:
+ for n in nodes:
+ n.lh.uninit()
+ raise e
+
--- /dev/null
+"""
+@PRIMARY: N/A
+@PRIMARY_DESC: Verify that UDSP rules prioritizing source net and source nid work correctly in combination
+@SECONDARY: N/A
+@DESIGN: N/A
+@TESTCASE:
+- configure two networks, multiple nids per network
+- add udsp rule that gives one of the networks highest priority
+- add udsp rule that gives the highest priority to one of the nids on the prioritized network
+- generate traffic
+- verify that the prioritized nid on the network with the highest priority was used
+"""
+
+import os
+import yaml
+import lnetconfig
+from lutf import agents, me
+from lutf_basetest import *
+from lnet import TheLNet
+from lutf_exception import LUTFError
+from lnet_helpers import LNetHelpers
+from lustre_node import SimpleLustreNode
+
+MIN_NODES = 2
+MIN_IFS_PER_NODE = 4
+PING_TIMES = 10
+PING_NID_NUM = 0
+LOCAL_NETS = ['tcp', 'tcp1']
+USE_NET_NUM = [0]
+USE_NID_NUM = 0
+
+class TestLustreTraffic:
+ def __init__(self, target=None):
+ self.lh = LNetHelpers(os.path.abspath(__file__), target=target)
+ self.sln = SimpleLustreNode(os.path.abspath(__file__), target=target)
+
+
+def getStatNIDStr(stats_dict, nid_str, nid_stat_str):
+ for x in stats_dict:
+ for y in x['local NI(s)']:
+ if y['nid'] == nid_str:
+ return y['statistics'][nid_stat_str]
+ return -1
+
+def getStatNet(stats_dict, net_list, net_num, nid_stat_str):
+ stat_val = 0
+ for x in stats_dict:
+ if (x['net type'] == net_list[net_num]):
+ for y in x['local NI(s)']:
+ stat_val += y['statistics'][nid_stat_str]
+ return stat_val
+
+def getNetNIDbyIdx(stats_dict, net_name, nid_idx):
+ nid_str = ""
+ for x in stats_dict:
+ if (x['net type'] == net_name):
+ nid_str = x['local NI(s)'][nid_idx]['nid']
+ break
+ return nid_str
+
+def run():
+ la = agents.keys()
+ if len(la) < MIN_NODES:
+ return lutfrc(LUTF_TEST_SKIP, "Not enough agents to run the test")
+ nodes = []
+ try:
+ for i in range(0, MIN_NODES):
+ node = TestLustreTraffic(la[i])
+ t = node.lh
+ intfs = t.get_available_devs()
+ if len(intfs) < MIN_IFS_PER_NODE:
+ return lutfrc(LUTF_TEST_SKIP, "Not enough interfaces")
+ t.configure_lnet()
+ if not t.check_udsp_present():
+ return lutfrc(LUTF_TEST_SKIP, "UDSP feature is missing")
+ for j, net in enumerate(LOCAL_NETS):
+ half = len(intfs)//len(LOCAL_NETS)
+ net_intfs_list = intfs[half*(j):half*(j+1)]
+ t.configure_net(net, net_intfs_list)
+ t.set_discovery(1)
+ nodes.append(node)
+
+ main = nodes[1]
+ main_nids = main.lh.list_nids()
+ agent = nodes[0]
+ agent_nids = agent.lh.list_nids()
+
+ # discover all the peers from main
+ if len(main.lh.discover(agent_nids[0])) == 0:
+ return lutfrc(LUTF_TEST_FAIL, "unable to discover" ,
+ target=agent_nids[0])
+
+ rc = main.lh.check_udsp_empty()
+ if not rc:
+ return lutfrc(LUTF_TEST_FAIL, "UDSP list not empty")
+
+ for net_num in USE_NET_NUM:
+ rc = main.lh.exec_udsp_cmd(" add --src "+LOCAL_NETS[net_num])
+ before_stats_main = main.lh.get_net_stats()
+ nid_str = getNetNIDbyIdx(before_stats_main, LOCAL_NETS[net_num], USE_NID_NUM)
+ rc = main.lh.exec_udsp_cmd(" add --src "+nid_str)
+
+ before_stats_main = main.lh.get_net_stats()
+ #print(before_stats_main)
+
+ for i in range(0, PING_TIMES):
+ rc = main.lh.exec_ping(agent_nids[PING_NID_NUM])
+ if not rc:
+ return lutfrc(LUTF_TEST_FAIL, "ping failed")
+
+ after_stats_main = main.lh.get_net_stats()
+ #print(after_stats_main)
+
+ send_count_before = {}
+ send_count_after = {}
+ total_send_count_before = 0
+ total_send_count_after = 0
+ send_count_nid_before = 0
+ send_count_nid_after = 0
+ for net_num in USE_NET_NUM:
+ send_count_before[net_num] = getStatNet(before_stats_main, LOCAL_NETS, net_num, 'send_count')
+ total_send_count_before += send_count_before[net_num]
+ send_count_after[net_num] = getStatNet(after_stats_main, LOCAL_NETS, net_num, 'send_count')
+ total_send_count_after += send_count_after[net_num]
+
+ send_count_nid_before = getStatNIDStr(before_stats_main, nid_str, 'send_count')
+ send_count_nid_after = getStatNIDStr(after_stats_main, nid_str, 'send_count')
+ #print(send_count_before, send_count_after)
+ #print(send_count_nid_before, send_count_nid_after)
+
+ # Check stats:
+ # 1) expect the total send_count to be no less than the number of pings issued
+ # 2) expect the send count on the preferred net to increase by no less than the
+ # number of pings issued
+ # 3) expect the preferred nid on the preferred net to be used
+ if (total_send_count_after - total_send_count_before) < PING_TIMES:
+ return lutfrc(LUTF_TEST_FAIL, "total send count mismatch")
+
+ for net_num in USE_NET_NUM:
+ if abs(send_count_after[net_num] - send_count_before[net_num]) < PING_TIMES:
+ print("send count increase on network ", LOCAL_NETS[net_num],
+ " insufficient. Expected ", PING_TIMES, " got ",
+ abs(send_count_after[net_num] - send_count_before[net_num]))
+ return lutfrc(LUTF_TEST_FAIL, "send count increase insufficient")
+
+ if (send_count_nid_after - send_count_nid_before) < PING_TIMES:
+ print("send count increase on the preferred nid insufficient. Expected ",
+ PING_TIMES, "got ", (send_count_nid_after - send_count_nid_before))
+ return lutfrc(LUTF_TEST_FAIL, "send count increase on the preferred nid insufficient")
+
+
+ for n in nodes:
+ n.lh.unconfigure_lnet()
+
+ return lutfrc(LUTF_TEST_PASS)
+ except Exception as e:
+ for n in nodes:
+ n.lh.uninit()
+ raise e
+
--- /dev/null
+"""
+@PRIMARY: N/A
+@PRIMARY_DESC: Verify that UDSP rules prioritizing source net and source nid work correctly in combination
+@SECONDARY: N/A
+@DESIGN: N/A
+@TESTCASE:
+- configure two networks, multiple nids per network
+- add udsp rule that gives one of the networks highest priority
+- add udsp rule that gives the highest priority to one of the nids on the lower priority network
+- generate traffic
+- verify that the nids on the network with the highest priority were used
+"""
+
+import os
+import yaml
+import lnetconfig
+from lutf import agents, me
+from lutf_basetest import *
+from lnet import TheLNet
+from lutf_exception import LUTFError
+from lnet_helpers import LNetHelpers
+from lustre_node import SimpleLustreNode
+
+MIN_NODES = 2
+MIN_IFS_PER_NODE = 4
+PING_TIMES = 10
+PING_NID_NUM = 0
+LOCAL_NETS = ['tcp', 'tcp1']
+USE_NET_NUM = [0]
+USE_NID_NUM = 0
+
+class TestLustreTraffic:
+ def __init__(self, target=None):
+ self.lh = LNetHelpers(os.path.abspath(__file__), target=target)
+ self.sln = SimpleLustreNode(os.path.abspath(__file__), target=target)
+
+
+def getStatNIDStr(stats_dict, nid_str, nid_stat_str):
+ for x in stats_dict:
+ for y in x['local NI(s)']:
+ if y['nid'] == nid_str:
+ return y['statistics'][nid_stat_str]
+ return -1
+
+def getStatNet(stats_dict, net_list, net_num, nid_stat_str):
+ stat_val = 0
+ for x in stats_dict:
+ if (x['net type'] == net_list[net_num]):
+ for y in x['local NI(s)']:
+ stat_val += y['statistics'][nid_stat_str]
+ return stat_val
+
+def getNetNIDbyIdx(stats_dict, net_name, nid_idx):
+ nid_str = ""
+ for x in stats_dict:
+ if (x['net type'] == net_name):
+ nid_str = x['local NI(s)'][nid_idx]['nid']
+ break
+ return nid_str
+
+def run():
+ la = agents.keys()
+ if len(la) < MIN_NODES:
+ return lutfrc(LUTF_TEST_SKIP, "Not enough agents to run the test")
+ nodes = []
+ try:
+ for i in range(0, MIN_NODES):
+ node = TestLustreTraffic(la[i])
+ t = node.lh
+ intfs = t.get_available_devs()
+ if len(intfs) < MIN_IFS_PER_NODE:
+ return lutfrc(LUTF_TEST_SKIP, "Not enough interfaces")
+ t.configure_lnet()
+ if not t.check_udsp_present():
+ return lutfrc(LUTF_TEST_SKIP, "UDSP feature is missing")
+ for j, net in enumerate(LOCAL_NETS):
+ half = len(intfs)//len(LOCAL_NETS)
+ net_intfs_list = intfs[half*(j):half*(j+1)]
+ t.configure_net(net, net_intfs_list)
+ t.set_discovery(1)
+ nodes.append(node)
+
+ main = nodes[1]
+ main_nids = main.lh.list_nids()
+ agent = nodes[0]
+ agent_nids = agent.lh.list_nids()
+
+ # discover all the peers from main
+ if len(main.lh.discover(agent_nids[0])) == 0:
+ return lutfrc(LUTF_TEST_FAIL, "unable to discover" ,
+ target=agent_nids[0])
+
+ rc = main.lh.check_udsp_empty()
+ if not rc:
+ return lutfrc(LUTF_TEST_FAIL)
+
+ for net_num in USE_NET_NUM:
+ rc = main.lh.exec_udsp_cmd(" add --src "+LOCAL_NETS[net_num])
+ before_stats_main = main.lh.get_net_stats()
+ nid_str = getNetNIDbyIdx(before_stats_main, LOCAL_NETS[(net_num+1) % (len(LOCAL_NETS))], USE_NID_NUM)
+ rc = main.lh.exec_udsp_cmd(" add --src "+nid_str)
+
+ before_stats_main = main.lh.get_net_stats()
+ #print(before_stats_main)
+
+ for i in range(0, PING_TIMES):
+ rc = main.lh.exec_ping(agent_nids[PING_NID_NUM])
+ if not rc:
+ return lutfrc(LUTF_TEST_FAILi, "ping failed")
+
+ after_stats_main = main.lh.get_net_stats()
+ #print(after_stats_main)
+
+ send_count_before = {}
+ send_count_after = {}
+ total_send_count_before = 0
+ total_send_count_after = 0
+ send_count_nid_before = 0
+ send_count_nid_after = 0
+ for net_num in USE_NET_NUM:
+ send_count_before[net_num] = getStatNet(before_stats_main, LOCAL_NETS, net_num, 'send_count')
+ total_send_count_before += send_count_before[net_num]
+ send_count_after[net_num] = getStatNet(after_stats_main, LOCAL_NETS, net_num, 'send_count')
+ total_send_count_after += send_count_after[net_num]
+
+ send_count_nid_before = getStatNIDStr(before_stats_main, nid_str, 'send_count')
+ send_count_nid_after = getStatNIDStr(after_stats_main, nid_str, 'send_count')
+ #print(send_count_before, send_count_after)
+ #print(send_count_nid_before, send_count_nid_after)
+
+ # Check stats:
+ # 1) expect the total send_count to be no less than the number of pings issued
+ # 2) expect the send count on the preferred net to increase by no less than the
+ # number of pings issued
+ # 3) expect the preferred nid on the non-preferred net not to be used
+ if (total_send_count_after - total_send_count_before) < PING_TIMES:
+ return lutfrc(LUTF_TEST_FAIL, "total send count mismatch")
+
+ for net_num in USE_NET_NUM:
+ if abs(send_count_after[net_num] - send_count_before[net_num]) < PING_TIMES:
+ print("send count increase on network ", LOCAL_NETS[net_num],
+ " insufficient. Expected ", PING_TIMES, " got ",
+ abs(send_count_after[net_num] - send_count_before[net_num]))
+ return lutfrc(LUTF_TEST_FAIL, "send count increase insufficient")
+
+ if (send_count_nid_after - send_count_nid_before) >= PING_TIMES//2:
+ print("send count increase on the preferred nid is unexpected. Expected none, ",
+ "got ", (send_count_nid_after - send_count_nid_before))
+ return lutfrc(LUTF_TEST_FAIL, "send count increase on the preferred nid is unexpected.")
+
+
+ for n in nodes:
+ n.lh.unconfigure_lnet()
+
+ return lutfrc(LUTF_TEST_PASS)
+ except Exception as e:
+ for n in nodes:
+ n.lh.uninit()
+ raise e
+
--- /dev/null
+"""
+@PRIMARY: N/A
+@PRIMARY_DESC: Verify that UDSP rules prioritizing source net on two peers work correctly in combination
+@SECONDARY: N/A
+@DESIGN: N/A
+@TESTCASE:
+- configure two networks, multiple nids per network
+- add udsp rule that gives one of the networks highest priority on the local node
+- add udsp rule that gives priority to the other network on the remote node
+- generate traffic
+- verify that the network with the highest priority was used
+"""
+
+import os
+import yaml
+import lnetconfig
+from lutf import agents, me
+from lutf_basetest import *
+from lnet import TheLNet
+from lutf_exception import LUTFError
+from lnet_helpers import LNetHelpers
+from lustre_node import SimpleLustreNode
+
+MIN_NODES = 2
+MIN_IFS_PER_NODE = 4
+PING_TIMES = 10
+PING_NID_NUM = 0
+LOCAL_NETS = ['tcp', 'tcp1']
+USE_NET_NUM = [0]
+
+class TestLustreTraffic:
+ def __init__(self, target=None):
+ self.lh = LNetHelpers(os.path.abspath(__file__), target=target)
+ self.sln = SimpleLustreNode(os.path.abspath(__file__), target=target)
+
+
+def getStatNIDStr(stats_dict, nid_str, nid_stat_str):
+ for x in stats_dict:
+ for y in x['local NI(s)']:
+ if y['nid'] == nid_str:
+ return y['statistics'][nid_stat_str]
+ return -1
+
+def getStatNet(stats_dict, net_list, net_num, nid_stat_str):
+ stat_val = 0
+ for x in stats_dict:
+ if (x['net type'] == net_list[net_num]):
+ for y in x['local NI(s)']:
+ stat_val += y['statistics'][nid_stat_str]
+ return stat_val
+
+def getNetNIDbyIdx(stats_dict, net_name, nid_idx):
+ nid_str = ""
+ for x in stats_dict:
+ if (x['net type'] == net_name):
+ nid_str = x['local NI(s)'][nid_idx]['nid']
+ break
+ return nid_str
+
+def run():
+ la = agents.keys()
+ if len(la) < MIN_NODES:
+ return lutfrc(LUTF_TEST_SKIP, "Not enough agents to run the test")
+ nodes = []
+ try:
+ for i in range(0, MIN_NODES):
+ node = TestLustreTraffic(la[i])
+ t = node.lh
+ intfs = t.get_available_devs()
+ if len(intfs) < MIN_IFS_PER_NODE:
+ return lutfrc(LUTF_TEST_SKIP, "Not enough interfaces")
+ t.configure_lnet()
+ if not t.check_udsp_present():
+ return lutfrc(LUTF_TEST_SKIP, "UDSP feature is missing")
+ for j, net in enumerate(LOCAL_NETS):
+ half = len(intfs)//len(LOCAL_NETS)
+ net_intfs_list = intfs[half*(j):half*(j+1)]
+ t.configure_net(net, net_intfs_list)
+ t.set_discovery(1)
+ nodes.append(node)
+
+ main = nodes[1]
+ main_nids = main.lh.list_nids()
+ agent = nodes[0]
+ agent_nids = agent.lh.list_nids()
+
+ # discover all the peers from main
+ if len(main.lh.discover(agent_nids[0])) == 0:
+ return lutfrc(LUTF_TEST_FAIL, "unable to discover" ,
+ target=agent_nids[0])
+
+ rc = main.lh.check_udsp_empty()
+ if not rc:
+ return lutfrc(LUTF_TEST_FAIL, "UDSP list not empty")
+
+ for net_num in USE_NET_NUM:
+ rc = main.lh.exec_udsp_cmd(" add --src "+LOCAL_NETS[net_num])
+ # Prioritize the other net on the agent
+ rc = agent.lh.exec_udsp_cmd(" add --src "+LOCAL_NETS[(net_num+1) % (len(LOCAL_NETS))])
+
+ before_stats_main = main.lh.get_net_stats()
+ #print(before_stats_main)
+
+ for i in range(0, PING_TIMES):
+ rc = main.lh.exec_ping(agent_nids[PING_NID_NUM])
+ if not rc:
+ return lutfrc(LUTF_TEST_FAIL, "ping failed")
+
+ after_stats_main = main.lh.get_net_stats()
+ #print(after_stats_main)
+
+ send_count_before = {}
+ send_count_after = {}
+ total_send_count_before = 0
+ total_send_count_after = 0
+ send_count_nid_before = 0
+ send_count_nid_after = 0
+ for net_num in USE_NET_NUM:
+ send_count_before[net_num] = getStatNet(before_stats_main, LOCAL_NETS, net_num, 'send_count')
+ total_send_count_before += send_count_before[net_num]
+ send_count_after[net_num] = getStatNet(after_stats_main, LOCAL_NETS, net_num, 'send_count')
+ total_send_count_after += send_count_after[net_num]
+
+ #print(send_count_before, send_count_after)
+
+ # Check stats:
+ # 1) expect the total send_count to be no less than the number of pings issued
+ # 2) expect the send count on the preferred net to increase by no less than the
+ # number of pings issued
+ if (total_send_count_after - total_send_count_before) < PING_TIMES:
+ return lutfrc(LUTF_TEST_FAIL, "total send count mismatch")
+
+ for net_num in USE_NET_NUM:
+ if abs(send_count_after[net_num] - send_count_before[net_num]) < PING_TIMES:
+ print("send count increase on network ", LOCAL_NETS[net_num],
+ " insufficient. Expected ", PING_TIMES, " got ",
+ abs(send_count_after[net_num] - send_count_before[net_num]))
+ return lutfrc(LUTF_TEST_FAIL, "send count increase insufficient.")
+
+ for n in nodes:
+ n.lh.unconfigure_lnet()
+
+ return lutfrc(LUTF_TEST_PASS)
+ except Exception as e:
+ for n in nodes:
+ n.lh.uninit()
+ raise e
+
--- /dev/null
+"""
+@PRIMARY: N/A
+@PRIMARY_DESC: Verify that adding a UDSP rule that prioritizes an unexisting network doesn't affect traffic distribution
+@SECONDARY: N/A
+@DESIGN: N/A
+@TESTCASE:
+- configure two networks, multiple nids per network
+- add udsp rule that gives highest priority to an unexisting network
+- generate traffic
+- verify that traffic is evenly distributed across both networks
+"""
+
+import os
+import yaml
+import lnetconfig
+from lutf import agents, me
+from lutf_basetest import *
+from lnet import TheLNet
+from lutf_exception import LUTFError
+from lnet_helpers import LNetHelpers
+from lustre_node import SimpleLustreNode
+
+MIN_NODES = 2
+MIN_IFS_PER_NODE = 4
+PING_TIMES = 10
+PING_NID_NUM = 0
+LOCAL_NETS = ['tcp', 'tcp1']
+USE_NET_NUM = [0, 1]
+UNEX_NET = "tcp2"
+EXP_UDSP_SHOW = {'udsp': [{'idx': 0, 'src': 'tcp2', 'dst': 'NA', 'rte': 'NA', 'action': {'priority': 0}}]}
+
+class TestLustreTraffic:
+ def __init__(self, target=None):
+ self.lh = LNetHelpers(os.path.abspath(__file__), target=target)
+ self.sln = SimpleLustreNode(os.path.abspath(__file__), target=target)
+
+
+def getStatNIDStr(stats_dict, nid_str, nid_stat_str):
+ for x in stats_dict:
+ for y in x['local NI(s)']:
+ if y['nid'] == nid_str:
+ return y['statistics'][nid_stat_str]
+ return -1
+
+def getStatNet(stats_dict, net_list, net_num, nid_stat_str):
+ stat_val = 0
+ for x in stats_dict:
+ if (x['net type'] == net_list[net_num]):
+ for y in x['local NI(s)']:
+ stat_val += y['statistics'][nid_stat_str]
+ return stat_val
+
+def getNetNIDbyIdx(stats_dict, net_name, nid_idx):
+ nid_str = ""
+ for x in stats_dict:
+ if (x['net type'] == net_name):
+ nid_str = x['local NI(s)'][nid_idx]['nid']
+ break
+ return nid_str
+
+def run():
+ la = agents.keys()
+ if len(la) < MIN_NODES:
+ return lutfrc(LUTF_TEST_SKIP, "Not enough agents to run the test")
+ nodes = []
+ try:
+ for i in range(0, MIN_NODES):
+ node = TestLustreTraffic(la[i])
+ t = node.lh
+ intfs = t.get_available_devs()
+ if len(intfs) < MIN_IFS_PER_NODE:
+ return lutfrc(LUTF_TEST_SKIP, "Not enough interfaces")
+ t.configure_lnet()
+ if not t.check_udsp_present():
+ return lutfrc(LUTF_TEST_SKIP, "UDSP feature is missing")
+ for j, net in enumerate(LOCAL_NETS):
+ half = len(intfs)//len(LOCAL_NETS)
+ net_intfs_list = intfs[half*(j):half*(j+1)]
+ t.configure_net(net, net_intfs_list)
+ t.set_discovery(1)
+ nodes.append(node)
+
+ main = nodes[1]
+ main_nids = main.lh.list_nids()
+ agent = nodes[0]
+ agent_nids = agent.lh.list_nids()
+
+ # discover all the peers from main
+ if len(main.lh.discover(agent_nids[0])) == 0:
+ return lutfrc(LUTF_TEST_FAIL, "unable to discover" ,
+ target=agent_nids[0])
+
+ rc = main.lh.check_udsp_empty()
+ if not rc:
+ return lutfrc(LUTF_TEST_FAIL, "UDSP list not empty")
+
+ # Prioritize unexisting net
+ rc = main.lh.exec_udsp_cmd(" add --src "+UNEX_NET)
+ # Verify the rule got added
+ rc = main.lh.check_udsp_expected(EXP_UDSP_SHOW)
+ if not rc:
+ return lutfrc(LUTF_TEST_FAIL)
+
+ before_stats_main = main.lh.get_net_stats()
+ #print(before_stats_main)
+
+ for i in range(0, PING_TIMES):
+ rc = main.lh.exec_ping(agent_nids[PING_NID_NUM])
+ if not rc:
+ return lutfrc(LUTF_TEST_FAIL, "ping failed")
+
+ after_stats_main = main.lh.get_net_stats()
+ #print(after_stats_main)
+
+ send_count_before = {}
+ send_count_after = {}
+ total_send_count_before = 0
+ total_send_count_after = 0
+ for net_num in USE_NET_NUM:
+ send_count_before[net_num] = getStatNet(before_stats_main, LOCAL_NETS, net_num, 'send_count')
+ total_send_count_before += send_count_before[net_num]
+ send_count_after[net_num] = getStatNet(after_stats_main, LOCAL_NETS, net_num, 'send_count')
+ total_send_count_after += send_count_after[net_num]
+
+ #print(send_count_before, send_count_after)
+
+ # Check stats:
+ # 1) expect the total send_count to be no less than the number of pings issued
+ # 2) expect the send count to be even across nets (no preference)
+ if (total_send_count_after - total_send_count_before) < PING_TIMES:
+ return lutfrc(LUTF_TEST_FAIL, "total send count mismatch")
+
+ for net_num in USE_NET_NUM:
+ if abs(send_count_after[net_num] - send_count_before[net_num]) > PING_TIMES//2:
+ print("send count increase on network ", LOCAL_NETS[net_num],
+ " excessive. Expected no more than", PING_TIMES//2, " got ",
+ abs(send_count_after[net_num] - send_count_before[net_num]))
+ return lutfrc(LUTF_TEST_FAIL, "send count increase excessive.")
+
+ for n in nodes:
+ n.lh.unconfigure_lnet()
+
+ return lutfrc(LUTF_TEST_PASS)
+ except Exception as e:
+ for n in nodes:
+ n.lh.uninit()
+ raise e
+
--- /dev/null
+"""
+@PRIMARY: N/A
+@PRIMARY_DESC: Verify that deleting prioritized local net results no issues
+@SECONDARY: N/A
+@DESIGN: N/A
+@TESTCASE:
+- configure two networks, multiple nids per network
+- add udsp rule that gives highest priority to one of the networks
+- generate traffic
+- verify that the prioritized network is used
+- delete the prioritized network
+- generate traffic
+- verify that the remaining network is used
+"""
+
+import os
+import yaml
+import lnetconfig
+from lutf import agents, me
+from lutf_basetest import *
+from lnet import TheLNet
+from lutf_exception import LUTFError
+from lnet_helpers import LNetHelpers
+from lustre_node import SimpleLustreNode
+
+MIN_NODES = 2
+MIN_IFS_PER_NODE = 4
+PING_TIMES = 10
+PING_NID_NUM = 0
+LOCAL_NETS = ['tcp', 'tcp1']
+USE_NET_NUM = [1]
+PRIO_NET = "tcp1"
+EXP_UDSP_SHOW = {'udsp': [{'idx': 0, 'src': 'tcp1', 'dst': 'NA', 'rte': 'NA', 'action': {'priority': 0}}]}
+
+class TestLustreTraffic:
+ def __init__(self, target=None):
+ self.lh = LNetHelpers(os.path.abspath(__file__), target=target)
+ self.sln = SimpleLustreNode(os.path.abspath(__file__), target=target)
+
+
+def getStatNIDStr(stats_dict, nid_str, nid_stat_str):
+ for x in stats_dict:
+ for y in x['local NI(s)']:
+ if y['nid'] == nid_str:
+ return y['statistics'][nid_stat_str]
+ return -1
+
+def getStatNet(stats_dict, net_list, net_num, nid_stat_str):
+ stat_val = 0
+ for x in stats_dict:
+ if (x['net type'] == net_list[net_num]):
+ for y in x['local NI(s)']:
+ stat_val += y['statistics'][nid_stat_str]
+ return stat_val
+
+def getNetNIDbyIdx(stats_dict, net_name, nid_idx):
+ nid_str = ""
+ for x in stats_dict:
+ if (x['net type'] == net_name):
+ nid_str = x['local NI(s)'][nid_idx]['nid']
+ break
+ return nid_str
+
+def run():
+ la = agents.keys()
+ if len(la) < MIN_NODES:
+ return lutfrc(LUTF_TEST_SKIP, "Not enough agents to run the test")
+ nodes = []
+ try:
+ for i in range(0, MIN_NODES):
+ node = TestLustreTraffic(la[i])
+ t = node.lh
+ intfs = t.get_available_devs()
+ if len(intfs) < MIN_IFS_PER_NODE:
+ return lutfrc(LUTF_TEST_SKIP, "Not enough interfaces")
+ t.configure_lnet()
+ if not t.check_udsp_present():
+ return lutfrc(LUTF_TEST_SKIP, "UDSP feature is missing")
+ for j, net in enumerate(LOCAL_NETS):
+ half = len(intfs)//len(LOCAL_NETS)
+ net_intfs_list = intfs[half*(j):half*(j+1)]
+ t.configure_net(net, net_intfs_list)
+ t.set_discovery(1)
+ nodes.append(node)
+
+ main = nodes[1]
+ main_nids = main.lh.list_nids()
+ agent = nodes[0]
+ agent_nids = agent.lh.list_nids()
+
+ # discover all the peers from main
+ if len(main.lh.discover(agent_nids[0])) == 0:
+ return lutfrc(LUTF_TEST_FAIL, "unable to discover" ,
+ target=agent_nids[0])
+
+ rc = main.lh.check_udsp_empty()
+ if not rc:
+ return lutfrc(LUTF_TEST_FAIL, "UDSP list not empty")
+
+ # Prioritize one of the nets
+ rc = main.lh.exec_udsp_cmd(" add --src "+PRIO_NET)
+ # Verify the rule got added
+ rc = main.lh.check_udsp_expected(EXP_UDSP_SHOW)
+ if not rc:
+ return lutfrc(LUTF_TEST_FAIL, "adding UDSP rule failed")
+
+ before_stats_main = main.lh.get_net_stats()
+ #print(before_stats_main)
+
+ for i in range(0, PING_TIMES):
+ rc = main.lh.exec_ping(agent_nids[PING_NID_NUM])
+ if not rc:
+ return lutfrc(LUTF_TEST_FAIL, "ping failed")
+
+ after_stats_main = main.lh.get_net_stats()
+ #print(after_stats_main)
+
+ send_count_before = {}
+ send_count_after = {}
+ total_send_count_before = 0
+ total_send_count_after = 0
+ for net_num in USE_NET_NUM:
+ send_count_before[net_num] = getStatNet(before_stats_main, LOCAL_NETS, net_num, 'send_count')
+ total_send_count_before += send_count_before[net_num]
+ send_count_after[net_num] = getStatNet(after_stats_main, LOCAL_NETS, net_num, 'send_count')
+ total_send_count_after += send_count_after[net_num]
+
+ #print(send_count_before, send_count_after)
+
+ # Check stats:
+ # 1) expect the total send_count to be no less than the number of pings issued
+ # 2) expect the send count to increase by the number of pings issued on the preferred net
+ if (total_send_count_after - total_send_count_before) < PING_TIMES:
+ return lutfrc(LUTF_TEST_FAIL, "total send count mismatch")
+
+ for net_num in USE_NET_NUM:
+ if abs(send_count_after[net_num] - send_count_before[net_num]) < PING_TIMES:
+ print("send count increase on network ", LOCAL_NETS[net_num],
+ " insufficient. Expected ", PING_TIMES, " got ",
+ abs(send_count_after[net_num] - send_count_before[net_num]))
+ return lutfrc(LUTF_TEST_FAIL, "send count increase insufficient")
+
+ main.lh.unconfigure_net(PRIO_NET)
+
+ before_stats_main = main.lh.get_net_stats()
+ #print(before_stats_main)
+
+ for i in range(0, PING_TIMES):
+ rc = main.lh.exec_ping(agent_nids[PING_NID_NUM])
+ if not rc:
+ return lutfrc(LUTF_TEST_FAIL, "ping failed")
+
+ after_stats_main = main.lh.get_net_stats()
+ #print(after_stats_main)
+
+ # Use remaining net
+ USE_NET_NUM[0] = 0
+
+ send_count_before = {}
+ send_count_after = {}
+ total_send_count_before = 0
+ total_send_count_after = 0
+ for net_num in USE_NET_NUM:
+ send_count_before[net_num] = getStatNet(before_stats_main, LOCAL_NETS, net_num, 'send_count')
+ total_send_count_before += send_count_before[net_num]
+ send_count_after[net_num] = getStatNet(after_stats_main, LOCAL_NETS, net_num, 'send_count')
+ total_send_count_after += send_count_after[net_num]
+
+ #print(send_count_before, send_count_after)
+
+ # Check stats:
+ # 1) expect the total send_count to be no less than the number of pings issued
+ # 2) expect the send count to increase by the number of pings issued on the preferred net
+ if (total_send_count_after - total_send_count_before) < PING_TIMES:
+ return lutfrc(LUTF_TEST_FAIL, "total send count mismatch")
+
+ for net_num in USE_NET_NUM:
+ if abs(send_count_after[net_num] - send_count_before[net_num]) < PING_TIMES:
+ print("send count increase on network ", LOCAL_NETS[net_num],
+ " insufficient. Expected ", PING_TIMES, " got ",
+ abs(send_count_after[net_num] - send_count_before[net_num]))
+ return lutfrc(LUTF_TEST_FAIL, "send count increase insufficient")
+ for n in nodes:
+ n.lh.unconfigure_lnet()
+
+ return lutfrc(LUTF_TEST_PASS)
+ except Exception as e:
+ for n in nodes:
+ n.lh.uninit()
+ raise e
+
--- /dev/null
+"""
+@PRIMARY: N/A
+@PRIMARY_DESC: Verify that deleting prioritized local net results no issues
+@SECONDARY: N/A
+@DESIGN: N/A
+@TESTCASE:
+- configure single local network with two nids
+- configure two gws each providing access to a remote network
+- configure remote peer to have access to the remote network with two nids
+- add udsp rule that designates a pair of a router and remote peer nid as preferred
+- add udsp rule that gives the same remote peer nid highest priority
+- generate traffic
+- verify that the preferred remote peer nid an router were used
+"""
+
+import os
+import yaml
+import lnetconfig
+import time
+from lutf import agents, me
+from lutf_basetest import *
+from lnet import TheLNet
+from lutf_exception import LUTFError
+from lnet_helpers import LNetHelpers
+from lustre_node import SimpleLustreNode
+
+MIN_NODES = 4
+MIN_IFS_PER_NODE = 2
+GW_MIN_IFS = 4
+PING_TIMES = 10
+PING_NID_NUM = 0
+LOCAL_NETS = ['tcp']
+REMOTE_NETS = ['tcp1']
+USE_NET_NUM = [0]
+MAIN_NODE_ID = 1
+REMOTE_NODE_ID = 3
+GW_NODE_IDS = [2, 0]
+
+class TestLustreTraffic:
+ def __init__(self, target=None):
+ self.lh = LNetHelpers(os.path.abspath(__file__), target=target)
+ self.sln = SimpleLustreNode(os.path.abspath(__file__), target=target)
+
+
+def getStatNIDStr(stats_dict, nid_str, nid_stat_str):
+ for x in stats_dict:
+ for y in x['local NI(s)']:
+ if y['nid'] == nid_str:
+ return y['statistics'][nid_stat_str]
+ return -1
+
+def getStatNet(stats_dict, net_list, net_num, nid_stat_str):
+ stat_val = 0
+ for x in stats_dict:
+ if (x['net type'] == net_list[net_num]):
+ for y in x['local NI(s)']:
+ stat_val += y['statistics'][nid_stat_str]
+ return stat_val
+
+def getNetNIDbyIdx(stats_dict, net_name, nid_idx):
+ nid_str = ""
+ for x in stats_dict:
+ if (x['net type'] == net_name):
+ nid_str = x['local NI(s)'][nid_idx]['nid']
+ break
+ return nid_str
+
+def run():
+ la = agents.keys()
+ #if len(la) < MIN_NODES:
+ # return lutfrc(LUTF_TEST_SKIP, "Not enough agents to run the test")
+ nodes = []
+ try:
+ # General config for all nodes
+ for i in range(0, MIN_NODES):
+ node = TestLustreTraffic(la[i])
+ t = node.lh
+ if i in GW_NODE_IDS:
+ node_num_ifs = GW_MIN_IFS
+ else:
+ node_num_ifs = MIN_IFS_PER_NODE
+ intfs = t.get_available_devs()
+ if len(intfs) < node_num_ifs:
+ return lutfrc(LUTF_TEST_SKIP, "Not enough interfaces")
+ t.configure_lnet()
+ if not t.check_udsp_present():
+ return lutfrc(LUTF_TEST_SKIP, "UDSP feature is missing")
+
+ if i == MAIN_NODE_ID:
+ # Peer A
+ net = LOCAL_NETS[0]
+ net_intfs_list = intfs[0:node_num_ifs]
+ t.configure_net(net, net_intfs_list)
+ elif i == REMOTE_NODE_ID:
+ # Peer B
+ net = REMOTE_NETS[0]
+ net_intfs_list = intfs[0:node_num_ifs]
+ t.configure_net(net, net_intfs_list)
+ else:
+ # GW1 and GW2
+ net = LOCAL_NETS[0]
+ net_intfs_list = intfs[0:node_num_ifs//2]
+ t.configure_net(net, net_intfs_list)
+ net = REMOTE_NETS[0]
+ net_intfs_list = intfs[node_num_ifs//2:node_num_ifs]
+ t.configure_net(net, net_intfs_list)
+
+ t.set_discovery(1)
+ nodes.append(node)
+
+ main = nodes[MAIN_NODE_ID]
+ main_nids = main.lh.list_nids()
+ remote = nodes[REMOTE_NODE_ID]
+ remote_nids = remote.lh.list_nids()
+ gw1 = nodes[GW_NODE_IDS[0]]
+ gw2 = nodes[GW_NODE_IDS[1]]
+ gw1_nids = gw1.lh.list_nids()
+ gw2_nids = gw2.lh.list_nids()
+
+ #print("main nids: ", main_nids)
+ #print("remote nids: ", remote_nids)
+ #print("gw1 nids: ", gw1_nids)
+ #print("gw2 nids: ", gw2_nids)
+
+ gw1.lh.api_set_routing(True)
+ gw2.lh.api_set_routing(True)
+
+ gw1_remote_nids = [x for x in gw1_nids if REMOTE_NETS[0] in x]
+ gw1_local_nids = [x for x in gw1_nids if x not in gw1_remote_nids]
+
+ gw2_remote_nids = [x for x in gw2_nids if REMOTE_NETS[0] in x]
+ gw2_local_nids = [x for x in gw2_nids if x not in gw2_remote_nids]
+
+ #print("gw1 local/remote nids: ", gw1_local_nids, gw1_remote_nids)
+ #print("gw2 local/remote nids: ", gw2_local_nids, gw2_remote_nids)
+
+ # discover the gateways from main and remote
+ if len(main.lh.exec_discover_cmd(gw1_local_nids[0])) == 0:
+ return lutfrc(LUTF_TEST_FAIL, "unable to discover" ,
+ target=gw1_local_nids[0])
+ if len(main.lh.exec_discover_cmd(gw2_local_nids[0])) == 0:
+ return lutfrc(LUTF_TEST_FAIL, "unable to discover" ,
+ target=gw2_local_nids[0])
+ if len(remote.lh.exec_discover_cmd(gw1_remote_nids[0])) == 0:
+ return lutfrc(LUTF_TEST_FAIL, "unable to discover" ,
+ target=gw1_remote_nids[0])
+ if len(remote.lh.exec_discover_cmd(gw2_remote_nids[0])) == 0:
+ return lutfrc(LUTF_TEST_FAIL, "unable to discover" ,
+ target=gw2_remote_nids[0])
+
+ # setup the routing
+ rt1 = "--net " + REMOTE_NETS[0] + " --gateway " + gw1_local_nids[0]
+ rt2 = "--net " + REMOTE_NETS[0] + " --gateway " + gw2_local_nids[0]
+ rc = main.lh.exec_route_cmd(" add "+rt1)
+ rc = main.lh.exec_route_cmd(" add "+rt2)
+ rt1 = "--net " + LOCAL_NETS[0] + " --gateway " + gw1_remote_nids[0]
+ rt2 = "--net " + LOCAL_NETS[0] + " --gateway " + gw2_remote_nids[0]
+ rc = remote.lh.exec_route_cmd(" add "+rt1)
+ rc = remote.lh.exec_route_cmd(" add "+rt2)
+
+ # commit configuration
+ #gw1.sln.commit()
+ #gw1.lh.api_set_routing(True)
+ #time.sleep(1)
+ #gw2.sln.commit()
+ #gw2.lh.api_set_routing(True)
+ #main.sln.commit()
+ #remote.sln.commit()
+ time.sleep(1)
+
+ rc = main.lh.exec_ping(remote_nids[0])
+ if not rc:
+ return lutfrc(LUTF_TEST_FAIL, "ping failed")
+
+ # let main and remote discover each other
+ try:
+ if len(main.lh.exec_discover_cmd(remote_nids[0])) == 0:
+ return lutfrc(LUTF_TEST_FAIL, "unable to discover" ,
+ target=remote_nids[0])
+ except Exception as e:
+ if len(main.lh.exec_discover_cmd(remote_nids[0])) == 0:
+ return lutfrc(LUTF_TEST_FAIL, "unable to discover" ,
+ target=remote_nids[0])
+ try:
+ if len(remote.lh.exec_discover_cmd(main_nids[0])) == 0:
+ return lutfrc(LUTF_TEST_FAIL, "unable to discover" ,
+ target=main_nids[0])
+ except Exception as e:
+ if len(remote.lh.exec_discover_cmd(main_nids[0])) == 0:
+ return lutfrc(LUTF_TEST_FAIL, "unable to discover" ,
+ target=main_nids[0])
+
+
+ rc = main.lh.check_udsp_empty()
+ if not rc:
+ return lutfrc(LUTF_TEST_FAIL, "UDSP list not empty")
+
+ for net_num in USE_NET_NUM:
+ # Add UDSP rule on main node to prioritize a nid on remote node
+ rc = main.lh.exec_udsp_cmd(" add --dst "+remote_nids[0])
+ # Prioritize gw1 for use with the prioritized remote node
+ rc = main.lh.exec_udsp_cmd(" add --rte "+\
+ gw1_local_nids[0]+\
+ " --dst "+remote_nids[0])
+
+ before_stats_gw1 = gw1.lh.get_stats()
+ #print(before_stats_gw1)
+
+ for i in range(0, PING_TIMES):
+ rc = main.lh.exec_ping(remote_nids[PING_NID_NUM])
+ if not rc:
+ return lutfrc(LUTF_TEST_FAIL, "ping failed")
+
+ after_stats_gw1 = gw1.lh.get_stats()
+ #print(after_stats_gw1)
+
+ route_count_before = before_stats_gw1['statistics']['route_count']
+ route_count_after = after_stats_gw1['statistics']['route_count']
+
+ #print(route_count_before, route_count_after)
+
+ # Check stats:
+ # 1) expect the difference between "before" and "after" route counts to be
+ # same as or greater than the number of pings issued
+ if (route_count_after - route_count_before) < PING_TIMES:
+ return lutfrc(LUTF_TEST_FAIL, "routed count mismatch")
+
+ for i in range(0, MIN_NODES):
+ if i == MAIN_NODE_ID or i == REMOTE_NODE_ID:
+ nodes[i].lh.unconfigure_lnet()
+
+ for i in range(0, MIN_NODES):
+ if i != MAIN_NODE_ID and i != REMOTE_NODE_ID:
+ nodes[i].lh.unconfigure_lnet()
+ #for n in nodes:
+ # n.lh.unconfigure_lnet()
+
+ return lutfrc(LUTF_TEST_PASS)
+ except Exception as e:
+ for n in nodes:
+ n.lh.uninit()
+ raise e
+
--- /dev/null
+"""
+@PRIMARY: N/A
+@PRIMARY_DESC: Verify that local nid can be prioritized for sending via UDSP
+@SECONDARY: N/A
+@DESIGN: N/A
+@TESTCASE:
+- configure one lnet and more than one nid
+- turn on LNet discovery
+- run lnetctl discover
+- verify that udsp list is empty
+- add udsp rule that gives one of the local nids higher priority
+- get lnet stats (lnetctl net show -v 4)
+- generate traffic by running lnetctl ping multiple times
+- get lnet stats again
+- verify that all sends were done using the prioritized nid
+"""
+
+import os
+import yaml
+import lnetconfig
+from lutf import agents, me
+from lutf_basetest import *
+from lnet import TheLNet
+from lutf_exception import LUTFError
+from lnet_helpers import LNetHelpers
+from lustre_node import SimpleLustreNode
+
+MIN_NODES = 2
+MIN_IFS_PER_NODE = 2
+PING_TIMES = 10
+PING_NID_NUM = 0
+LOCAL_NET = 'tcp'
+USE_NID_NUM = 0
+
+class TestLustreTraffic:
+ def __init__(self, target=None):
+ self.lh = LNetHelpers(os.path.abspath(__file__), target=target)
+ self.sln = SimpleLustreNode(os.path.abspath(__file__), target=target)
+
+def run():
+ la = agents.keys()
+ if len(la) < MIN_NODES:
+ return lutfrc(LUTF_TEST_SKIP, "Not enough agents to run the test")
+ nodes = []
+ try:
+ for i in range(0, MIN_NODES):
+ node = TestLustreTraffic(la[i])
+ t = node.lh
+ intfs = t.get_available_devs()
+ if len(intfs) < MIN_IFS_PER_NODE:
+ return lutfrc(LUTF_TEST_SKIP, "Not enough interfaces")
+ if not t.check_udsp_present():
+ return lutfrc(LUTF_TEST_SKIP, "UDSP feature is missing")
+ t.configure_lnet()
+ t.configure_net(LOCAL_NET, intfs)
+ t.set_discovery(1)
+ nodes.append(node)
+
+
+ main = nodes[1]
+ main_nids = main.lh.list_nids()
+ agent = nodes[0]
+ agent_nids = agent.lh.list_nids()
+
+ rc = main.lh.check_udsp_empty()
+ if not rc:
+ print("UDSP list not empty")
+ return lutfrc(LUTF_TEST_FAIL)
+ rc = main.lh.exec_udsp_cmd(" add --src "+main_nids[USE_NID_NUM])
+ #rc = main.lh.exec_udsp_cmd("
+
+ before_stats_main = main.sln.get_lnet().get_net_stats()
+ print(before_stats_main)
+
+ for i in range(0, PING_TIMES):
+ rc = main.lh.exec_ping(agent_nids[PING_NID_NUM])
+ if not rc:
+ print("ping failed")
+ return lutfrc(LUTF_TEST_FAIL)
+
+ after_stats_main = main.sln.get_lnet().get_net_stats()
+ print(after_stats_main)
+
+ send_count_before = -1
+ send_count_after = -1
+ for x in before_stats_main:
+ if (x['net type'] == LOCAL_NET):
+ for y in x['local NI(s)']:
+ if y['nid'] == main_nids[USE_NID_NUM]:
+ send_count_before = y['statistics']['send_count']
+ for x in after_stats_main:
+ if (x['net type'] == LOCAL_NET):
+ for y in x['local NI(s)']:
+ if y['nid'] == main_nids[USE_NID_NUM]:
+ send_count_after = y['statistics']['send_count']
+ if send_count_before < 0 or send_count_after < 0:
+ print("failed to parse net stats")
+ return lutfrc(LUTF_TEST_FAIL)
+
+ if send_count_after - send_count_before < PING_TIMES:
+ print("Unexpected number of sends: ", send_count_after - send_count_before)
+ return lutfrc(LUTF_TEST_FAIL)
+
+ for n in nodes:
+ n.lh.unconfigure_lnet()
+
+ return lutfrc(LUTF_TEST_PASS)
+ except Exception as e:
+ for n in nodes:
+ n.lh.uninit()
+ raise e
+
--- /dev/null
+"""
+@PRIMARY: N/A
+@PRIMARY_DESC: N/A
+@PRIMARY_DESC: Verify that multiple local nids can be prioritized for sending via UDSP
+@SECONDARY: N/A
+@DESIGN: N/A
+@TESTCASE:
+- configure one lnet and more than two nids
+- turn on LNet discovery
+- run lnetctl discover
+- verify that udsp list is empty
+- add udsp rule that gives two of the local nids higher priority
+- get lnet stats (lnetctl net show -v 4)
+- generate traffic by running lnetctl ping multiple times
+- get lnet stats again
+- verify that all sends were done using the prioritized nids
+"""
+
+import os
+import yaml
+import lnetconfig
+from lutf import agents, me
+from lutf_basetest import *
+from lnet import TheLNet
+from lutf_exception import LUTFError
+from lnet_helpers import LNetHelpers
+from lustre_node import SimpleLustreNode
+
+MIN_NODES = 2
+MIN_IFS_PER_NODE = 3
+PING_TIMES = 10
+PING_NID_NUM = 0
+LOCAL_NET = 'tcp'
+USE_NID_NUM = [0, 1]
+
+class TestLustreTraffic:
+ def __init__(self, target=None):
+ self.lh = LNetHelpers(os.path.abspath(__file__), target=target)
+ self.sln = SimpleLustreNode(os.path.abspath(__file__), target=target)
+
+def getStatNID(stats_dict, nid_list, nid_num, nid_stat_str):
+ for x in stats_dict:
+ if (x['net type'] == LOCAL_NET):
+ for y in x['local NI(s)']:
+ if y['nid'] == nid_list[nid_num]:
+ return y['statistics'][nid_stat_str]
+ return -1
+
+
+def run():
+ la = agents.keys()
+ if len(la) < MIN_NODES:
+ return lutfrc(LUTF_TEST_SKIP, "Not enough agents to run the test")
+ nodes = []
+ try:
+ for i in range(0, MIN_NODES):
+ node = TestLustreTraffic(la[i])
+ t = node.lh
+ intfs = t.get_available_devs()
+ if len(intfs) < MIN_IFS_PER_NODE:
+ return lutfrc(LUTF_TEST_SKIP, "Not enough interfaces")
+ if not t.check_udsp_present():
+ return lutfrc(LUTF_TEST_SKIP, "UDSP feature is missing")
+ t.configure_lnet()
+ t.configure_net(LOCAL_NET, intfs)
+ t.set_discovery(1)
+ nodes.append(node)
+
+ main = nodes[1]
+ main_nids = main.lh.list_nids()
+ agent = nodes[0]
+ agent_nids = agent.lh.list_nids()
+ print("nids: ", main_nids)
+
+ # discover all the peers from main
+ if len(main.lh.discover(agent_nids[0])) == 0:
+ return lutfrc(LUTF_TEST_FAIL, "unable to discover" ,
+ target=agent_nids[0])
+
+ rc = main.lh.check_udsp_empty()
+ if not rc:
+ print("UDSP list not empty")
+ return lutfrc(LUTF_TEST_FAIL)
+
+ for nid_num in USE_NID_NUM:
+ rc = main.lh.exec_udsp_cmd(" add --src "+main_nids[nid_num])
+
+ before_stats_main = main.sln.get_lnet().get_net_stats()
+ print(before_stats_main)
+
+ for i in range(0, PING_TIMES):
+ rc = main.lh.exec_ping(agent_nids[PING_NID_NUM])
+ if not rc:
+ print("ping failed")
+ return lutfrc(LUTF_TEST_FAIL)
+
+ after_stats_main = main.sln.get_lnet().get_net_stats()
+ print(after_stats_main)
+
+ send_count_before = {}
+ send_count_after = {}
+ total_send_count_before = 0
+ total_send_count_after = 0
+ for nid_num in USE_NID_NUM:
+ #print({nid_num: getStatNID(before_stats_main, main_nids, nid_num, 'send_count')})
+ #print({nid_num: getStatNID(after_stats_main, main_nids, nid_num, 'send_count')})
+ send_count_before[nid_num] = getStatNID(before_stats_main, main_nids, nid_num, 'send_count')
+ total_send_count_before += send_count_before[nid_num]
+ send_count_after[nid_num] = getStatNID(after_stats_main, main_nids, nid_num, 'send_count')
+ total_send_count_after += send_count_after[nid_num]
+
+ """
+ for x in before_stats_main:
+ if (x['net type'] == LOCAL_NET):
+ for y in x['local NI(s)']:
+ if y['nid'] == main_nids[USE_NID_NUM]:
+ send_count_before = y['statistics']['send_count']
+ for x in after_stats_main:
+ if (x['net type'] == LOCAL_NET):
+ for y in x['local NI(s)']:
+ if y['nid'] == main_nids[USE_NID_NUM]:
+ send_count_after = y['statistics']['send_count']
+ if send_count_before < 0 or send_count_after < 0:
+ print("failed to parse net stats")
+ return lutfrc(LUTF_TEST_FAIL)
+
+ if send_count_after - send_count_before < PING_TIMES:
+ print("Unexpected number of sends: ", send_count_after - send_count_before)
+ return lutfrc(LUTF_TEST_FAIL)
+ """
+
+ print(send_count_before, send_count_after)
+
+ # Check stats:
+ # 1) expect send counts on both interfaces to increase
+ # 2) expect the total send_count to be no less than the number of pings issued
+ # 3) expect the send counts on the two interfaces used to be close (diff < 2)
+ if (total_send_count_after - total_send_count_before) < PING_TIMES:
+ print("total send count mismatch")
+ return lutfrc(LUTF_TEST_FAIL)
+
+ if abs(send_count_after[0] - send_count_after[1]) > 1:
+ print("uneven tx traffic distribution across interfaces")
+ return lutfrc(LUTF_TEST_FAIL)
+
+
+ for n in nodes:
+ n.lh.unconfigure_lnet()
+
+ return lutfrc(LUTF_TEST_PASS)
+ except Exception as e:
+ for n in nodes:
+ n.lh.uninit()
+ raise e
+
--- /dev/null
+"""
+@PRIMARY: N/A
+@PRIMARY_DESC: Verify that local nid can be deprioritized for sending via UDSP
+@SECONDARY: N/A
+@DESIGN: N/A
+@TESTCASE:
+- configure one lnet and more than two nids
+- turn on LNet discovery
+- run lnetctl discover
+- verify that udsp list is empty
+- add udsp rule that gives two of the local nids higher priority
+- get lnet stats (lnetctl net show -v 4)
+- generate traffic by running lnetctl ping multiple times
+- get lnet stats again
+- verify that all sends were done using the prioritized nids
+- delete udsp rule for one of the nids
+- generate traffic by running lnetctl ping multiple times
+- get lnet stats again
+- verify that all sends were done using the remaining prioritized nid
+"""
+
+import os
+import yaml
+import lnetconfig
+from lutf import agents, me
+from lutf_basetest import *
+from lnet import TheLNet
+from lutf_exception import LUTFError
+from lnet_helpers import LNetHelpers
+from lustre_node import SimpleLustreNode
+
+MIN_NODES = 2
+MIN_IFS_PER_NODE = 3
+PING_TIMES = 10
+PING_NID_NUM = 0
+LOCAL_NET = 'tcp'
+NID_NUM_A = 0
+NID_NUM_B = 1
+USE_NID_NUM = [NID_NUM_A, NID_NUM_B]
+
+class TestLustreTraffic:
+ def __init__(self, target=None):
+ self.lh = LNetHelpers(os.path.abspath(__file__), target=target)
+ self.sln = SimpleLustreNode(os.path.abspath(__file__), target=target)
+
+def getStatNID(stats_dict, nid_list, nid_num, nid_stat_str):
+ for x in stats_dict:
+ if (x['net type'] == LOCAL_NET):
+ for y in x['local NI(s)']:
+ if y['nid'] == nid_list[nid_num]:
+ return y['statistics'][nid_stat_str]
+ return -1
+
+
+def run():
+ la = agents.keys()
+ if len(la) < MIN_NODES:
+ return lutfrc(LUTF_TEST_SKIP, "Not enough agents to run the test")
+ nodes = []
+ try:
+ for i in range(0, MIN_NODES):
+ node = TestLustreTraffic(la[i])
+ t = node.lh
+ intfs = t.get_available_devs()
+ if len(intfs) < MIN_IFS_PER_NODE:
+ return lutfrc(LUTF_TEST_SKIP, "Not enough interfaces")
+ if not t.check_udsp_present():
+ return lutfrc(LUTF_TEST_SKIP, "UDSP feature is missing")
+ t.configure_lnet()
+ t.configure_net(LOCAL_NET, intfs)
+ t.set_discovery(1)
+ nodes.append(node)
+
+ main = nodes[1]
+ main_nids = main.lh.list_nids()
+ agent = nodes[0]
+ agent_nids = agent.lh.list_nids()
+
+ # discover all the peers from main
+ if len(main.lh.discover(agent_nids[0])) == 0:
+ return lutfrc(LUTF_TEST_FAIL, "unable to discover" ,
+ target=agent_nids[0])
+
+ rc = main.lh.check_udsp_empty()
+ if not rc:
+ print("UDSP list not empty")
+ return lutfrc(LUTF_TEST_FAIL)
+
+ for nid_num in USE_NID_NUM:
+ rc = main.lh.exec_udsp_cmd(" add --src "+main_nids[nid_num])
+
+ before_stats_main = main.sln.get_lnet().get_net_stats()
+ print(before_stats_main)
+
+ for i in range(0, PING_TIMES):
+ rc = main.lh.exec_ping(agent_nids[PING_NID_NUM])
+ if not rc:
+ print("ping failed")
+ return lutfrc(LUTF_TEST_FAIL)
+
+ after_stats_main = main.sln.get_lnet().get_net_stats()
+ print(after_stats_main)
+
+ send_count_before = {}
+ send_count_after = {}
+ total_send_count_before = 0
+ total_send_count_after = 0
+ for nid_num in USE_NID_NUM:
+ #print({nid_num: getStatNID(before_stats_main, main_nids, nid_num, 'send_count')})
+ #print({nid_num: getStatNID(after_stats_main, main_nids, nid_num, 'send_count')})
+ send_count_before[nid_num] = getStatNID(before_stats_main, main_nids, nid_num, 'send_count')
+ total_send_count_before += send_count_before[nid_num]
+ send_count_after[nid_num] = getStatNID(after_stats_main, main_nids, nid_num, 'send_count')
+ total_send_count_after += send_count_after[nid_num]
+
+ print(send_count_before, send_count_after)
+
+ # Check stats:
+ # 1) expect send counts on both interfaces to increase
+ # 2) expect the total send_count to be no less than the number of pings issued
+ # 3) expect the send counts on the two interfaces used to be close (diff < 2)
+ if (total_send_count_after - total_send_count_before) < PING_TIMES:
+ print("total send count mismatch")
+ return lutfrc(LUTF_TEST_FAIL)
+
+ if abs(send_count_after[NID_NUM_A] - send_count_after[NID_NUM_B]) > 1:
+ print("uneven tx traffic distribution across interfaces")
+ return lutfrc(LUTF_TEST_FAIL)
+
+ # Delete the UDSP for the first of the nids
+ rc = main.lh.exec_udsp_cmd(" del --idx 0 ")
+
+ # Generate traffic and collect stats
+ before_stats_main = main.sln.get_lnet().get_net_stats()
+ print(before_stats_main)
+
+ for i in range(0, PING_TIMES):
+ rc = main.lh.exec_ping(agent_nids[PING_NID_NUM])
+ if not rc:
+ print("ping failed")
+ return lutfrc(LUTF_TEST_FAIL)
+
+ after_stats_main = main.sln.get_lnet().get_net_stats()
+ print(after_stats_main)
+
+ send_count_before = {}
+ send_count_after = {}
+ total_send_count_before = 0
+ total_send_count_after = 0
+ for nid_num in USE_NID_NUM:
+ #print({nid_num: getStatNID(before_stats_main, main_nids, nid_num, 'send_count')})
+ #print({nid_num: getStatNID(after_stats_main, main_nids, nid_num, 'send_count')})
+ send_count_before[nid_num] = getStatNID(before_stats_main, main_nids, nid_num, 'send_count')
+ total_send_count_before += send_count_before[nid_num]
+ send_count_after[nid_num] = getStatNID(after_stats_main, main_nids, nid_num, 'send_count')
+ total_send_count_after += send_count_after[nid_num]
+
+ print(send_count_before, send_count_after)
+
+ # Check stats:
+ # 1) expect send count on second interface to increase
+ # 2) expect the total send_count to be no less than the number of pings issued
+ # 3) expect the send count on the preferred interface to increase by no less than the
+ # number of pings issued
+ # 4) expect the send count on the preferred interface to be greater than on the other interface
+ # by at least the number of pings less one
+ if (total_send_count_after - total_send_count_before) < PING_TIMES:
+ print("total send count mismatch")
+ return lutfrc(LUTF_TEST_FAIL)
+
+ if abs(send_count_after[NID_NUM_B] - send_count_before[NID_NUM_B]) < PING_TIMES:
+ print("send count increase on preferred interface ", NID_NUM_B,
+ " insufficient. Expected ", PING_TIMES, " got",
+ abs(send_count_after[NID_NUM_B] - send_count_before[NID_NUM_B]))
+ return lutfrc(LUTF_TEST_FAIL)
+
+ for n in nodes:
+ n.lh.unconfigure_lnet()
+
+ return lutfrc(LUTF_TEST_PASS)
+ except Exception as e:
+ for n in nodes:
+ n.lh.uninit()
+ raise e
+
--- /dev/null
+"""
+@PRIMARY: N/A
+@PRIMARY_DESC: Verify that multiple local nids can be deprioritized for sending via UDSP
+@SECONDARY: N/A
+@DESIGN: N/A
+@TESTCASE:
+- configure one lnet and more than two nids
+- turn on LNet discovery
+- run lnetctl discover
+- verify that udsp list is empty
+- add udsp rule that gives two of the local nids higher priority
+- get lnet stats (lnetctl net show -v 4)
+- generate traffic by running lnetctl ping multiple times
+- get lnet stats again
+- verify that all sends were done using the prioritized nids
+- delete udsp rules for both prioritized nids
+- generate traffic by running lnetctl ping multiple times
+- get lnet stats again
+- verify that all sends were spread evenly across local nids
+"""
+
+import os
+import yaml
+import lnetconfig
+from lutf import agents, me
+from lutf_basetest import *
+from lnet import TheLNet
+from lutf_exception import LUTFError
+from lnet_helpers import LNetHelpers
+from lustre_node import SimpleLustreNode
+
+MIN_NODES = 2
+MIN_IFS_PER_NODE = 3
+PING_TIMES = 10
+PING_NID_NUM = 0
+LOCAL_NET = 'tcp'
+NID_NUM_A = 0
+NID_NUM_B = 1
+USE_NID_NUM = [NID_NUM_A, NID_NUM_B]
+
+class TestLustreTraffic:
+ def __init__(self, target=None):
+ self.lh = LNetHelpers(os.path.abspath(__file__), target=target)
+ self.sln = SimpleLustreNode(os.path.abspath(__file__), target=target)
+
+def getStatNID(stats_dict, nid_list, nid_num, nid_stat_str):
+ for x in stats_dict:
+ if (x['net type'] == LOCAL_NET):
+ for y in x['local NI(s)']:
+ if y['nid'] == nid_list[nid_num]:
+ return y['statistics'][nid_stat_str]
+ return -1
+
+
+def run():
+ la = agents.keys()
+ if len(la) < MIN_NODES:
+ return lutfrc(LUTF_TEST_SKIP, "Not enough agents to run the test")
+ nodes = []
+ try:
+ for i in range(0, MIN_NODES):
+ node = TestLustreTraffic(la[i])
+ t = node.lh
+ intfs = t.get_available_devs()
+ if len(intfs) < MIN_IFS_PER_NODE:
+ return lutfrc(LUTF_TEST_SKIP, "Not enough interfaces")
+ if not t.check_udsp_present():
+ return lutfrc(LUTF_TEST_SKIP, "UDSP feature is missing")
+ t.configure_lnet()
+ t.configure_net(LOCAL_NET, intfs)
+ t.set_discovery(1)
+ nodes.append(node)
+
+ main = nodes[1]
+ main_nids = main.lh.list_nids()
+ agent = nodes[0]
+ agent_nids = agent.lh.list_nids()
+ print("nids: ", main_nids)
+
+ # discover all the peers from main
+ if len(main.lh.discover(agent_nids[0])) == 0:
+ return lutfrc(LUTF_TEST_FAIL, "unable to discover" ,
+ target=agent_nids[0])
+
+ rc = main.lh.check_udsp_empty()
+ if not rc:
+ print("UDSP list not empty")
+ return lutfrc(LUTF_TEST_FAIL)
+
+ for nid_num in USE_NID_NUM:
+ rc = main.lh.exec_udsp_cmd(" add --src "+main_nids[nid_num])
+
+ before_stats_main = main.sln.get_lnet().get_net_stats()
+ print(before_stats_main)
+
+ for i in range(0, PING_TIMES):
+ rc = main.lh.exec_ping(agent_nids[PING_NID_NUM])
+ if not rc:
+ print("ping failed")
+ return lutfrc(LUTF_TEST_FAIL)
+
+ after_stats_main = main.sln.get_lnet().get_net_stats()
+ print(after_stats_main)
+
+ send_count_before = {}
+ send_count_after = {}
+ total_send_count_before = 0
+ total_send_count_after = 0
+ for nid_num in USE_NID_NUM:
+ #print({nid_num: getStatNID(before_stats_main, main_nids, nid_num, 'send_count')})
+ #print({nid_num: getStatNID(after_stats_main, main_nids, nid_num, 'send_count')})
+ send_count_before[nid_num] = getStatNID(before_stats_main, main_nids, nid_num, 'send_count')
+ total_send_count_before += send_count_before[nid_num]
+ send_count_after[nid_num] = getStatNID(after_stats_main, main_nids, nid_num, 'send_count')
+ total_send_count_after += send_count_after[nid_num]
+
+ print(send_count_before, send_count_after)
+
+ # Check stats:
+ # 1) expect send counts on both interfaces to increase
+ # 2) expect the total send_count to be no less than the number of pings issued
+ # 3) expect the send counts on the two interfaces used to be close (diff < 2)
+ if (total_send_count_after - total_send_count_before) < PING_TIMES:
+ print("total send count mismatch")
+ return lutfrc(LUTF_TEST_FAIL)
+
+ if abs(send_count_after[NID_NUM_A] - send_count_after[NID_NUM_B]) > 1:
+ print("uneven tx traffic distribution across interfaces")
+ return lutfrc(LUTF_TEST_FAIL)
+
+ # Delete the UDSP for both nids
+ rc = main.lh.exec_udsp_cmd(" del --idx 0 ")
+ rc = main.lh.exec_udsp_cmd(" del --idx 0 ")
+
+ # Generate traffic and collect stats
+ before_stats_main = main.sln.get_lnet().get_net_stats()
+ print(before_stats_main)
+
+ for i in range(0, PING_TIMES):
+ rc = main.lh.exec_ping(agent_nids[PING_NID_NUM])
+ if not rc:
+ print("ping failed")
+ return lutfrc(LUTF_TEST_FAIL)
+
+ after_stats_main = main.sln.get_lnet().get_net_stats()
+ print(after_stats_main)
+
+ send_count_before = {}
+ send_count_after = {}
+ total_send_count_before = 0
+ total_send_count_after = 0
+ nid_num = 0
+ for nid in main_nids:
+ #print({nid_num: getStatNID(before_stats_main, main_nids, nid_num, 'send_count')})
+ #print({nid_num: getStatNID(after_stats_main, main_nids, nid_num, 'send_count')})
+ send_count_before[nid_num] = getStatNID(before_stats_main, main_nids, nid_num, 'send_count')
+ total_send_count_before += send_count_before[nid_num]
+ send_count_after[nid_num] = getStatNID(after_stats_main, main_nids, nid_num, 'send_count')
+ total_send_count_after += send_count_after[nid_num]
+ nid_num+=1
+
+ print(send_count_before, send_count_after)
+
+ # Check stats:
+ # 1) expect send count on all interfaces to increase
+ # 2) expect the total send_count to increase by no less than the number of pings issued
+ # 3) expect the send counts on all interfaces to be close
+ if (total_send_count_after - total_send_count_before) < PING_TIMES:
+ print("total send count mismatch")
+ return lutfrc(LUTF_TEST_FAIL)
+
+ if abs(send_count_after[NID_NUM_A] - send_count_after[NID_NUM_B]) > 1 or \
+ abs(send_count_after[NID_NUM_A] - send_count_after[NID_NUM_B+1]) > 1:
+ print("uneven tx traffic distribution across interfaces")
+ return lutfrc(LUTF_TEST_FAIL)
+
+ for n in nodes:
+ n.lh.unconfigure_lnet()
+
+ return lutfrc(LUTF_TEST_PASS)
+ except Exception as e:
+ for n in nodes:
+ n.lh.uninit()
+ raise e
+
--- /dev/null
+"""
+@PRIMARY: N/A
+@PRIMARY_DESC: Verify that destination nid can be prioritized for receiving via UDSP
+@SECONDARY: N/A
+@DESIGN: N/A
+@TESTCASE:
+- configure one lnet and more than one nid
+- turn on LNet discovery
+- run lnetctl discover
+- verify that udsp list is empty
+- add udsp rule that gives one of the peer nids higher priority
+- get lnet stats (lnetctl net show -v 4)
+- generate traffic by running lnetctl ping multiple times
+- get lnet stats again
+- verify that the prioritized nid received all pings
+"""
+
+import os
+import yaml
+import lnetconfig
+from lutf import agents, me
+from lutf_basetest import *
+from lnet import TheLNet
+from lutf_exception import LUTFError
+from lnet_helpers import LNetHelpers
+from lustre_node import SimpleLustreNode
+
+MIN_IFS_PER_NODE = 3
+PING_TIMES = 10
+PING_NID_NUM = 0
+LOCAL_NET = 'tcp'
+USE_NID_NUM = [0]
+MIN_NODES = 2
+LOCAL_DEBUG = 1
+
+class TestLustreTraffic:
+ def __init__(self, target=None):
+ self.lh = LNetHelpers(os.path.abspath(__file__), target=target)
+ self.sln = SimpleLustreNode(os.path.abspath(__file__), target=target)
+
+def getNetStatNID(stats_dict, nid_list, nid_num, nid_stat_str):
+ for x in stats_dict:
+ if (x['net type'] == LOCAL_NET):
+ for y in x['local NI(s)']:
+ if y['nid'] == nid_list[nid_num]:
+ return y['statistics'][nid_stat_str]
+ return -1
+
+def udsp_print(*args):
+ if LOCAL_DEBUG:
+ print(args)
+
+def run():
+ la = agents.keys()
+ if len(la) < MIN_NODES:
+ return lutfrc(LUTF_TEST_SKIP, "Not enough agents to run the test")
+ nodes = []
+ try:
+ for i in range(0, MIN_NODES):
+ node = TestLustreTraffic(la[i])
+ t = node.lh
+ intfs = t.get_available_devs()
+ if len(intfs) < MIN_IFS_PER_NODE:
+ return lutfrc(LUTF_TEST_SKIP, "Not enough interfaces")
+ if not t.check_udsp_present():
+ return lutfrc(LUTF_TEST_SKIP, "UDSP feature is missing")
+ t.configure_lnet()
+ t.configure_net(LOCAL_NET, intfs)
+ t.set_discovery(1)
+ nodes.append(node)
+
+ main = nodes[1]
+ main_nids = main.lh.list_nids()
+ agent = nodes[0]
+ agent_nids = agent.lh.list_nids()
+
+ # discover all the peers from main
+ if len(main.lh.discover(agent_nids[0])) == 0:
+ return lutfrc(LUTF_TEST_FAIL, "unable to discover" ,
+ target=agent_nids[0])
+
+ rc = main.lh.check_udsp_empty()
+ if not rc:
+ print("UDSP list not empty")
+ return lutfrc(LUTF_TEST_FAIL)
+
+ for nid_num in USE_NID_NUM:
+ rc = main.lh.exec_udsp_cmd(" add --dst "+agent_nids[nid_num])
+
+ before_stats_agent = agent.lh.get_net_stats()
+ udsp_print(before_stats_agent)
+
+ for i in range(0, PING_TIMES):
+ rc = main.lh.exec_ping(agent_nids[PING_NID_NUM])
+ if not rc:
+ udsp_print("ping failed")
+ return lutfrc(LUTF_TEST_FAIL)
+
+ after_stats_agent = agent.lh.get_net_stats()
+ udsp_print(after_stats_agent)
+
+ recv_count_before = {}
+ recv_count_after = {}
+ total_recv_count_before = 0
+ total_recv_count_after = 0
+ nid_num = 0
+ for nid in agent_nids:
+ #print({nid_num: getStatNID(before_stats_main, main_nids, nid_num, 'send_count')})
+ #print({nid_num: getStatNID(after_stats_main, main_nids, nid_num, 'send_count')})
+ recv_count_before[nid_num] = getNetStatNID(before_stats_agent, agent_nids, nid_num, 'recv_count')
+ total_recv_count_before += recv_count_before[nid_num]
+ recv_count_after[nid_num] = getNetStatNID(after_stats_agent, agent_nids, nid_num, 'recv_count')
+ total_recv_count_after += recv_count_after[nid_num]
+ nid_num += 1
+
+ """
+ for x in before_stats_main:
+ if (x['net type'] == LOCAL_NET):
+ for y in x['local NI(s)']:
+ if y['nid'] == main_nids[USE_NID_NUM]:
+ send_count_before = y['statistics']['send_count']
+ for x in after_stats_main:
+ if (x['net type'] == LOCAL_NET):
+ for y in x['local NI(s)']:
+ if y['nid'] == main_nids[USE_NID_NUM]:
+ send_count_after = y['statistics']['send_count']
+ if send_count_before < 0 or send_count_after < 0:
+ print("failed to parse net stats")
+ return lutfrc(LUTF_TEST_FAIL)
+
+ if send_count_after - send_count_before < PING_TIMES:
+ print("Unexpected number of sends: ", send_count_after - send_count_before)
+ return lutfrc(LUTF_TEST_FAIL)
+ """
+
+ udsp_print(recv_count_before, recv_count_after)
+
+ # Check
+ # 1) expect the total recv_count on the agent to be no less than the number of pings issued
+ # 2) expect the recv count on the preferred peer interface to increase by the number of pings issues
+ if (total_recv_count_after - total_recv_count_before) < PING_TIMES:
+ udsp_print("total recv count mismatch")
+ return lutfrc(LUTF_TEST_FAIL)
+
+ if abs(recv_count_after[0] - recv_count_before[0]) < PING_TIMES:
+ udsp_print("less than expected traffic on the prioritized peer nid")
+ return lutfrc(LUTF_TEST_FAIL)
+
+
+ for n in nodes:
+ n.lh.unconfigure_lnet()
+
+ return lutfrc(LUTF_TEST_PASS)
+ except Exception as e:
+ for n in nodes:
+ n.lh.uninit()
+ raise e
+
--- /dev/null
+"""
+@PRIMARY: N/A
+@PRIMARY_DESC: Verify UDSP nid pair rule in combination with local nid prioritization rule
+@SECONDARY: N/A
+@DESIGN: N/A
+@TESTCASE:
+- configure one lnet and more than one nid
+- turn on LNet discovery
+- run lnetctl discover
+- verify that udsp list is empty
+- add udsp rule that gives one of the local nids higher priority
+- add udsp rule that pairs the prioritized local nid with a certain peer nid
+- get lnet stats (lnetctl net show -v 4)
+- generate traffic by running lnetctl ping multiple times
+- get lnet stats again
+- verify that the paired destination nid received all pings
+"""
+
+import os
+import yaml
+import lnetconfig
+from lutf import agents, me
+from lutf_basetest import *
+from lnet import TheLNet
+from lutf_exception import LUTFError
+from lnet_helpers import LNetHelpers
+from lustre_node import SimpleLustreNode
+
+MIN_IFS_PER_NODE = 3
+PING_TIMES = 10
+PING_NID_NUM = 0
+LOCAL_NET = 'tcp'
+USE_NID_NUM = [0]
+MIN_NODES = 2
+LOCAL_DEBUG = 1
+
+class TestLustreTraffic:
+ def __init__(self, target=None):
+ self.lh = LNetHelpers(os.path.abspath(__file__), target=target)
+ self.sln = SimpleLustreNode(os.path.abspath(__file__), target=target)
+
+def getNetStatNID(stats_dict, nid_list, nid_num, nid_stat_str):
+ for x in stats_dict:
+ if (x['net type'] == LOCAL_NET):
+ for y in x['local NI(s)']:
+ if y['nid'] == nid_list[nid_num]:
+ return y['statistics'][nid_stat_str]
+ return -1
+
+def udsp_print(*args):
+ if LOCAL_DEBUG:
+ print(args)
+
+def run():
+ la = agents.keys()
+ if len(la) < MIN_NODES:
+ return lutfrc(LUTF_TEST_SKIP, "Not enough agents to run the test")
+ nodes = []
+ try:
+ for i in range(0, MIN_NODES):
+ node = TestLustreTraffic(la[i])
+ t = node.lh
+ intfs = t.get_available_devs()
+ if len(intfs) < MIN_IFS_PER_NODE:
+ return lutfrc(LUTF_TEST_SKIP, "Not enough interfaces")
+ if not t.check_udsp_present():
+ return lutfrc(LUTF_TEST_SKIP, "UDSP feature is missing")
+ t.configure_lnet()
+ t.configure_net(LOCAL_NET, intfs)
+ t.set_discovery(1)
+ nodes.append(node)
+
+ main = nodes[1]
+ main_nids = main.lh.list_nids()
+ agent = nodes[0]
+ agent_nids = agent.lh.list_nids()
+
+ # discover all the peers from main
+ if len(main.lh.discover(agent_nids[0])) == 0:
+ return lutfrc(LUTF_TEST_FAIL, "unable to discover" ,
+ target=agent_nids[0])
+
+ rc = main.lh.check_udsp_empty()
+ if not rc:
+ udsp_print("UDSP list not empty")
+ return lutfrc(LUTF_TEST_FAIL)
+
+ for nid_num in USE_NID_NUM:
+ rc = main.lh.exec_udsp_cmd(" add --src "+main_nids[nid_num])
+ rc = main.lh.exec_udsp_cmd(" add --src "+main_nids[nid_num]+" --dst "+agent_nids[nid_num])
+ before_stats_agent = agent.lh.get_net_stats()
+ udsp_print(before_stats_agent)
+
+ for i in range(0, PING_TIMES):
+ rc = main.lh.exec_ping(agent_nids[PING_NID_NUM])
+ if not rc:
+ udsp_print("ping failed")
+ return lutfrc(LUTF_TEST_FAIL)
+
+ after_stats_agent = agent.lh.get_net_stats()
+ udsp_print(after_stats_agent)
+
+ recv_count_before = {}
+ recv_count_after = {}
+ total_recv_count_before = 0
+ total_recv_count_after = 0
+ nid_num = 0
+ for nid in agent_nids:
+ recv_count_before[nid_num] = getNetStatNID(before_stats_agent, agent_nids, nid_num, 'recv_count')
+ total_recv_count_before += recv_count_before[nid_num]
+ recv_count_after[nid_num] = getNetStatNID(after_stats_agent, agent_nids, nid_num, 'recv_count')
+ total_recv_count_after += recv_count_after[nid_num]
+ nid_num += 1
+
+ udsp_print(recv_count_before, recv_count_after)
+
+ # Check
+ # 1) expect the total recv_count on the agent to be no less than the number of pings issued
+ # 2) expect the recv count on the preferred peer interface to increase by the number of pings issues
+ if (total_recv_count_after - total_recv_count_before) < PING_TIMES:
+ udsp_print("total recv count mismatch")
+ return lutfrc(LUTF_TEST_FAIL)
+
+ if abs(recv_count_after[0] - recv_count_before[0]) < PING_TIMES:
+ udsp_print("less than expected traffic on the prioritized peer nid")
+ return lutfrc(LUTF_TEST_FAIL)
+
+ for n in nodes:
+ n.lh.unconfigure_lnet()
+
+ return lutfrc(LUTF_TEST_PASS)
+ except Exception as e:
+ for n in nodes:
+ n.lh.uninit()
+ raise e
+
--- /dev/null
+"""
+@PRIMARY: N/A
+@PRIMARY_DESC: Verify multiple nid pairing UDSP rule
+@SECONDARY: N/A
+@DESIGN: N/A
+@TESTCASE:
+- configure one lnet and more than two nids
+- turn on LNet discovery
+- run lnetctl discover
+- verify that udsp list is empty
+- add udsp rule that pairs all local nids with one of the peer nids
+- get lnet stats (lnetctl net show -v 4)
+- generate traffic by running lnetctl ping multiple times
+- get lnet stats again
+- verify that the paired peer nid received all pings
+"""
+
+import os
+import yaml
+import lnetconfig
+from lutf import agents, me
+from lutf_basetest import *
+from lnet import TheLNet
+from lutf_exception import LUTFError
+from lnet_helpers import LNetHelpers
+from lustre_node import SimpleLustreNode
+
+MIN_IFS_PER_NODE = 3
+PING_TIMES = 10
+PING_NID_NUM = 0
+LOCAL_NET = 'tcp'
+USE_NID_NUM = [0, 1, 2]
+MIN_NODES = 2
+LOCAL_DEBUG = 1
+
+class TestLustreTraffic:
+ def __init__(self, target=None):
+ self.lh = LNetHelpers(os.path.abspath(__file__), target=target)
+ self.sln = SimpleLustreNode(os.path.abspath(__file__), target=target)
+
+def getNetStatNID(stats_dict, nid_list, nid_num, nid_stat_str):
+ for x in stats_dict:
+ if (x['net type'] == LOCAL_NET):
+ for y in x['local NI(s)']:
+ if y['nid'] == nid_list[nid_num]:
+ return y['statistics'][nid_stat_str]
+ return -1
+
+def udsp_print(*args):
+ if LOCAL_DEBUG:
+ print(args)
+
+def run():
+ la = agents.keys()
+ if len(la) < MIN_NODES:
+ return lutfrc(LUTF_TEST_SKIP, "Not enough agents to run the test")
+ nodes = []
+ try:
+ for i in range(0, MIN_NODES):
+ node = TestLustreTraffic(la[i])
+ t = node.lh
+ intfs = t.get_available_devs()
+ if len(intfs) < MIN_IFS_PER_NODE:
+ return lutfrc(LUTF_TEST_SKIP, "Not enough interfaces")
+ if not t.check_udsp_present():
+ return lutfrc(LUTF_TEST_SKIP, "UDSP feature is missing")
+ t.configure_lnet()
+ t.configure_net(LOCAL_NET, intfs)
+ t.set_discovery(1)
+ nodes.append(node)
+
+ main = nodes[1]
+ main_nids = main.lh.list_nids()
+ agent = nodes[0]
+ agent_nids = agent.lh.list_nids()
+ udsp_print("nids: ", main_nids)
+
+ # discover all the peers from main
+ if len(main.lh.discover(agent_nids[0])) == 0:
+ return lutfrc(LUTF_TEST_FAIL, "unable to discover" ,
+ target=agent_nids[0])
+
+ rc = main.lh.check_udsp_empty()
+ if not rc:
+ udsp_print("UDSP list not empty")
+ return lutfrc(LUTF_TEST_FAIL)
+
+ cmd_str = " add --src *.*.*.*@" + LOCAL_NET + " --dst "+agent_nids[PING_NID_NUM]
+
+ rc = main.lh.exec_udsp_cmd(cmd_str)
+
+ rc = main.lh.exec_udsp_cmd(" show")
+ print(rc)
+
+ before_stats_agent = agent.lh.get_net_stats()
+ udsp_print(before_stats_agent)
+
+ for i in range(0, PING_TIMES):
+ rc = main.lh.exec_ping(agent_nids[PING_NID_NUM])
+ if not rc:
+ udsp_print("ping failed")
+ return lutfrc(LUTF_TEST_FAIL)
+
+ after_stats_agent = agent.lh.get_net_stats()
+ udsp_print(after_stats_agent)
+
+ recv_count_before = {}
+ recv_count_after = {}
+ total_recv_count_before = 0
+ total_recv_count_after = 0
+ nid_num = 0
+ for nid in agent_nids:
+ recv_count_before[nid_num] = getNetStatNID(before_stats_agent, agent_nids, nid_num, 'recv_count')
+ total_recv_count_before += recv_count_before[nid_num]
+ recv_count_after[nid_num] = getNetStatNID(after_stats_agent, agent_nids, nid_num, 'recv_count')
+ total_recv_count_after += recv_count_after[nid_num]
+ nid_num += 1
+
+ """
+ for x in before_stats_main:
+ if (x['net type'] == LOCAL_NET):
+ for y in x['local NI(s)']:
+ if y['nid'] == main_nids[USE_NID_NUM]:
+ send_count_before = y['statistics']['send_count']
+ for x in after_stats_main:
+ if (x['net type'] == LOCAL_NET):
+ for y in x['local NI(s)']:
+ if y['nid'] == main_nids[USE_NID_NUM]:
+ send_count_after = y['statistics']['send_count']
+ if send_count_before < 0 or send_count_after < 0:
+ print("failed to parse net stats")
+ return lutfrc(LUTF_TEST_FAIL)
+
+ if send_count_after - send_count_before < PING_TIMES:
+ print("Unexpected number of sends: ", send_count_after - send_count_before)
+ return lutfrc(LUTF_TEST_FAIL)
+ """
+
+ udsp_print(recv_count_before, recv_count_after)
+
+ # Check
+ # 1) expect the total recv_count on the agent to be no less than the number of pings issued
+ # 2) expect the recv count on the preferred peer interface to increase by the number of pings issues
+ if (total_recv_count_after - total_recv_count_before) < PING_TIMES:
+ udsp_print("total recv count mismatch")
+ return lutfrc(LUTF_TEST_FAIL)
+
+ if abs(recv_count_after[PING_NID_NUM] - recv_count_before[PING_NID_NUM]) < PING_TIMES:
+ udsp_print("less than expected traffic on the prioritized peer nid")
+ return lutfrc(LUTF_TEST_FAIL)
+
+
+ for n in nodes:
+ n.lh.unconfigure_lnet()
+
+ return lutfrc(LUTF_TEST_PASS)
+ except Exception as e:
+ for n in nodes:
+ n.lh.uninit()
+ raise e
+
git://git.whamcloud.com / fs / lustre-release.git / commitdiff