From ea6a9513de57c412d2391b50d2e8721b9bde6f78 Mon Sep 17 00:00:00 2001 From: Amir Shehata Date: Fri, 26 Jun 2020 21:15:04 -0700 Subject: [PATCH] LU-10973 lnet: LUTF dynamic discovery test suite Add the dynamic discovery test suite to the LUTF test cases. Updated some of the infrastructure scripts with methods needed for the DD test cases Test-Parameters: @lnet Signed-off-by: Amir Shehata Change-Id: I0cfef4ae6f88b4deca12f1a3d5ef3291137a6c04 Reviewed-on: https://review.whamcloud.com/c/fs/lustre-release/+/39195 Tested-by: jenkins Tested-by: Maloo Reviewed-by: jsimmons Reviewed-by: Serguei Smirnov Reviewed-by: Oleg Drokin --- lustre/tests/lutf/Makefile.am | 1 + .../tests/suite_dynamic-discovery/callbacks.py | 6 ++ .../test_dd_UT-DD-CFG-0001.py | 54 ++++++++++ .../test_dd_UT-DD-CFG-0002.py | 53 ++++++++++ .../test_dd_UT-DD-CFG-0004.py | 60 +++++++++++ .../test_dd_UT-DD-CFG-0005.py | 39 +++++++ .../test_dd_UT-DD-CFG-0006.py | 54 ++++++++++ .../test_dd_UT-DD-EN-0001.py | 101 +++++++++++++++++++ .../test_dd_UT-DD-EN-0002.py | 112 +++++++++++++++++++++ 9 files changed, 480 insertions(+) create mode 100644 lustre/tests/lutf/python/tests/suite_dynamic-discovery/callbacks.py create mode 100644 lustre/tests/lutf/python/tests/suite_dynamic-discovery/test_dd_UT-DD-CFG-0001.py create mode 100644 lustre/tests/lutf/python/tests/suite_dynamic-discovery/test_dd_UT-DD-CFG-0002.py create mode 100644 lustre/tests/lutf/python/tests/suite_dynamic-discovery/test_dd_UT-DD-CFG-0004.py create mode 100644 lustre/tests/lutf/python/tests/suite_dynamic-discovery/test_dd_UT-DD-CFG-0005.py create mode 100644 lustre/tests/lutf/python/tests/suite_dynamic-discovery/test_dd_UT-DD-CFG-0006.py create mode 100644 lustre/tests/lutf/python/tests/suite_dynamic-discovery/test_dd_UT-DD-EN-0001.py create mode 100644 lustre/tests/lutf/python/tests/suite_dynamic-discovery/test_dd_UT-DD-EN-0002.py diff --git a/lustre/tests/lutf/Makefile.am b/lustre/tests/lutf/Makefile.am index deccccc..c3b600f 100644 --- a/lustre/tests/lutf/Makefile.am +++ b/lustre/tests/lutf/Makefile.am @@ -18,6 +18,7 @@ nobase_noinst_SCRIPTS += $(wildcard python/tests/*.py) nobase_noinst_SCRIPTS += $(wildcard python/tests-infra/*.py) nobase_noinst_SCRIPTS += $(wildcard python/tests/suite_dlc/*.py) nobase_noinst_SCRIPTS += $(wildcard python/tests/suite_samples/*.py) +nobase_noinst_SCRIPTS += $(wildcard python/tests/suite_dynamic-discovery/*.py) nobase_noinst_SCRIPTS += $(wildcard python/tests/suite_dynamic-nids/*.py) EXTRA_DIST=$(nobase_noinst_DATA) $(nobase_noinst_SCRIPTS) diff --git a/lustre/tests/lutf/python/tests/suite_dynamic-discovery/callbacks.py b/lustre/tests/lutf/python/tests/suite_dynamic-discovery/callbacks.py new file mode 100644 index 0000000..d02f7e2 --- /dev/null +++ b/lustre/tests/lutf/python/tests/suite_dynamic-discovery/callbacks.py @@ -0,0 +1,6 @@ +from lnet_cleanup import clean_lnet +from lustre_cleanup import clean_lustre + +def lutf_clean_setup(): + clean_lustre() + clean_lnet() diff --git a/lustre/tests/lutf/python/tests/suite_dynamic-discovery/test_dd_UT-DD-CFG-0001.py b/lustre/tests/lutf/python/tests/suite_dynamic-discovery/test_dd_UT-DD-CFG-0001.py new file mode 100644 index 0000000..9189599 --- /dev/null +++ b/lustre/tests/lutf/python/tests/suite_dynamic-discovery/test_dd_UT-DD-CFG-0001.py @@ -0,0 +1,54 @@ +""" +@PRIMARY: N/A +@PRIMARY_DESC: N/A +@SECONDARY: N/A +@DESIGN: N/A +@TESTCASE: 1. turn on LNet discovery +2. run lnetctl discover +3. ensure that peers are discovered +""" + +import os +import yaml +import lnetconfig +from lutf import agents, me +from lutf_basetest import * +from lnet import TheLNet +from lutf_exception import LUTFError +from lnet_helpers import LNetHelpers + +def run(): + la = agents.keys() + if len(la) < 2: + return lutfrc(LUTF_TEST_SKIP, "Not enough agents to run the test") + helpers = [] + try: + for i in range(0, len(la)): + t = LNetHelpers(target=la[i]) + intfs = t.get_available_devs() + t.configure_lnet() + t.configure_net('tcp', intfs) + t.set_discovery(1) + helpers.append(t) + + main = helpers[0] + for i in range(1, len(helpers)): + actual_nids = helpers[i].list_nids() + discovered_nids = main.discover(actual_nids[0]) + if len(actual_nids) != len(discovered_nids): + return lutfrc(LUTF_TEST_FAIL, "discovered NIDs are not correct", + discovered_nids=discovered_nids, actual_nids=actual_nids) + for nid in discovered_nids: + if nid not in actual_nids: + return lutfrc(LUTF_TEST_FAIL, "discovered NID is unknown", + discovered_nids=discovered_nids, actual_nids=actual_nids) + + for h in helpers: + h.unconfigure_lnet() + + return lutfrc(LUTF_TEST_PASS) + except Exception as e: + for h in helpers: + h.uninit() + raise e + diff --git a/lustre/tests/lutf/python/tests/suite_dynamic-discovery/test_dd_UT-DD-CFG-0002.py b/lustre/tests/lutf/python/tests/suite_dynamic-discovery/test_dd_UT-DD-CFG-0002.py new file mode 100644 index 0000000..4e54b44 --- /dev/null +++ b/lustre/tests/lutf/python/tests/suite_dynamic-discovery/test_dd_UT-DD-CFG-0002.py @@ -0,0 +1,53 @@ +""" +@PRIMARY: N/A +@PRIMARY_DESC: N/A +@SECONDARY: N/A +@DESIGN: N/A +@TESTCASE: 1. turn off LNet discovery +2. run lnetctl discover +3. ensure that peers are discovered +""" + +import os +import yaml +import lnetconfig +from lutf import agents, me +from lutf_basetest import * +from lnet import TheLNet +from lutf_exception import LUTFError +from lnet_helpers import LNetHelpers + +def run(): + la = agents.keys() + if len(la) < 2: + return lutfrc(LUTF_TEST_SKIP, "Not enough agents to run the test") + helpers = [] + try: + for i in range(0, len(la)): + t = LNetHelpers(target=la[i]) + intfs = t.get_available_devs() + t.configure_lnet() + t.configure_net('tcp', intfs) + t.set_discovery(0) + helpers.append(t) + + main = helpers[0] + for i in range(1, len(helpers)): + actual_nids = helpers[i].list_nids() + discovered_nids = main.discover(actual_nids[0]) + if len(discovered_nids) > 1: + return lutfrc(LUTF_TEST_FAIL, "discovered NIDs are not correct", + discovered_nids=discovered_nids, actual_nids=actual_nids) + if discovered_nids[0] != actual_nids[0]: + return lutfrc(LUTF_TEST_FAIL, "discovered NID is unknown", + discovered_nids=discovered_nids, actual_nids=actual_nids) + + for h in helpers: + h.unconfigure_lnet() + + return lutfrc(LUTF_TEST_PASS) + except Exception as e: + for h in helpers: + h.uninit() + raise e + diff --git a/lustre/tests/lutf/python/tests/suite_dynamic-discovery/test_dd_UT-DD-CFG-0004.py b/lustre/tests/lutf/python/tests/suite_dynamic-discovery/test_dd_UT-DD-CFG-0004.py new file mode 100644 index 0000000..0421a04 --- /dev/null +++ b/lustre/tests/lutf/python/tests/suite_dynamic-discovery/test_dd_UT-DD-CFG-0004.py @@ -0,0 +1,60 @@ +""" +@PRIMARY: N/A +@PRIMARY_DESC: N/A +@SECONDARY: N/A +@DESIGN: N/A +@TESTCASE: 1. Turn off discovery via YAML +2. Grab global configuration and check discovery is off +3. Turn on discovery via YAML +4. Grab global configuration and check discovery is on +""" + +import os +import yaml, random +import lnetconfig +from lutf import agents, me, lutf_tmp_dir +from lutf_basetest import * +from lnet import TheLNet +from lutf_exception import LUTFError +from lnet_helpers import LNetHelpers +from lutf_file import LutfFile + +def discovery_set_check(target, helper, value): + tmpFile = 'rtr'+str(random.getrandbits(32)) + '.yaml' + tmpFile = os.path.join(lutf_tmp_dir, tmpFile) + tmp = LutfFile(tmpFile, target=target) + cfg = 'global:\n'+ \ + ' discovery: %d\n' % value + tmp.open('w') + tmp.write(cfg) + tmp.close() + helper.api_yaml_cfg(tmp.get_full_path(), 1, delete=False) + tmp.remove() + cfg = helper.get_globals() + return cfg['global']['discovery'] == value, cfg + +def run(): + la = agents.keys() + if len(la) < 1: + return lutfrc(LUTF_TEST_SKIP, "Not enough agents to run the test") + t = LNetHelpers(target=la[0]) + try: + t.configure_lnet() + intfs = t.get_available_devs() + t.configure_net('tcp', intfs) + rc, cfg = discovery_set_check(la[0], t, 0) + if not rc: + return lutfrc(LUTF_TEST_FAIL, "Discovery wasn't turned off", + global_cfg=cfg) + + rc, cfg = discovery_set_check(la[0], t, 1) + if not rc: + return lutfrc(LUTF_TEST_FAIL, "Discovery wasn't turned on", + global_cfg=cfg) + + t.unconfigure_lnet() + + return lutfrc(LUTF_TEST_PASS) + except Exception as e: + t.uninit() + raise e diff --git a/lustre/tests/lutf/python/tests/suite_dynamic-discovery/test_dd_UT-DD-CFG-0005.py b/lustre/tests/lutf/python/tests/suite_dynamic-discovery/test_dd_UT-DD-CFG-0005.py new file mode 100644 index 0000000..d0748ff --- /dev/null +++ b/lustre/tests/lutf/python/tests/suite_dynamic-discovery/test_dd_UT-DD-CFG-0005.py @@ -0,0 +1,39 @@ +""" +@PRIMARY: N/A +@PRIMARY_DESC: N/A +@SECONDARY: N/A +@DESIGN: N/A +@TESTCASE: 1. set max_intf uing lnetct +2. Grab global configuration and check max_intf value +""" + +import os +import yaml, random +import lnetconfig +from lutf import agents, me +from lutf_basetest import * +from lnet import TheLNet +from lutf_exception import LUTFError +from lnet_helpers import LNetHelpers + +def run(): + la = agents.keys() + if len(la) < 1: + return lutfrc(LUTF_TEST_SKIP, "Not enough agents to run the test") + t = LNetHelpers(target=la[0]) + try: + t.configure_lnet() + intfs = t.get_available_devs() + t.configure_net('tcp', intfs) + t.set_max_intf(500) + cfg = t.get_globals() + if not cfg['global']['max_intf'] == 500: + return lutfrc(LUTF_TEST_FAIL, "max_intf wasn't set properly", + intended_value=500, actual_value=cfg['global']['max_intf']) + + t.unconfigure_lnet() + return lutfrc(LUTF_TEST_PASS) + except Exception as e: + t.uninit() + raise e + diff --git a/lustre/tests/lutf/python/tests/suite_dynamic-discovery/test_dd_UT-DD-CFG-0006.py b/lustre/tests/lutf/python/tests/suite_dynamic-discovery/test_dd_UT-DD-CFG-0006.py new file mode 100644 index 0000000..a3357ab --- /dev/null +++ b/lustre/tests/lutf/python/tests/suite_dynamic-discovery/test_dd_UT-DD-CFG-0006.py @@ -0,0 +1,54 @@ +""" +@PRIMARY: N/A +@PRIMARY_DESC: N/A +@SECONDARY: N/A +@DESIGN: N/A +@TESTCASE: 1. set max interfaces via YAML +2. Grab global configuration and check max interfaces is set properly +""" + +import os +import yaml, random +import lnetconfig +from lutf import agents, me, lutf_tmp_dir +from lutf_basetest import * +from lnet import TheLNet +from lutf_exception import LUTFError +from lnet_helpers import LNetHelpers +from lutf_file import LutfFile + +def set_check(target, helper, value): + tmpFile = 'rtr'+str(random.getrandbits(32)) + '.yaml' + tmpFile = os.path.join(lutf_tmp_dir, tmpFile) + tmp = LutfFile(tmpFile, target=target) + cfg = 'global:\n'+ \ + ' max_intf: %d\n' % value + tmp.open('w') + tmp.write(cfg) + tmp.close() + helper.api_yaml_cfg(tmp.get_full_path(), 1, delete=False) + tmp.remove() + cfg = helper.get_globals() + return cfg['global']['max_intf'] == value, cfg + +def run(): + la = agents.keys() + if len(la) < 1: + return lutfrc(LUTF_TEST_SKIP, "Not enough agents to run the test") + t = LNetHelpers(target=la[0]) + try: + t.configure_lnet() + intfs = t.get_available_devs() + t.configure_net('tcp', intfs) + rc, cfg = set_check(la[0], t, 500) + if not rc: + return lutfrc(LUTF_TEST_FAIL, "max interfaces wasn't set to desired value", + intended_value=500, global_cfg=cfg) + + t.unconfigure_lnet() + + return lutfrc(LUTF_TEST_PASS) + except Exception as e: + t.uninit() + raise e + diff --git a/lustre/tests/lutf/python/tests/suite_dynamic-discovery/test_dd_UT-DD-EN-0001.py b/lustre/tests/lutf/python/tests/suite_dynamic-discovery/test_dd_UT-DD-EN-0001.py new file mode 100644 index 0000000..7d4bb47 --- /dev/null +++ b/lustre/tests/lutf/python/tests/suite_dynamic-discovery/test_dd_UT-DD-EN-0001.py @@ -0,0 +1,101 @@ +""" +@PRIMARY: N/A +@PRIMARY_DESC: N/A +@SECONDARY: N/A +@DESIGN: N/A +@TESTCASE: - MR Node with more than two interface, P1..Pn +- MR Peers with more than one interface +- Ping all the peers from the node +- Verify that node sees different peers per NID +- Discover all the peers' first NID from the node +- Verify that node sees one MR peer with all its NIDS +""" + +import os +import yaml, random +import lnetconfig +from lutf import agents, me +from lutf_basetest import * +from lnet import TheLNet +from lutf_exception import LUTFError +from lnet_helpers import LNetHelpers +from lutf_file import LutfFile + +def run(): + la = agents.keys() + if len(la) < 2: + return lutfrc(LUTF_TEST_SKIP, "Not enough agents to run the test") + helpers = [] + try: + # configure all the nodes + for i in range(0, len(la)): + t = LNetHelpers(target=la[i]) + intfs = t.get_available_devs() + if len(intfs) < 2: + return lutfrc(LUTF_TEST_SKIP, "Not enough interfaces") + t.configure_lnet() + t.configure_net('tcp', intfs) + t.set_discovery(1) + helpers.append(t) + + # ping all the nodes from main + main = helpers[0] + main_prim_nid = main.list_nids()[0] + total_nids = 0 + for i in range(1, len(helpers)): + actual_nids = helpers[i].list_nids() + total_nids += len(actual_nids) + for nid in actual_nids: + if len(main.ping(nid)) == 0: + return lutfrc(LUTF_TEST_FAIL, "unable to ping" , + target=nid) + + # check that the peers created + peers = main.get_peers() + if len(peers['peer']) != total_nids: + return lutfrc(LUTF_TEST_FAIL, "1. unexpected number of peers" , + peers=peers, expected_num_peers=len(peers), actual_num_peers=total_nids) + prim_nids = [] + for peer in peers['peer']: + prim_nids.append(peer['primary nid']) + for i in range(1, len(helpers)): + actual_nids = helpers[i].list_nids() + for nid in actual_nids: + if not nid in prim_nids: + return lutfrc(LUTF_TEST_FAIL, "unexpected peer set" , + actual_nids=actual_nids, prim_nids=prim_nids) + + # discover all the peers from main + for i in range(1, len(helpers)): + actual_nids = helpers[i].list_nids() + if len(main.discover(actual_nids[0])) == 0: + return lutfrc(LUTF_TEST_FAIL, "unable to discover" , + target=actual_nids[0]) + + # make sure the peers is what we'd expect + peers = main.get_peers() + if len(peers['peer']) != len(helpers) - 1: + return lutfrc(LUTF_TEST_FAIL, "2. unexpected number of peers" , + peers=peers, expected_num_peers=len(peers), actual_num_peers=len(helpers)-1) + for i in range(1, len(helpers)): + actual_nids = helpers[i].list_nids() + for peer in peers['peer']: + if peer['primary nid'] == actual_nids[0] and \ + len(peer['peer ni']) != len(actual_nids): + return lutfrc(LUTF_TEST_FAIL, "3. unexpected number of peers", + peers=peers, peer_nids=actual_nids) + elif peer['primary nid'] == actual_nids[0]: + for ni in peer['peer ni']: + if not ni['nid'] in actual_nids: + return lutfrc(LUTF_TEST_FAIL, "unexpected peer nid", + unkown_nid=ni['nid'], actual_nids=actual_nids) + + for h in helpers: + h.unconfigure_lnet() + + return lutfrc(LUTF_TEST_PASS) + except Exception as e: + for h in helpers: + h.uninit() + raise e + diff --git a/lustre/tests/lutf/python/tests/suite_dynamic-discovery/test_dd_UT-DD-EN-0002.py b/lustre/tests/lutf/python/tests/suite_dynamic-discovery/test_dd_UT-DD-EN-0002.py new file mode 100644 index 0000000..299f531 --- /dev/null +++ b/lustre/tests/lutf/python/tests/suite_dynamic-discovery/test_dd_UT-DD-EN-0002.py @@ -0,0 +1,112 @@ +""" +@PRIMARY: N/A +@PRIMARY_DESC: N/A +@SECONDARY: N/A +@DESIGN: N/A +@TESTCASE: - MR Node with more than two interface, P1..Pn +- MR Peers with more than one interface +- Ping all the peers from the node +- Verify that node sees different peers per NID +- Discover all the peers' second from the node +- Verify that node sees one MR peer with all its NIDS +""" + +import os +import yaml, random, logging +import lnetconfig +from lutf import agents, me +from lutf_basetest import * +from lnet import TheLNet +from lutf_exception import LUTFError +from lnet_helpers import LNetHelpers +from lutf_file import LutfFile +import lustre_logs + +def run(): + la = agents.keys() + if len(la) < 2: + return lutfrc(LUTF_TEST_SKIP, "Not enough agents to run the test") + helpers = [] + logs = None + try: + # configure all the nodes + for i in range(0, len(la)): + t = LNetHelpers(target=la[i]) + intfs = t.get_available_devs() + if len(intfs) < 2: + return lutfrc(LUTF_TEST_SKIP, "Not enough interfaces") + t.configure_lnet() + t.configure_net('tcp', intfs) + t.set_discovery(1) + helpers.append(t) + + logs = lustre_logs.LustreLog(target=la[0]) + logs.set_level('net') + logs.start() + + # ping all the nodes from main + main = helpers[0] + main_prim_nid = main.list_nids()[0] + total_nids = 0 + for i in range(1, len(helpers)): + actual_nids = helpers[i].list_nids() + total_nids += len(actual_nids) + for nid in actual_nids: + ping_cfg = main.ping(nid) + logging.debug(str(ping_cfg)) + if len(ping_cfg) == 0: + return lutfrc(LUTF_TEST_FAIL, "unable to ping" , + target=nid) + + # check that the peers created + peers = main.get_peers() + if len(peers['peer']) != total_nids: + return lutfrc(LUTF_TEST_FAIL, "1. unexpected number of peers" , + peers=peers, expected_num_peers=len(peers), actual_num_peers=total_nids) + prim_nids = [] + for peer in peers['peer']: + prim_nids.append(peer['primary nid']) + for i in range(1, len(helpers)): + actual_nids = helpers[i].list_nids() + for nid in actual_nids: + if not nid in prim_nids: + return lutfrc(LUTF_TEST_FAIL, "unexpected peer set" , + actual_nids=actual_nids, prim_nids=prim_nids) + + # discover all the peers from main + for i in range(1, len(helpers)): + actual_nids = helpers[i].list_nids() + print("attempting to discover %s from %s" % (actual_nids[1], main_prim_nid)) + if len(main.discover(actual_nids[1])) == 0: + return lutfrc(LUTF_TEST_FAIL, "unable to discover" , + target=actual_nids[0]) + + # make sure the peers is what we'd expect + peers = main.get_peers() + if len(peers['peer']) != len(helpers) - 1: + return lutfrc(LUTF_TEST_FAIL, "2. unexpected number of peers" , + peers=peers, expected_num_peers=len(peers), actual_num_peers=len(helpers)-1) + for i in range(1, len(helpers)): + actual_nids = helpers[i].list_nids() + for peer in peers['peer']: + if peer['primary nid'] == actual_nids[0] and \ + len(peer['peer ni']) != len(actual_nids): + return lutfrc(LUTF_TEST_FAIL, "3. unexpected number of peers", + peers=peers, peer_nids=actual_nids) + elif peer['primary nid'] == actual_nids[0]: + for ni in peer['peer ni']: + if not ni['nid'] in actual_nids: + return lutfrc(LUTF_TEST_FAIL, "unexpected peer nid", + unkown_nid=ni['nid'], actual_nids=actual_nids) + + for h in helpers: + h.unconfigure_lnet() + + logs.stop('test_dd_UT-DD-EN-0002.log') + return lutfrc(LUTF_TEST_PASS) + except Exception as e: + logs.stop('test_dd_UT-DD-EN-0002.log') + for h in helpers: + h.uninit() + raise e + -- 1.8.3.1