lustre/tests/lutf/python/tests-infra/lnet_helpers.py

   1 import os, re
   2 import yaml, ast, psutil
   3 import lnetconfig, logging
   4 from lutf import agents, me
   5 from lutf_basetest import BaseTest, lutfrc
   6 from lnet import TheLNet
   7 from lutf_exception import LUTFError
   8 from lutf_cmd import lutf_exec_local_cmd
   9 from utility_paths import get_lnetctl, CAT, MAN
  10
  11 LNET_NRB_TINY_MIN = 512
  12 LNET_NRB_TINY = LNET_NRB_TINY_MIN * 4
  13 LNET_NRB_SMALL_MIN = 4096
  14 LNET_NRB_SMALL = LNET_NRB_SMALL_MIN * 4
  15 LNET_NRB_LARGE_MIN = 256
  16 LNET_NRB_LARGE = LNET_NRB_LARGE_MIN * 4
  17
  18 class LNetHelpers(BaseTest):
  19         def __init__(self, script=os.path.abspath(__file__),
  20                      target=None, exceptions=True):
  21                 super().__init__(script, target=target)
  22                 self.exceptions = exceptions
  23                 self.__nid = None
  24                 logging.debug('LNetHelpers: %s == %s' % (me.name, target))
  25                 if not target or me.name == target:
  26                         logging.debug('Initializing LNetHelper')
  27                         rc = lnetconfig.lustre_lnet_config_lib_init()
  28                         if (rc != lnetconfig.LUSTRE_CFG_RC_NO_ERR):
  29                                 raise LUTFError("Failed to initialize the liblnetconfig library")
  30
  31         def __del__(self):
  32                 lnetconfig.lustre_lnet_config_lib_uninit()
  33                 super().__del__()
  34
  35         def uninit(self):
  36                 logging.debug('uninit: Uninitializing LNetHelper')
  37                 lnetconfig.lustre_lnet_config_lib_uninit()
  38
  39         def set_exception(self, exception):
  40                 self.exceptions = exception
  41
  42         def get_mem_info(self):
  43                 return psutil.virtual_memory()
  44
  45         def configure_lnet(self):
  46                 L = TheLNet()
  47
  48         def unconfigure_lnet(self):
  49                 L = TheLNet()
  50                 L.unconfigure()
  51
  52         def get_num_cpus(self):
  53                 return me.get_num_cpus()
  54
  55         def configure_net(self, net, pintfs=None, pycfg=None):
  56                 L = TheLNet()
  57                 if pycfg:
  58                         L.configure_yaml(pycfg)
  59                         L1 = TheLNet()
  60                         L1.update()
  61                         if not L1.nets == L.nets and not net in L1.nets:
  62                                         if self.exceptions:
  63                                                 raise LUTFError("LNet %s configuration failed" % net)
  64                                         return False, net
  65                         return True, None
  66
  67                 if pintfs:
  68                         intfs = pintfs
  69                 else:
  70                         intfs = self.get_available_devs()
  71                 if len(intfs) == 0:
  72                         if self.exceptions:
  73                                 raise LUTFError("node doesn't have any interfaces")
  74                         return False, net
  75                 # configure the first interface
  76                 nets = []
  77                 for intf in intfs:
  78                         logging.debug("configuring: %s" % intf)
  79                         nets.append(L.make_net(intf))
  80                 logging.debug(str(nets))
  81                 L.nets[net] = nets
  82                 L.configure()
  83                 L1 = TheLNet()
  84                 L1.update()
  85                 if not L1.nets == L.nets and not net in L1.nets:
  86                                 if self.exceptions:
  87                                         raise LUTFError("LNet %s configuration failed" % net)
  88                                 return False, net
  89                 net_show = L1.get_net()
  90                 for n in net_show['net']:
  91                         if n['net type'] == 'lo' or n['net type'] != net:
  92                                 continue
  93                         self.__nid = n['local NI(s)'][0]['nid']
  94                         break
  95
  96                 logging.debug(self.__nid)
  97                 return True, None
  98
  99         def unconfigure_net(self, net):
 100                 if not net:
 101                         return True
 102                 L = TheLNet()
 103                 L.update()
 104                 L.unconfigure_net(net)
 105                 nets = L.export_nets(op=None)
 106                 for n in nets['net']:
 107                         if net == n['net type']:
 108                                 if self.exceptions:
 109                                         raise LUTFError("net %s was not unconfigure properly" % net)
 110                                 return False
 111                 return True
 112
 113         def get_nid(self):
 114                 return self.__nid
 115
 116         def list_nids(self):
 117                 L = TheLNet()
 118                 nids = []
 119                 net_show = L.get_net()
 120                 for n in net_show['net']:
 121                         if n['net type'] == 'lo':
 122                                 continue
 123                         for nid in n['local NI(s)']:
 124                                 nids.append(nid['nid'])
 125                 return nids
 126
 127         def get_available_devs(self):
 128                 intfs = me.list_intfs()
 129                 return list(intfs['interfaces'].keys())
 130
 131         def get_available_intfs(self):
 132                 return me.list_intfs()
 133
 134         def api_config_ni(self, net, device_list=[], global_cpts=None, ip2nets=None,
 135                           peer_credits=128, peer_timeout=180, peer_buffer_credits=0,
 136                           credits=256):
 137                 tunables = lnetconfig.lnet_ioctl_config_lnd_tunables()
 138                 tunables.lt_cmn.lct_peer_timeout = peer_timeout
 139                 tunables.lt_cmn.lct_peer_tx_credits = peer_credits;
 140                 tunables.lt_cmn.lct_max_tx_credits = credits;
 141                 tunables.lt_cmn.lct_peer_rtr_credits = peer_buffer_credits
 142
 143                 if (ip2nets == None):
 144                         nwd = lnetconfig.lnet_dlc_network_descr()
 145                         lnetconfig.lustre_lnet_init_nw_descr(nwd)
 146                         nwd.nw_id = lnetconfig.libcfs_str2net(net)
 147                         devices_str = ''
 148                         for device in device_list[:-1]:
 149                                 devices_str += device + ','
 150                         if len(device_list) > 0:
 151                                 devices_str += device_list[-1]
 152                         rc = lnetconfig.lustre_lnet_parse_interfaces(devices_str, nwd)
 153                         if (rc != lnetconfig.LUSTRE_CFG_RC_NO_ERR):
 154                                 if self.exceptions:
 155                                         raise LUTFError("Failed to parse interfaces %d" % rc)
 156                                 return False, [rc, net, device_list, global_cpts, ip2nets]
 157                 else:
 158                         nwd = None
 159                 g_cpts = None
 160                 if global_cpts != None and type(global_cpts) is list:
 161                         rc, g_cpts = lnetconfig.cfs_expr_list_parse(str(global_cpts), len(str(global_cpts)), 0, lnetconfig.UINT_MAX)
 162                         if rc != 0:
 163                                 if self.exceptions:
 164                                         raise LUTFError("Failed to parse global_cpts")
 165                                 return False, [rc, net, device_list, global_cpts, ip2nets]
 166                 else:
 167                         g_cpts = None
 168                 rc, yaml_err = lnetconfig.lustre_lnet_config_ni(nwd, g_cpts, ip2nets, tunables, -1)
 169                 #Freeing the g_cpts causes a segmentation fault
 170                 #if g_cpts:
 171                 #       lnetconfig.cfs_expr_list_free(g_cpts)
 172                 self.cYAML_free(yaml_err)
 173                 if (rc != lnetconfig.LUSTRE_CFG_RC_NO_ERR):
 174                         if self.exceptions:
 175                                 raise LUTFError("Failed to config ni %s:%s:%s:%s" %
 176                                                 (str(net), str(device_list),
 177                                                  str(global_cpts), str(ip2nets)))
 178                         return False, [rc, net, device_list, global_cpts, ip2nets]
 179                 return True, [rc, net, device_list, global_cpts, ip2nets]
 180
 181         def api_del_ni(self, net, device_list):
 182                 nwd = lnetconfig.lnet_dlc_network_descr()
 183                 lnetconfig.lustre_lnet_init_nw_descr(nwd)
 184                 nwd.nw_id = lnetconfig.libcfs_str2net(net)
 185                 devices_str = ''
 186                 for device in device_list[:-1]:
 187                         devices_str += device + ','
 188                 if len(device_list) > 0:
 189                         devices_str += device_list[-1]
 190                 rc = lnetconfig.lustre_lnet_parse_interfaces(devices_str, nwd)
 191                 if (rc != lnetconfig.LUSTRE_CFG_RC_NO_ERR):
 192                         if self.exceptions:
 193                                 raise LUTFError("Failed to parse interfaces")
 194                         return False, [rc, net, device_list]
 195                 rc, yaml_err = lnetconfig.lustre_lnet_del_ni(nwd, -1)
 196                 self.cYAML_free(yaml_err)
 197                 if (rc != lnetconfig.LUSTRE_CFG_RC_NO_ERR):
 198                         if self.exceptions:
 199                                 raise LUTFError("Failed to del ni")
 200                         return False, [rc, net, device_list]
 201                 return True, [rc, net, device_list]
 202
 203         def api_check_ni(self, net = None, num = 1, global_cpts=None, iname=None, peer_credits=128,
 204                          peer_timeout=180, peer_buffer_credits=0, credits=256):
 205                 rc, yaml_show, yaml_err = lnetconfig.lustre_lnet_show_net(net, 0, -1, False)
 206                 err = lnetconfig.cYAML_dump(yaml_err)
 207                 self.cYAML_free(yaml_err)
 208                 if (rc != lnetconfig.LUSTRE_CFG_RC_NO_ERR):
 209                         self.cYAML_free(yaml_show)
 210                         if self.exceptions:
 211                                 raise LUTFError("Failed to show NIs")
 212                         return False, [rc, num, err]
 213                 else:
 214                         # basic check to make sure there are the right number of nets
 215                         # configured
 216                         show = lnetconfig.cYAML_dump(yaml_show)
 217                         self.cYAML_free(yaml_show)
 218                         pyy = yaml.load(show, Loader=yaml.FullLoader)
 219                         count = len(pyy['net'][0]['local NI(s)'])
 220                         if pyy['net'][0]['net type'] != net or count != num:
 221                                 if self.exceptions:
 222                                         raise LUTFError("Show doesn't match %d != %d\n%s" % (count, num, show))
 223                                 return False, [rc, count, num, show]
 224                         # Check the tunables match
 225                         for n in pyy['net']:
 226                                 if n['net type'] == net:
 227                                         for i in n['local NI(s)']:
 228                                                 if iname and iname in list(i['interfaces'].values()):
 229                                                         if i['tunables']['peer_timeout'] != peer_timeout or \
 230                                                            i['tunables']['peer_credits'] != peer_credits or \
 231                                                            i['tunables']['peer_buffer_credits'] != peer_buffer_credits or \
 232                                                            i['tunables']['credits'] != credits or \
 233                                                            (global_cpts and ast.literal_eval(i['CPT']) != global_cpts):
 234                                                                 if self.exceptions:
 235                                                                         raise LUTFError("configured ni tunables don't match")
 236                                                                 return False, [rc, count, num, show]
 237                         return True, [rc, show]
 238
 239         def api_configure_route(self, rnet=None, gw=None, hop=-1, prio=0, sen=1):
 240                 rc, yaml_err = lnetconfig.lustre_lnet_config_route(rnet, gw, hop, prio, sen, -1)
 241                 self.cYAML_free(yaml_err)
 242                 if rc != lnetconfig.LUSTRE_CFG_RC_NO_ERR:
 243                         if self.exceptions:
 244                                 raise LUTFError("failed to configure route. rc=%s, rnet=%s gw=%s hop=%s pio=%s sen=%s" % (str(rc), str(rnet), str(gw), str(hop), str(prio), str(sen)))
 245                         return False, [rnet, gw, hop, prio, sen]
 246
 247                 # check the route was configured as expected
 248                 L1 = TheLNet()
 249                 L1.update()
 250                 L1.export(op=logging.debug)
 251                 if rnet not in L1.routes.keys():
 252                         if self.exceptions:
 253                                 raise LUTFError("failed to configure remote net %s" % rnet)
 254                         return False, [rnet, gw, hop, prio, sen]
 255                 route = L1.routes[rnet].get()
 256                 if len(route) == 0:
 257                         if self.exceptions:
 258                                 raise LUTFError("failed to configure remote net %s" % rnet)
 259                         return False, [rnet, gw, hop, prio, sen]
 260                 logging.debug(yaml.dump({'original': [rnet, gw, hop, prio, sen], 'configured': route[0]}))
 261                 if route[0]['gateway'] != gw or \
 262                    route[0]['hop'] != hop or \
 263                    route[0]['priority'] != prio or \
 264                    (route[0]['health_sensitivity'] != sen and sen != -1) or \
 265                    (route[0]['health_sensitivity'] != 1 and sen == -1):
 266                         if self.exceptions:
 267                                 raise LUTFError("Configured route is not expected", {'original': [rnet, gw, hop, prio, sen], 'configured': route[0]})
 268                         return False, [[rnet, gw, hop, prio, sen], route[0]]
 269                 return True, [rnet, gw, hop, prio, sen]
 270
 271         def cYAML_count(self, blk):
 272                 if (blk == None):
 273                         return 0
 274                 yy = yaml.load(blk, Loader=yaml.FullLoader)
 275                 logging.debug(str(yy))
 276                 return len(yy[next(iter(yy))])
 277
 278         def cYAML_free(self, cyaml):
 279                 if (cyaml):
 280                         lnetconfig.cYAML_free_tree(cyaml.cy_child)
 281
 282         def api_del_route(self, rnet=None, gw=None):
 283                 # delete route but missing net
 284                 rc, yaml_err = lnetconfig.lustre_lnet_del_route(rnet, gw, -1)
 285                 self.cYAML_free(yaml_err)
 286                 if (rc == lnetconfig.LUSTRE_CFG_RC_MISSING_PARAM):
 287                         if self.exceptions:
 288                                 raise LUTFError("Failed to delete route")
 289                         return False, [rc, rnet, gw]
 290                 return True, [rc, rnet, gw]
 291
 292         def api_check_route(self, num, network = None, gateway = None, hop = -1, prio = -1):
 293                 logging.debug("show route: %s %s %s %s" % (str(network), str(gateway), str(hop), str(prio)))
 294                 rc, yaml_show, yaml_err = lnetconfig.lustre_lnet_show_route(network, gateway, hop, prio, 1, -1, False)
 295                 logging.debug("show_route: rc = %s" % (str(rc)))
 296                 self.cYAML_free(yaml_err)
 297                 if (rc == lnetconfig.LUSTRE_CFG_RC_NO_ERR):
 298                         if yaml_show is None:
 299                                 count = 0
 300                         else:
 301                                 show = lnetconfig.cYAML_dump(yaml_show)
 302                                 count = self.cYAML_count(show)
 303                                 logging.debug("%s Routes detected" % (str(count)))
 304                                 logging.debug(show)
 305                                 # free the memory (This is a call into the C code)
 306                                 self.cYAML_free(yaml_show)
 307                         if (count != num):
 308                                 if self.exceptions:
 309                                         raise LUTFError("%d doesn't match number of configured routes %d" % (count, num))
 310                                 return False, [count, num, network, gateway, hop, prio]
 311                 elif num > 0:
 312                         self.cYAML_free(yaml_show)
 313                         if self.exceptions:
 314                                 raise LUTFError("No routes configured")
 315                         return False, [count, num, network, gateway, hop, prio]
 316                 return True, None
 317
 318         def api_config_rtr_buffers(self, tiny=-1, small=-1, large=-1):
 319                 rc, yaml_err = lnetconfig.lustre_lnet_config_buffers(tiny, small, large, -1)
 320                 err = lnetconfig.cYAML_dump(yaml_err)
 321                 self.cYAML_free(yaml_err)
 322                 if (rc != lnetconfig.LUSTRE_CFG_RC_NO_ERR):
 323                         if self.exceptions:
 324                                 raise LUTFError("Failed to configure the buffers")
 325                         return False, [rc, err]
 326                 return True, None
 327
 328         def api_set_routing(self, enable):
 329                 rc, yaml_err = lnetconfig.lustre_lnet_enable_routing(enable, -1)
 330                 logging.debug("rc = %d" % rc)
 331                 err = lnetconfig.cYAML_dump(yaml_err)
 332                 logging.debug(err)
 333                 self.cYAML_free(yaml_err)
 334                 if (rc != lnetconfig.LUSTRE_CFG_RC_NO_ERR):
 335                         if self.exceptions:
 336                                 raise LUTFError("Failed to set routing")
 337                         return False, [rc, err]
 338                 return True, None
 339
 340         def api_check_rtr_buffers(self, tiny=LNET_NRB_TINY, small=LNET_NRB_SMALL, large=LNET_NRB_LARGE):
 341                 rc, yaml_show, yaml_err = lnetconfig.lustre_lnet_show_routing(-1, False)
 342                 err = lnetconfig.cYAML_dump(yaml_err)
 343                 show = lnetconfig.cYAML_dump(yaml_show)
 344                 self.cYAML_free(yaml_err)
 345                 self.cYAML_free(yaml_show)
 346                 if rc != lnetconfig.LUSTRE_CFG_RC_NO_ERR:
 347                         if self.exceptions:
 348                                 raise LUTFError("Couldn't configure router buffers: %d, %d, %d" % (tiny, small, large))
 349                         return False, [rc, err]
 350                 pyshow = yaml.load(show, Loader=yaml.FullLoader)
 351                 if pyshow['buffers']['tiny'] != tiny or \
 352                    pyshow['buffers']['small'] != small or \
 353                    pyshow['buffers']['large'] != large:
 354                         if self.exceptions:
 355                                 raise LUTFError("rtr buffer values configured do not match %d != %d, %d != %d, %d != %d" %
 356                                     (pyshow['buffers']['tiny'], tiny, pyshow['buffers']['small'], small,
 357                                      pyshow['buffers']['large'], large))
 358                         return False, [rc, pyshow]
 359                 return True, [rc, pyshow]
 360
 361         def replace_sep(self, nidstr, old, new):
 362                 bracket = 0
 363                 for i in range(0, len(nidstr)):
 364                         if nidstr[i] == '[':
 365                                 bracket += 1
 366                         elif nidstr[i] == ']':
 367                                 bracket -= 1
 368                         elif nidstr[i] == old and bracket == 0:
 369                                 tmp = list(nidstr)
 370                                 tmp[i] = new
 371                                 nidstr = "".join(tmp)
 372                 return nidstr
 373
 374         def api_verify_peer(self, prim_nid, nids):
 375                 rc, yaml_show, yaml_err = lnetconfig.lustre_lnet_show_peer(prim_nid, 4, -1, False)
 376                 err = lnetconfig.cYAML_dump(yaml_err)
 377                 self.cYAML_free(yaml_err)
 378                 try:
 379                         show = lnetconfig.cYAML_dump(yaml_show)
 380                         self.cYAML_free(yaml_show)
 381                 except:
 382                         show = ''
 383                 if rc != lnetconfig.LUSTRE_CFG_RC_NO_ERR:
 384                         if self.exceptions:
 385                                 raise LUTFError("Couldn't show peer %s" % (prim_nid))
 386                         pyerr = yaml.load(err, Loader=yaml.FullLoader)
 387                         return False, [rc, pyerr]
 388                 nidlist = []
 389                 if nids:
 390                         nids = self.replace_sep(nids, ',', ' ')
 391                         nidl = lnetconfig.lutf_parse_nidlist(nids, len(nids),
 392                                 lnetconfig.LNET_MAX_NIDS_PER_PEER)
 393                         for n in nidl:
 394                                 nidlist.append(lnetconfig.lutf_nid2str(n))
 395                 pyshow = yaml.load(show, Loader=yaml.FullLoader)
 396                 if prim_nid:
 397                         nidlist.insert(0, prim_nid)
 398                 nids_found = []
 399                 for nid in nidlist:
 400                         for peerni in pyshow['peer'][0]['peer ni']:
 401                                 if peerni['nid'] == nid:
 402                                         nids_found.append(nid)
 403                                         break;
 404                 return True, [rc, nidlist, nids_found]
 405
 406         def api_config_peer(self, prim_nid=None, nids=None, is_mr=True):
 407                 rc, yaml_err = lnetconfig.lustre_lnet_modify_peer(prim_nid, nids, is_mr, lnetconfig.LNETCTL_ADD_CMD, -1)
 408                 if rc != lnetconfig.LUSTRE_CFG_RC_NO_ERR:
 409                         err = lnetconfig.cYAML_dump(yaml_err)
 410                         self.cYAML_free(yaml_err)
 411                         if self.exceptions:
 412                                 raise LUTFError("Couldn't configure peer %s: %s" % (prim_nid+' '+nids, err))
 413                         pyerr = yaml.load(err, Loader=yaml.FullLoader)
 414                         return False, [rc, pyerr]
 415                 # verify config
 416                 if prim_nid:
 417                         key = prim_nid
 418                 else:
 419                         mynids = self.replace_sep(nids, ',', ' ')
 420                         nidl = lnetconfig.lutf_parse_nidlist(mynids, len(mynids),
 421                             lnetconfig.LNET_MAX_NIDS_PER_PEER)
 422                         key = lnetconfig.lutf_nid2str(nidl[0])
 423                 rc, info = self.api_verify_peer(key, nids)
 424                 if rc == False:
 425                         if self.exceptions:
 426                                 raise LUTFError("Couldn't verify peer " + key)
 427                         return rc, info
 428                 # expect that the NIDs found is the same as the ones we're looking for
 429                 if info[1] !=  info[2]:
 430                         if self.exceptions:
 431                                 raise LUTFError("Configured nids are in correct" + key)
 432                         return False, info
 433                 return True, None
 434
 435         def api_del_peer(self, prim_nid=None, nids=None, all=True):
 436                 rc, yaml_err = lnetconfig.lustre_lnet_modify_peer(prim_nid, nids, False,
 437                         lnetconfig.LNETCTL_DEL_CMD, -1)
 438                 if rc != lnetconfig.LUSTRE_CFG_RC_NO_ERR:
 439                         err = lnetconfig.cYAML_dump(yaml_err)
 440                         if self.exceptions:
 441                                 raise LUTFError("Couldn't del peer %s:%d" % (prim_nid, rc))
 442                         pyerr = yaml.load(err, Loader=yaml.FullLoader)
 443                         return False, [rc, pyerr]
 444                 self.cYAML_free(yaml_err)
 445                 # verify config
 446                 try:
 447                         rc, info = self.api_verify_peer(prim_nid, nids)
 448                         if rc and not all:
 449                                 # verify that all the peers indicated are gone, excluding the primary nid
 450                                 nidlist = info[1]
 451                                 foundlist = info[2]
 452                                 for n in nidlist[1:]:
 453                                         if n in foundlist:
 454                                                 if self.exceptions:
 455                                                         raise LUTFError("nid %s wasn't deleted" % (n))
 456                                                 return False, info
 457                         elif rc:
 458                                 if self.exceptions:
 459                                         raise LUTFError("Peer %s was not deleted" % (prim_nid))
 460                                 return False, [rc, info]
 461                 except:
 462                         if not all:
 463                                 if self.exceptions:
 464                                         raise LUTFError("Peer %s was not deleted properly" % (prim_nid))
 465                                 return False, [rc, info]
 466                         pass
 467                 return True, None
 468
 469         def api_yaml_cfg(self, yaml_file, count, del_count=0, delete = True):
 470                 logging.debug("configuring yaml file %s" % yaml_file)
 471                 rc, yaml_err = lnetconfig.lustre_yaml_config(yaml_file)
 472                 err = lnetconfig.cYAML_dump(yaml_err)
 473                 self.cYAML_free(yaml_err)
 474                 if (rc != lnetconfig.LUSTRE_CFG_RC_NO_ERR):
 475                         logging.debug("config failed with: %d \n%s" % (rc, err))
 476                         if self.exceptions:
 477                                 raise LUTFError("configuration failed")
 478                         return False, [rc, err]
 479
 480                 rc, yaml_show, yaml_err = lnetconfig.lustre_yaml_show(yaml_file)
 481                 #rc, yaml_show, yaml_err = lnetconfig.lustre_lnet_show_net(None, 0, -1, False)
 482                 self.cYAML_free(yaml_err)
 483                 if (rc != lnetconfig.LUSTRE_CFG_RC_NO_ERR):
 484                         logging.debug(lnetconfig.cYAML_dump(yaml_show))
 485                         err = lnetconfig.cYAML_dump(yaml_err)
 486                         return False, [rc, err]
 487
 488                 show = lnetconfig.cYAML_dump(yaml_show)
 489                 pyy = yaml.load(show, Loader=yaml.FullLoader)
 490                 show_count = 0
 491                 for k, v in pyy.items():
 492                         logging.debug("key = %s, value = %s, len = %d" % (str(k), str(v), len(v)))
 493                         show_count += len(v)
 494                 #show_count = self.cYAML_count(show)
 495                 logging.debug("show count = %d\n%s" % (show_count, show))
 496
 497                 # verify the show through the count only
 498                 self.cYAML_free(yaml_show)
 499                 if (show_count != count):
 500                         error = "show count doesn't match. %d != %d\n%s" % (show_count, count, show)
 501                         logging.debug(error)
 502                         if self.exceptions:
 503                                 raise LUTFError(error)
 504                         return False, [rc, show]
 505
 506                 if (delete == True):
 507                         logging.debug("deleting yaml file: %s" % yaml_file)
 508                         rc, yaml_err = lnetconfig.lustre_yaml_del(yaml_file)
 509                         err = lnetconfig.cYAML_dump(yaml_err)
 510                         self.cYAML_free(yaml_err)
 511                         if (rc != lnetconfig.LUSTRE_CFG_RC_NO_ERR):
 512                                 if self.exceptions:
 513                                         raise LUTFError("configuration failed")
 514                                 return False, [rc, err]
 515
 516                         logging.debug("showing after deleting yaml file: %s" % yaml_file)
 517                         rc, yaml_show, yaml_err = lnetconfig.lustre_yaml_show(yaml_file)
 518                         err = lnetconfig.cYAML_dump(yaml_err)
 519                         self.cYAML_free(yaml_err)
 520                         if (rc != lnetconfig.LUSTRE_CFG_RC_NO_ERR):
 521                                 if self.exceptions:
 522                                         raise LUTFError("configuration failed")
 523                                 return False, [rc, err]
 524
 525                         # verify the show through the count only
 526                         logging.debug("yaml_show type is %s" % str(type(yaml_show)))
 527                         # handle two cases:
 528                         #  yaml_show is NULL or an empty tree
 529                         if yaml_show == None:
 530                                 return True, None
 531                         show = lnetconfig.cYAML_dump(yaml_show)
 532                         pyy = yaml.load(show, Loader=yaml.FullLoader)
 533                         show_count = 0
 534                         for k, v in pyy.items():
 535                                 show_count += len(v)
 536                         self.cYAML_free(yaml_show)
 537                         if (show_count != del_count):
 538                                 error = "show count doesn't match. %d != %d\n%s" % (show_count, del_count, show)
 539                                 logging.debug(error)
 540                                 if self.exceptions:
 541                                         raise LUTFError(error)
 542                                 return False, [rc, show]
 543
 544                 return True, None
 545
 546         def import_config(self, data):
 547                 L = TheLNet()
 548                 L.import_config(data)
 549
 550         def import_del(self, data):
 551                 L = TheLNet()
 552                 L.import_del(data)
 553
 554         def discover(self, nid):
 555                 L = TheLNet()
 556                 rc = L.discover(nid)
 557                 nids = []
 558                 if 'manage' in list(rc.keys()):
 559                         return []
 560                 for entry in rc['discover']:
 561                         for nid in entry['peer ni']:
 562                                 nids.append(nid['nid'])
 563                 return nids
 564
 565         def ping(self, nid):
 566                 L = TheLNet()
 567                 rc = L.ping(nid)
 568                 nids = []
 569                 if 'manage' in list(rc.keys()):
 570                         return []
 571                 for entry in rc['ping']:
 572                         for nid in entry['peer ni']:
 573                                 nids.append(nid['nid'])
 574                 return nids
 575
 576         def get_nets(self, net=None, wrapper=False, detail=False):
 577                 L = TheLNet()
 578                 if not wrapper:
 579                         if detail:
 580                                 return L.get_net_detail(net)
 581                         return L.get_net(net)
 582                 else:
 583                         L.update()
 584                         return L.nets[net].get()
 585
 586         def get_net_stats(self, net=None):
 587                 L = TheLNet()
 588                 return L.get_net_stats(net)
 589
 590         def get_peers(self, nid=None, detailed=False):
 591                 L = TheLNet()
 592                 if not detailed:
 593                         return L.get_peers(nid=nid)
 594                 return L.get_peers_detail(nid=nid)
 595
 596         def get_peer_stats(self, nid=None):
 597                 L = TheLNet()
 598                 return L.get_peer_stats(nid=nid)
 599
 600         def get_stats(self):
 601                 L = TheLNet()
 602                 return L.get_stats()
 603
 604         def set_discovery(self, value):
 605                 L = TheLNet()
 606                 L.set_global_param('discovery', value)
 607
 608         def set_max_intf(self, value):
 609                 L = TheLNet()
 610                 L.set_global_param('max_interfaces', value)
 611
 612         def set_numa_range(self, value):
 613                 L = TheLNet()
 614                 L.set_global_param('numa_range', value)
 615
 616         def set_drop_asym_route(self, value):
 617                 L = TheLNet()
 618                 L.set_global_param('drop_asym_route', value)
 619
 620         def set_retry_count(self, value):
 621                 L = TheLNet()
 622                 L.set_global_param('retry_count', value)
 623
 624         def set_transaction_timeout(self, value):
 625                 L = TheLNet()
 626                 L.set_global_param('transaction_timeout', value)
 627
 628         def set_health_sensitivity(self, value):
 629                 L = TheLNet()
 630                 L.set_global_param('health_sensitivity', value)
 631
 632         def set_recovery_interval(self, value):
 633                 L = TheLNet()
 634                 L.set_global_param('recovery_interval', value)
 635
 636         def set_router_sensitivity(self, value):
 637                 L = TheLNet()
 638                 L.set_global_param('router_sensitivity', value)
 639
 640         def get_globals(self):
 641                 L = TheLNet()
 642                 return L.get_global()
 643
 644         def get_config(self):
 645                 L = TheLNet()
 646                 return L.get_config()
 647
 648         def configure_yaml(self, yml):
 649                 L = TheLNet()
 650                 L.configure_yaml(yml)
 651
 652         def get_cpu_partition_distance(self):
 653                 p = {}
 654                 cptd = os.path.join(os.sep, 'sys', 'kernel', 'debug', 'lnet', 'cpu_partition_distance')
 655                 if not os.path.isfile(cptd):
 656                         return p
 657                 rc = lutf_exec_local_cmd(CAT + " " + cptd)
 658                 p = yaml.load(rc[0].decode('utf-8').replace('\t', ' '),
 659                               Loader=yaml.FullLoader)
 660                 for k, v in p.items():
 661                         l = v.split()
 662                         d = {}
 663                         for e in l:
 664                                 entry = e.split(':')
 665                                 d[int(entry[0])] = int(entry[1])
 666                         p[k] = d
 667                 return p
 668
 669         def check_udsp_present(self):
 670                 rc = lutf_exec_local_cmd(MAN + " lnetctl")
 671                 ret_str = str(rc)
 672                 if "UDSP" in ret_str:
 673                         return True
 674                 return False
 675
 676         def cleanup_udsp(self, num_rules=1):
 677                 for ii in range (0, num_rules):
 678                         rc = lutf_exec_local_cmd(get_lnetctl() + " udsp del --idx 0")
 679                 return rc
 680
 681         def check_udsp_empty(self):
 682                 rc = lutf_exec_local_cmd(get_lnetctl() + " udsp show")
 683                 y = yaml.load(rc[0].decode('utf-8'), Loader=yaml.FullLoader)
 684                 print(y)
 685                 if y != None:
 686                         #print("UDSP list not empty")
 687                         error = "UDSP list not empty"
 688                         logging.debug(error)
 689                         return False
 690                 else:
 691                         return True
 692
 693         def check_udsp_expected(self, udsp_conf_expected_dict):
 694                 rc = lutf_exec_local_cmd(get_lnetctl() + " udsp show")
 695                 y = yaml.load(rc[0].decode('utf-8'), Loader=yaml.FullLoader)
 696                 print(y)
 697                 #print ("Out")
 698                 #print(rc[0])
 699                 if y == udsp_conf_expected_dict:
 700                         return True
 701                 else:
 702                         error = "%s doesn't match expected: %s" % (str(y), str(udsp_conf_expected_dict))
 703                         #print("%s doesn't match expected: %s ", str(y), str(udsp_conf_expected_dict))
 704                         logging.debug(error)
 705                 return False
 706
 707         def exec_udsp_cmd(self, udsp_cmd_string):
 708                 rc = lutf_exec_local_cmd(get_lnetctl() + " udsp " + udsp_cmd_string)
 709                 return rc
 710
 711         def exec_ping(self, dest_nid):
 712                 rc = lutf_exec_local_cmd(get_lnetctl() + " ping " + dest_nid)
 713                 ret_str = str(rc)
 714                 if "primary nid" in ret_str:
 715                         return True
 716                 return False
 717
 718         def exec_route_cmd(self, route_cmd):
 719                 rc = lutf_exec_local_cmd(get_lnetctl() + " route " + route_cmd)
 720                 return rc
 721
 722         def exec_discover_cmd(self, nid):
 723                 rc = lutf_exec_local_cmd(get_lnetctl() + " discover " + nid)
 724                 y = yaml.load(rc[0].decode('utf-8'), Loader=yaml.FullLoader)
 725                 nids = []
 726                 if 'manage' in list(y.keys()):
 727                         return []
 728                 for entry in y['discover']:
 729                         for nid in entry['peer ni']:
 730                                 nids.append(nid['nid'])
 731                 return nids