lustre/utils/Lustre/lustredb.py

   1 import sys, types, string, os
   2 import re, exceptions
   3 import xml.dom.minidom
   4 import Lustre
   5
   6 # ============================================================
   7 # XML processing and query
   8
   9 class LustreDB:
  10     caching_enabled = 1
  11
  12     def __init__(self):
  13         self.lookup_uuid_cache = {}
  14         self.lookup_name_cache = {}
  15         self.lookup_class_cache = {}
  16         self.lookup_val_cache = {}
  17         self.lookup_refs_cache = {}
  18         self.lookup_lovtgts_cache = {}
  19         self.lookup_nid2srv_cache = {}
  20         self.lookup_activedev_cache = {}
  21         self.lookup_tgtdev_cache = {}
  22         self.lookup_group_cache = {}
  23
  24         self.lookup_allrefs_cache = None
  25         self.lookup_networks_cache = None
  26
  27     def lookup(self, uuid):
  28         """ lookup returns a new LustreDB instance"""
  29         if self.caching_enabled and self.lookup_uuid_cache.has_key(uuid):
  30             res = self.lookup_uuid_cache[uuid]
  31         else:
  32             res = self._lookup_by_uuid(uuid)
  33             if self.caching_enabled:
  34                 self.lookup_uuid_cache[uuid] = res
  35         return res
  36
  37     def lookup_name(self, name, class_name = ""):
  38         """ lookup returns a new LustreDB instance"""
  39         if self.caching_enabled and self.lookup_name_cache.has_key((name, class_name)):
  40             res = self.lookup_name_cache[(name, class_name)]
  41         else:
  42             res = self._lookup_by_name(name, class_name)
  43             if self.caching_enabled:
  44                 self.lookup_name_cache[(name, class_name)] = res
  45         return res
  46
  47     def lookup_class(self, class_name):
  48         """ lookup returns a new LustreDB instance"""
  49         if self.caching_enabled and self.lookup_class_cache.has_key(class_name):
  50             res = self.lookup_class_cache[class_name]
  51         else:
  52             res = self._lookup_by_class(class_name)
  53             if self.caching_enabled:
  54                 self.lookup_class_cache[class_name] = res
  55         return res
  56
  57     def get_val(self, tag, default=None):
  58         if self.caching_enabled and self.lookup_class_cache.has_key(tag):
  59             v = self.lookup_val_cache[tag]
  60         else:
  61             v = self._get_val(tag)
  62             if self.caching_enabled:
  63                 self.lookup_val_cache[tag] = v
  64         if v:
  65             return v
  66         if default != None:
  67             return default
  68         return None
  69
  70     def get_class(self):
  71         return self._get_class()
  72
  73     def get_val_int(self, tag, default=0):
  74         str = self.get_val(tag)
  75         try:
  76             if str:
  77                 return int(str)
  78             return default
  79         except ValueError:
  80             raise Lustre.LconfError("text value is not integer: " + str)
  81
  82     def get_first_ref(self, tag):
  83         """ Get the first uuidref of the type TAG. Only
  84         one is expected.  Returns the uuid."""
  85         uuids = self.get_refs(tag)
  86         if len(uuids) > 0:
  87             return  uuids[0]
  88         return None
  89
  90     def get_refs(self, tag):
  91         """ Get all the refs of type TAG.  Returns list of uuids. """
  92         if self.caching_enabled and self.lookup_refs_cache.has_key(tag):
  93             uuids = self.lookup_refs_cache[tag]
  94         else:
  95             uuids = self._get_refs(tag)
  96             if self.caching_enabled:
  97                 self.lookup_refs_cache[tag] = uuids
  98         return uuids
  99
 100     def get_all_refs(self):
 101         """ Get all the refs.  Returns list of uuids. """
 102         if self.caching_enabled and self.lookup_allrefs_cache:
 103             uuids = self.lookup_allrefs_cache
 104         else:
 105             uuids = self._get_all_refs()
 106             if self.caching_enabled:
 107                 self.lookup_allrefs_cache = uuids
 108         return uuids
 109
 110     def get_lov_tgts(self, tag):
 111         """ Returns list of lov tgts. """
 112         if self.caching_enabled and self.lookup_lovtgts_cache.has_key(tag):
 113             tgts = self.lookup_lovtgts_cache[tag]
 114         else:
 115             tgts = self._get_lov_tgts(tag)
 116             if self.caching_enabled:
 117                 self.lookup_lovtgts_cache[tag] = tgts
 118         return tgts
 119
 120     def nid2server(self, nid, net_type, cluster_id):
 121         if self.caching_enabled and self.lookup_nid2srv_cache.has_key((nid, net_type, cluster_id)):
 122             res = self.lookup_nid2srv_cache[(nid, net_type, cluster_id)]
 123         else:
 124             netlist = self.lookup_class('network')
 125             for net_db in netlist:
 126                 if (net_db.get_val('nid') == nid and
 127                     net_db.get_val('nettype') == net_type and
 128                     net_db.get_val('clusterid') == cluster_id):
 129                     res = net_db
 130                     break
 131             if self.caching_enabled:
 132                 self.lookup_nid2srv_cache[(nid, net_type, cluster_id)] = res
 133         return res
 134
 135     # Find the target_device for target on a node
 136     # node->profiles->device_refs->target
 137     def get_node_tgt_dev(self, node_name, target_uuid):
 138         node_db = self.lookup_name(node_name)
 139         if not node_db:
 140             return None
 141         return node_db.get_tgt_dev(target_uuid)
 142
 143     # get all network uuids for this node
 144     def get_networks(self):
 145         if self.caching_enabled and self.lookup_networks_cache:
 146             ret = self.lookup_networks_cache
 147         else:
 148             ret = []
 149             prof_list = self.get_refs('profile')
 150             for prof_uuid in prof_list:
 151                 prof_db = self.lookup(prof_uuid)
 152                 net_list = prof_db.get_refs('network')
 153                 for net_uuid in net_list:
 154                     ret.append(net_uuid)
 155             if self.caching_enabled:
 156                 self.lookup_networks_cache = ret
 157         return ret
 158
 159     def get_active_dev(self, tgtuuid):
 160         if self.caching_enabled and self.lookup_activedev_cache.has_key(tgtuuid):
 161             tgt_dev_uuid = self.lookup_activedev_cache[tgtuuid]
 162         else:
 163             tgt = self.lookup(tgtuuid)
 164             tgt_dev_uuid = tgt.get_first_ref('active')
 165             if self.caching_enabled:
 166                 self.lookup_activedev_cache[tgtuuid] = tgt_dev_uuid
 167         return tgt_dev_uuid
 168
 169     def get_tgt_dev(self, tgtuuid):
 170         if self.caching_enabled and self.lookup_tgtdev_cache.has_key(tgtuuid):
 171             res = self.lookup_tgtdev_cache[tgtuuid]
 172         else:
 173             prof_list = self.get_refs('profile')
 174             res = None
 175             for prof_uuid in prof_list:
 176                 prof_db = self.lookup(prof_uuid)
 177                 if not prof_db:
 178                     panic("profile:", profile, "not found.")
 179                 for ref_class, ref_uuid in prof_db.get_all_refs():
 180                     if ref_class in ('osd', 'mdsdev'):
 181                         devdb = self.lookup(ref_uuid)
 182                         uuid = devdb.get_first_ref('target')
 183                         if tgtuuid == uuid:
 184                             res = ref_uuid
 185                             break
 186                 if not res is None:
 187                     break
 188             if self.caching_enabled:
 189                 self.lookup_tgtdev_cache[tgtuuid] = res
 190         return res
 191
 192     def get_group(self, group):
 193         if self.caching_enabled and self.lookup_group_cache.has_key(group):
 194             ret = self.lookup_group_cache[group]
 195         else:
 196             ret = []
 197             devs = self.lookup_class('mds')
 198             for tgt in devs:
 199                 if tgt.get_val('group', tgt.get_val('name')) == group:
 200                     ret.append(tgt.getUUID())
 201             devs = self.lookup_class('ost')
 202             for tgt in devs:
 203                 if tgt.get_val('group', tgt.get_val('name')) == group:
 204                     ret.append(tgt.getUUID())
 205             if self.caching_enabled:
 206                 self.lookup_group_cache[group] = ret
 207         return ret
 208
 209     # Change the current active device for a target
 210     def update_active(self, tgtuuid, new_uuid):
 211         self._update_active(tgtuuid, new_uuid)
 212
 213     def get_version(self):
 214         return self.get_val('version')
 215
 216     def get_mtime(self):
 217         return self.get_val('mtime')
 218
 219 class LustreDB_XML(LustreDB):
 220     def __init__(self, dom, root_node):
 221         LustreDB.__init__(self)
 222
 223         # init xmlfile
 224         self.dom_node = dom
 225         self.root_node = root_node
 226
 227     def close(self):
 228         # do nothing
 229         return None
 230
 231     def xmltext(self, dom_node, tag):
 232         list = dom_node.getElementsByTagName(tag)
 233         if len(list) > 0:
 234             dom_node = list[0]
 235             dom_node.normalize()
 236             if dom_node.firstChild:
 237                 txt = string.strip(dom_node.firstChild.data)
 238                 if txt:
 239                     return txt
 240
 241     def xmlattr(self, dom_node, attr):
 242         return dom_node.getAttribute(attr)
 243
 244     def _get_val(self, tag):
 245         """a value could be an attribute of the current node
 246         or the text value in a child node"""
 247         ret  = self.xmlattr(self.dom_node, tag)
 248         if not ret:
 249             ret = self.xmltext(self.dom_node, tag)
 250         return ret
 251
 252     def _get_class(self):
 253         return self.dom_node.nodeName
 254
 255     def get_ref_type(self, ref_tag):
 256         res = string.split(ref_tag, '_')
 257         return res[0]
 258
 259     #
 260     # [(ref_class, ref_uuid),]
 261     def _get_all_refs(self):
 262         list = []
 263         for n in self.dom_node.childNodes:
 264             if n.nodeType == n.ELEMENT_NODE:
 265                 ref_uuid = self.xml_get_ref(n)
 266                 ref_class = self.get_ref_type(n.nodeName)
 267                 list.append((ref_class, ref_uuid))
 268
 269         list.sort()
 270         return list
 271
 272     def _get_refs(self, tag):
 273         """ Get all the refs of type TAG.  Returns list of uuids. """
 274         uuids = []
 275         refname = '%s_ref' % tag
 276         reflist = self.dom_node.getElementsByTagName(refname)
 277         for r in reflist:
 278             uuids.append(self.xml_get_ref(r))
 279         return uuids
 280
 281     def _get_lov_tgts(self, tag):
 282         """ Get all the refs of type TAG.  Returns list of lov_tgts. """
 283         tgts = []
 284         tgtlist = self.dom_node.getElementsByTagName(tag)
 285         for tgt in tgtlist:
 286             uuidref = tgt.getAttribute('uuidref')
 287             index = tgt.getAttribute('index')
 288             generation = tgt.getAttribute('generation')
 289             active = int(tgt.getAttribute('active'))
 290             tgts.append((uuidref, index, generation, active))
 291         return tgts
 292
 293     def xmllookup_by_uuid(self, dom_node, uuid):
 294         for n in dom_node.childNodes:
 295             if n.nodeType == n.ELEMENT_NODE:
 296                 if self.xml_get_uuid(n) == uuid:
 297                     return n
 298                 else:
 299                     n = self.xmllookup_by_uuid(n, uuid)
 300                     if n: return n
 301         return None
 302
 303     def _lookup_by_uuid(self, uuid):
 304         dom = self. xmllookup_by_uuid(self.root_node, uuid)
 305         if dom:
 306             return LustreDB_XML(dom, self.root_node)
 307
 308     def xmllookup_by_name(self, dom_node, name):
 309         for n in dom_node.childNodes:
 310             if n.nodeType == n.ELEMENT_NODE:
 311                 if self.xml_get_name(n) == name:
 312                     return n
 313                 else:
 314                     n = self.xmllookup_by_name(n, name)
 315                     if n: return n
 316         return None
 317
 318     def _lookup_by_name(self, name, class_name):
 319         dom = self.xmllookup_by_name(self.root_node, name)
 320         if dom:
 321             return LustreDB_XML(dom, self.root_node)
 322
 323     def xmllookup_by_class(self, dom_node, class_name):
 324         return dom_node.getElementsByTagName(class_name)
 325
 326     def _lookup_by_class(self, class_name):
 327         ret = []
 328         domlist = self.xmllookup_by_class(self.root_node, class_name)
 329         for node in domlist:
 330             ret.append(LustreDB_XML(node, self.root_node))
 331         return ret
 332
 333     def xml_get_name(self, n):
 334         return n.getAttribute('name')
 335
 336     def getName(self):
 337         return self.xml_get_name(self.dom_node)
 338
 339     def xml_get_ref(self, n):
 340         return n.getAttribute('uuidref')
 341
 342     def xml_get_uuid(self, dom_node):
 343         return dom_node.getAttribute('uuid')
 344
 345     def getUUID(self):
 346         return self.xml_get_uuid(self.dom_node)
 347
 348     # Convert routes from the router to a route that will be used
 349     # on the local system.  The network type and gw are changed to the
 350     # interface on the router the local system will connect to.
 351     def get_local_routes(self, type, gw):
 352         """ Return the routes as a list of tuples of the form:
 353         [(type, gw, lo, hi),]"""
 354         res = []
 355         tbl = self.dom_node.getElementsByTagName('routetbl')
 356         for t in tbl:
 357             routes = t.getElementsByTagName('route')
 358             for r in routes:
 359                 net_type = self.xmlattr(r, 'type')
 360                 if type != net_type:
 361                     lo = self.xmlattr(r, 'lo')
 362                     hi = self.xmlattr(r, 'hi')
 363                     tgt_cluster_id = self.xmlattr(r, 'tgtclusterid')
 364                     res.append((type, gw, tgt_cluster_id, lo, hi))
 365         return res
 366
 367     def get_route_tbl(self):
 368         ret = []
 369         for r in self.dom_node.getElementsByTagName('route'):
 370             net_type = self.xmlattr(r, 'type')
 371             gw = self.xmlattr(r, 'gw')
 372             gw_cluster_id = self.xmlattr(r, 'gwclusterid')
 373             tgt_cluster_id = self.xmlattr(r, 'tgtclusterid')
 374             lo = self.xmlattr(r, 'lo')
 375             hi = self.xmlattr(r, 'hi')
 376             ret.append((net_type, gw, gw_cluster_id, tgt_cluster_id, lo, hi))
 377         return ret
 378
 379     def get_hostaddr(self):
 380         ret = []
 381         list = self.dom_node.getElementsByTagName('hostaddr')
 382         for node in list:
 383             ret.append(node.firstChild.data)
 384         return ret
 385
 386     def _update_active(self, tgt, new):
 387         raise Lustre.LconfError("updates not implemented for XML")
 388
 389 # ================================================================
 390 # LDAP Support
 391 class LustreDB_LDAP(LustreDB):
 392     def __init__(self, name, attrs,
 393                  base = "fs=lustre",
 394                  parent = None,
 395                  url  = "ldap://localhost",
 396                  user = "cn=Manager, fs=lustre",
 397                  pw   = ""
 398                  ):
 399         LustreDB.__init__(self)
 400
 401         self._name = name
 402         self._attrs = attrs
 403         self._base = base
 404         self._parent = parent
 405         self._url  = url
 406         self._user = user
 407         self._pw   = pw
 408         if parent:
 409             self.l = parent.l
 410             self._base = parent._base
 411         else:
 412             self.open()
 413
 414     def open(self):
 415         import ldap
 416         try:
 417             self.l = ldap.initialize(self._url)
 418             # Set LDAP protocol version used
 419             self.l.protocol_version=ldap.VERSION3
 420             # user and pw only needed if modifying db
 421             self.l.bind_s(self._user, self._pw, ldap.AUTH_SIMPLE);
 422         except ldap.LDAPError, e:
 423             raise Lustre.LconfError('Unable to connect to ldap server:' + self._url)
 424
 425         try:
 426             self._name, self._attrs = self.l.search_s(self._base,
 427                                                       ldap.SCOPE_BASE)[0]
 428         except ldap.LDAPError, e:
 429             raise Lustre.LconfError("no config found in ldap: %s"
 430                                       % (self._base,))
 431     def close(self):
 432         self.l.unbind_s()
 433
 434     def ldap_search(self, filter):
 435         """Return list of uuids matching the filter."""
 436         import ldap
 437         dn = self._base
 438         ret = []
 439         uuids = []
 440         try:
 441             for name, attrs in self.l.search_s(dn, ldap.SCOPE_ONELEVEL,
 442                                         filter, ["uuid"]):
 443                 for v in attrs['uuid']:
 444                     uuids.append(v)
 445         except ldap.NO_SUCH_OBJECT, e:
 446             pass
 447         except ldap.LDAPError, e:
 448             print e                     # FIXME: die here?
 449         if len(uuids) > 0:
 450             for uuid in uuids:
 451                 ret.append(self._lookup_by_uuid(uuid))
 452         return ret
 453
 454     def _lookup_by_name(self, name, class_name):
 455         list =  self.ldap_search("lustreName=%s" %(name))
 456         if len(list) == 1:
 457             return list[0]
 458         return None
 459
 460     def _lookup_by_class(self, class_name):
 461         return self.ldap_search("objectclass=%s" %(string.upper(class_name)))
 462
 463     def _lookup_by_uuid(self, uuid):
 464         import ldap
 465         dn = "uuid=%s,%s" % (uuid, self._base)
 466         ret = None
 467         try:
 468             for name, attrs in self.l.search_s(dn, ldap.SCOPE_BASE,
 469                                                "objectclass=*"):
 470                 ret = LustreDB_LDAP(name, attrs,  parent = self)
 471
 472         except ldap.NO_SUCH_OBJECT, e:
 473             pass                        # just return empty list
 474         except ldap.LDAPError, e:
 475             print e                     # FIXME: die here?
 476         return ret
 477
 478
 479     def _get_val(self, k):
 480         ret = None
 481         if k == 'name':
 482             k = 'lustreName'
 483         if self._attrs.has_key(k):
 484             v = self._attrs[k]
 485             if type(v) == types.ListType:
 486                 ret = str(v[0])
 487             else:
 488                 ret = str(v)
 489         return ret
 490
 491     def _get_class(self):
 492         return string.lower(self._attrs['objectClass'][0])
 493
 494     def get_ref_type(self, ref_tag):
 495         return ref_tag[:-3]
 496
 497     def _get_lov_tgts(self, tag):
 498         """ Get all the refs of type TAG.  Returns list of lov_tgts. """
 499         tgts = []
 500         return tgts
 501
 502     #
 503     # [(ref_class, ref_uuid),]
 504     def _get_all_refs(self):
 505         reflist = []
 506         for k in self._attrs.keys():
 507             if re.search('.*Ref', k):
 508                 for uuid in self._attrs[k]:
 509                     ref_class = self.get_ref_type(k)
 510                     reflist.append((ref_class, uuid))
 511         return reflist
 512
 513     def _get_refs(self, tag):
 514         """ Get all the refs of type TAG.  Returns list of uuids. """
 515         refname = '%sRef' % tag
 516
 517         if self._attrs.has_key(refname):
 518             return self._attrs[refname]
 519
 520         reflist = []
 521         for obj in self._lookup_by_class("*"):
 522             if obj._attrs.has_key(refname):
 523                 reflist.extend(obj._attrs[refname])
 524
 525         return reflist
 526
 527     def getName(self):
 528         return self._get_val('lustreName')
 529
 530     def getUUID(self):
 531         return self._get_val('uuid')
 532
 533     def get_route_tbl(self):
 534         return []
 535
 536     def get_hostaddr(self):
 537         return self._get_refs('hostaddr')
 538
 539     def _update_active(self, tgtuuid, newuuid):
 540         """Return list of uuids matching the filter."""
 541         import ldap
 542         dn = "uuid=%s,%s" %(tgtuuid, self._base)
 543         ret = []
 544         uuids = []
 545         try:
 546             self.l.modify_s(dn, [(ldap.MOD_REPLACE, "activeRef", newuuid)])
 547         except ldap.NO_SUCH_OBJECT, e:
 548             print e
 549         except ldap.LDAPError, e:
 550             print e                     # FIXME: die here?
 551         return