3 # Copyright (C) 2002 Cluster File Systems, Inc.
4 # Author: Robert Read <rread@clusterfs.com>
6 # This file is part of Lustre, http://www.lustre.org.
8 # Lustre is free software; you can redistribute it and/or
9 # modify it under the terms of version 2 of the GNU General Public
10 # License as published by the Free Software Foundation.
12 # Lustre is distributed in the hope that it will be useful,
13 # but WITHOUT ANY WARRANTY; without even the implied warranty of
14 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
15 # GNU General Public License for more details.
17 # You should have received a copy of the GNU General Public License
18 # along with Lustre; if not, write to the Free Software
19 # Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
21 # lconf - lustre configuration tool
23 # lconf is the main driver script for starting and stopping
24 # lustre filesystem services.
26 # Based in part on the XML obdctl modifications done by Brian Behlendorf
29 import string, os, stat, popen2
31 import xml.dom.minidom
34 TCP_ACCEPTOR = 'acceptor'
38 # Maximum number of devices to search for.
39 # (the /dev/loop* nodes need to be created beforehand)
40 MAX_LOOP_DEVICES = 256
44 print """usage: lconf --ldap server | config.xml
46 config.xml Lustre configuration in xml format.
47 --ldap server LDAP server with lustre config database
51 --debug Don't send lctl commenads
52 --reformat Reformat all devices (will confirm)
53 --lustre="src dir" Base directory of lustre sources. Used to search
55 --portals=src Portals source
56 --makeldiff Translate xml source to LDIFF
57 --cleanup Cleans up config. (Shutdown)
60 (SCRIPT STILL UNDER DEVELOPMENT, MOST FUNCTIONALITY UNIMPLEMENTED)
63 # ============================================================
64 # debugging and error funcs
66 def fixme(msg = "this feature"):
67 raise RuntimeError, msg + ' not implmemented yet.'
70 msg = string.join(map(str,args))
72 raise RuntimeError, msg
75 msg = string.join(map(str,args))
83 msg = string.join(map(str,args))
84 if isverbose(): print msg
87 return options.has_key('verbose') and options['verbose'] == 1
90 return options.has_key('debug') and options['debug'] == 1
92 # ============================================================
93 # locally defined exceptions
94 class CommandError (exceptions.Exception):
95 def __init__(self, args=None):
98 # ============================================================
99 # handle lctl interface
102 Manage communication with lctl
105 def __init__(self, cmd):
107 Initialize close by finding the lctl binary.
109 syspath = string.split(os.environ['PATH'], ':')
110 syspath.insert(0, "../utils");
113 lctl = os.path.join(d,cmd)
114 if os.access(lctl, os.X_OK):
118 raise RuntimeError, "unable to find lctl binary."
123 the cmds are written to stdin of lctl
124 lctl doesn't return errors when run in script mode, so
126 should modify command line to accept multiple commands, or
127 create complex command line options
129 debug("+", self.lctl, cmds)
130 if isnotouch(): return ([], 0)
131 p = popen2.Popen3(self.lctl, 1)
132 p.tochild.write(cmds + "\n")
134 out = p.fromchild.readlines()
136 err = p.childerr.readlines()
138 log (self.lctl, "error:", ret)
140 raise CommandError, err
144 # create a new device with lctl
145 def network(self, net, nid):
152 # create a new connection
153 def connect(self, net, nid, port, servuuid):
158 quit""" % (net, nid, port, servuuid, nid)
161 # create a new device with lctl
162 def disconnect(self, net, nid, port, servuuid):
169 # create a new device with lctl
170 def newdev(self, attach, setup):
175 quit""" % (attach, setup)
179 def cleanup(self, name, uuid):
188 def lovconfig(self, uuid, mdcuuid, stripe_cnt, stripe_sz, pattern, devlist):
192 lovconfig %s %d %d %s %s
193 quit""" % (mdcuuid, uuid, stripe_cnt, stripe_sz, pattern, devlist)
196 # ============================================================
197 # Various system-level functions
198 # (ideally moved to their own module)
200 # Run a command and return the output and status.
201 # stderr is sent to /dev/null, could use popen3 to
202 # save it if necessary
204 cmd = string.join(map(str,args))
206 if isnotouch(): return ([], 0)
207 f = os.popen(cmd + ' 2>&1')
217 # is the path a block device?
224 return stat.S_ISBLK(s[stat.ST_MODE])
226 # build fs according to type
228 def mkfs(fstype, dev):
229 if(fstype == 'ext3'):
230 mkfs = 'mkfs.ext2 -j'
231 elif (fstype == 'extN'):
232 mkfs = 'mkfs.ext2 -j'
234 print 'unsupported fs type: ', fstype
235 if not is_block(dev):
239 run (mkfs, force, dev)
241 # some systems use /dev/loopN, some /dev/loop/N
245 if not os.access(loop + str(0), os.R_OK):
247 if not os.access(loop + str(0), os.R_OK):
248 panic ("can't access loop devices")
251 # find loop device assigned to thefile
254 for n in xrange(0, MAX_LOOP_DEVICES):
256 if os.access(dev, os.R_OK):
257 (stat, out) = run('losetup', dev)
258 if (out and stat == 0):
259 m = re.search(r'\((.*)\)', out[0])
260 if m and file == m.group(1):
266 # create file if necessary and assign the first free loop device
267 def init_loop(file, size, fstype):
268 dev = find_loop(file)
270 print 'WARNING file:', file, 'already mapped to', dev
272 if not os.access(file, os.R_OK | os.W_OK):
273 run("dd if=/dev/zero bs=1k count=0 seek=%d of=%s" %(size, file))
275 # find next free loop
276 for n in xrange(0, MAX_LOOP_DEVICES):
278 if os.access(dev, os.R_OK):
279 (stat, out) = run('losetup', dev)
281 run('losetup', dev, file)
284 print "out of loop devices"
286 print "out of loop devices"
289 # undo loop assignment
290 def clean_loop(file):
291 dev = find_loop(file)
293 ret, out = run('losetup -d', dev)
295 log('unable to clean loop device:', dev, 'for file:', file)
298 # initialize a block device if needed
299 def block_dev(dev, size, fstype, format):
300 if isnotouch(): return dev
301 if not is_block(dev):
302 dev = init_loop(dev, size, fstype)
303 if (format == 'yes'):
307 # ============================================================
308 # Functions to prepare the various objects
310 def prepare_ldlm(node):
311 (name, uuid) = getNodeAttr(node)
312 print 'LDLM:', name, uuid
313 lctl.newdev(attach="ldlm %s %s" % (name, uuid),
316 def prepare_lov(node):
317 (name, uuid, mdcuuid, stripe_cnt, strip_sz, pattern, devlist) = getLOVInfo(node)
318 print 'LOV:', name, uuid
319 lctl.lovconfig(uuid, mdcuuid, stripe_cnt, strip_sz, pattern, devlist)
321 def prepare_network(node):
322 (name, uuid, type, nid, port) = getNetworkInfo(node)
323 print 'NETWORK:', type, nid, port
325 run(TCP_ACCEPTOR, port)
326 lctl.network(type, nid)
329 # need to check /proc/mounts and /etc/mtab before
330 # formatting anything.
331 # FIXME: check if device is already formatted.
332 def prepare_obd(obd):
333 (name, uuid, obdtype, dev, size, fstype, format) = getOBDInfo(obd)
334 print "OBD: ", name, obdtype, dev, size, fstype, format
335 dev = block_dev(dev, size, fstype, format)
336 lctl.newdev(attach="%s %s %s" % (obdtype, name, uuid),
337 setup ="%s %s" %(dev, fstype))
340 def prepare_ost(ost):
341 name, uuid, obd = getOSTInfo(ost)
342 print "OST: ", name, uuid, obd
343 lctl.newdev(attach="ost %s %s" % (name, uuid),
344 setup ="$%s" % (obd))
346 def prepare_mds(node):
347 (name, uuid, dev, size, fstype, format) = getMDSInfo(node)
348 print "MDS: ", name, dev, size, fstype
349 # setup network for mds, too
350 dev = block_dev(dev, size, fstype, format)
351 lctl.newdev(attach="mds %s %s" % (name, uuid),
352 setup ="%s %s" %(dev, fstype))
354 def prepare_osc(node):
355 (name, uuid, obduuid, srvuuid) = getOSCInfo(node)
356 print 'OSC:', name, uuid, obduuid, srvuuid
357 net = lookup(node.parentNode, srvuuid)
358 srvname, srvuuid, net, server, port = getNetworkInfo(net)
359 lctl.connect(net, server, port, srvuuid)
360 lctl.newdev(attach="osc %s %s" % (name, uuid),
361 setup ="%s %s" %(obduuid, srvuuid))
363 def prepare_mdc(node):
364 (name, uuid, mdsuuid, netuuid) = getMDCInfo(node)
365 print 'MDC:', name, uuid, mdsuuid, netuuid
366 lctl.newdev(attach="mdc %s %s" % (name, uuid),
367 setup ="%s %s" %(mdsuuid, netuuid))
369 def prepare_mountpoint(node):
372 # ============================================================
373 # Functions to cleanup the various objects
375 def cleanup_ldlm(node):
376 (name, uuid) = getNodeAttr(node)
377 print 'LDLM:', name, uuid
379 lctl.cleanup(name, uuid)
381 print "cleanup failed: ", name
383 def cleanup_lov(node):
384 (name, uuid) = getNodeAttr(node)
385 print 'LOV:', name, uuid
387 #lctl.cleanup(name, uuid)
389 def cleanup_network(node):
390 (name, uuid, type, nid, port) = getNetworkInfo(node)
391 print 'NETWORK:', type, nid, port
392 #lctl.network(type, nid)
394 # need to check /proc/mounts and /etc/mtab before
395 # formatting anything.
396 # FIXME: check if device is already formatted.
397 def cleanup_obd(obd):
398 (name, uuid, obdtype, dev, size, fstype, format) = getOBDInfo(obd)
399 print "OBD: ", name, obdtype, dev, size, fstype, format
401 lctl.cleanup(name, uuid)
403 print "cleanup failed: ", name
406 def cleanup_ost(ost):
407 name, uuid, obd = getOSTInfo(ost)
408 print "OST: ", name, uuid, obd
410 lctl.cleanup(name, uuid)
412 print "cleanup failed: ", name
414 def cleanup_mds(node):
415 (name, uuid, dev, size, fstype, format) = getMDSInfo(node)
416 print "MDS: ", name, dev, size, fstype
418 lctl.cleanup(name, uuid)
420 print "cleanup failed: ", name
424 def cleanup_mdc(node):
425 (name, uuid, mdsuuid, netuuid) = getMDCInfo(node)
426 print 'MDC:', name, uuid, mdsuuid, netuuid
428 lctl.cleanup(name, uuid)
430 print "cleanup failed: ", name
433 def cleanup_osc(node):
434 (name, uuid, obduuid, srvuuid) = getOSCInfo(node)
435 print 'OSC:', name, uuid, obduuid, srvuuid
436 net = lookup(node.parentNode, srvuuid)
437 netname, netuuid, net, server, port = getNetworkInfo(net)
439 lctl.disconnect(net, server, port, srvuuid)
440 lctl.cleanup(name, uuid)
442 print "cleanup failed: ", name
444 def cleanup_mountpoint(node):
447 # ============================================================
448 # XML processing and query
451 dev = obd.getElementsByTagName('device')[0]
454 size = int(dev.getAttribute('size'))
457 return dev.firstChild.data, size
460 def getNetworkInfo(node):
461 name, uuid = getNodeAttr(node);
462 type = node.getAttribute('type')
463 nid = getText(node, 'server', "")
464 port = int(getText(node, 'port', 0))
465 return name, uuid, type, nid, port
467 # extract device attributes for an obd
468 def getNodeAttr(node):
469 name = node.getAttribute('name')
470 uuid = node.getAttribute('uuid')
474 name, uuid = getNodeAttr(obd);
475 obdtype = obd.getAttribute('type')
476 devname, size = getDevice(obd)
477 fstype = getText(obd, 'fstype')
478 format = getText(obd, 'autoformat')
479 return (name, uuid, obdtype, devname, size, fstype, format)
482 def getLOVInfo(node):
483 name, uuid = getNodeAttr(node)
484 devs = node.getElementsByTagName('devices')[0]
485 stripe_sz = int(devs.getAttribute('stripesize'))
486 pattern = int(devs.getAttribute('pattern'))
487 mdcref = node.getElementsByTagName('mdc_ref')[0]
488 mdcuuid = mdcref.getAttribute('uuidref')
489 mdc= lookup(node.parentNode, mdcuuid)
490 mdcname = getName(mdc)
493 for child in devs.childNodes:
494 if child.nodeName == 'obd_ref':
495 devlist = devlist + child.getAttribute('uuidref') + " "
496 strip_cnt = stripe_cnt + 1
497 return (name, uuid, mdcname, stripe_cnt, stripe_sz, pattern, devlist)
499 # extract device attributes for an obd
500 def getMDSInfo(node):
501 name, uuid = getNodeAttr(node)
502 devname, size = getDevice(node)
503 fstype = getText(node, 'fstype')
504 format = getText(node, 'autoformat', "no")
505 return (name, uuid, devname, size, fstype, format)
507 # extract device attributes for an obd
508 def getMDCInfo(node):
509 name, uuid = getNodeAttr(node)
510 ref = node.getElementsByTagName('mds_ref')[0]
511 mdsuuid = ref.getAttribute('uuidref')
512 ref = node.getElementsByTagName('network_ref')[0]
513 netuuid = ref.getAttribute('uuidref')
514 return (name, uuid, mdsuuid, netuuid)
517 # extract device attributes for an obd
518 def getOSTInfo(node):
519 name, uuid = getNodeAttr(node)
520 ref = node.getElementsByTagName('obd_ref')[0]
521 uuid = ref.getAttribute('uuidref')
522 obd = lookup(node.parentNode, uuid)
524 obdname = getOBDInfo(obd)[0]
526 obdname = "OBD NOT FOUND"
527 return (name, uuid, obdname)
529 # extract device attributes for an obd
530 def getOSCInfo(node):
531 name, uuid = getNodeAttr(node)
532 ref = node.getElementsByTagName('obd_ref')[0]
533 obduuid = ref.getAttribute('uuidref')
534 ref = node.getElementsByTagName('network_ref')[0]
535 ostuuid = ref.getAttribute('uuidref')
536 return (name, uuid, obduuid, ostuuid)
539 # Get the text content from the first matching child
540 def getText(node, tag, default=""):
541 list = node.getElementsByTagName(tag)
545 return node.firstChild.data
549 # Recusively search from node for a uuid
550 def lookup(node, uuid):
551 for n in node.childNodes:
552 # this service_id check is ugly. need some other way to
553 # differentiate between definitions and references
554 if n.nodeType == n.ELEMENT_NODE:
555 if getUUID(n) == uuid:
562 # Get name attribute of node
564 return node.getAttribute('name')
567 return node.getAttribute('uuidref')
569 # Get name attribute of node
571 return node.getAttribute('uuid')
573 # the tag name is the service type
574 # fixme: this should do some checks to make sure the node is a service
575 def getServiceType(node):
579 # determine what "level" a particular node is at.
580 # the order of iniitailization is based on level. objects
581 # are assigned a level based on type:
582 # net,devices:1, obd, mdd:2 mds,ost:3 osc,mdc:4 mounts:5
583 def getServiceLevel(node):
584 type = getServiceType(node)
585 if type in ('network', 'device', 'ldlm'):
587 elif type in ('obd', 'mdd'):
589 elif type in ('mds','ost'):
591 elif type in ('mdc','osc'):
593 elif type in ('lov',):
595 elif type in ('mountpoint',):
600 # return list of services in a profile. list is a list of tuples
602 def getServices(lustreNode, profileNode):
604 for n in profileNode.childNodes:
605 if n.nodeType == n.ELEMENT_NODE:
606 servNode = lookup(lustreNode, getRef(n))
608 panic('service not found: ' + getName(n))
609 level = getServiceLevel(servNode)
610 list.append((level, servNode))
614 def getByName(lustreNode, tag, name):
615 ndList = lustreNode.getElementsByTagName(tag)
617 if getName(nd) == name:
622 # ============================================================
623 # lconf type level logic
628 def startService(node):
629 type = getServiceType(node)
630 debug('Starting service:', type, getName(node), getUUID(node))
631 # there must be a more dynamic way of doing this...
636 elif type == 'network':
637 prepare_network(node)
648 elif type == 'mountpoint':
649 prepare_mountpoint(node)
652 # Prepare the system to run lustre using a particular profile
653 # in a the configuration.
654 # * load & the modules
655 # * setup networking for the current node
656 # * make sure partitions are in place and prepared
657 # * initialize devices with lctl
658 # Levels is important, and needs to be enforced.
659 def startProfile(lustreNode, profileNode):
661 panic("profile:", profile, "not found.")
662 services = getServices(lustreNode, profileNode)
668 def stopService(node):
669 type = getServiceType(node)
670 debug('Stopping service:', type, getName(node), getUUID(node))
671 # there must be a more dynamic way of doing this...
676 elif type == 'network':
677 cleanup_network(node)
688 elif type == 'mountpoint':
689 cleanup_mountpoint(node)
691 # Shutdown services in reverse order than they were started
692 def cleanupProfile(lustreNode, profileNode):
694 panic("profile:", profile, "not found.")
695 services = getServices(lustreNode, profileNode)
701 def doHost(lustreNode, hostname, cleanFlag):
702 node = getByName(lustreNode, 'node', hostname)
704 node = getByName(lustreNode, 'node', 'localhost')
706 panic("no node for ", hostname)
707 reflist = node.getElementsByTagName('profile_ref')
710 cleanupProfile(lustreNode, lookup(lustreNode, getRef(r)))
712 startProfile(lustreNode, lookup(lustreNode, getRef(r)))
715 # Command line processing
717 def parse_cmdline(argv):
719 long_opts = ["ldap", "reformat", "lustre=",
720 "portals=", "makeldiff", "cleanup", "iam=",
726 opts, args = getopt.getopt(argv, short_opts, long_opts)
727 except getopt.GetoptError:
733 if o in ("-h", "--help"):
737 options['cleanup'] = 1
738 if o in ("-v", "--verbose"):
739 options['verbose'] = 1
743 options['portals'] = a
745 options['lustre'] = a
746 if o == "--reformat":
747 options['reformat'] = 1
751 # Initialize or shutdown lustre according to a configuration file
752 # * prepare the system for lustre
753 # * configure devices with lctl
754 # Shutdown does steps in reverse
756 lctl = LCTLInterface('lctl')
759 args = parse_cmdline(sys.argv[1:])
761 dom = xml.dom.minidom.parse(args[0])
764 fixme("ldap not implemented yet")
766 if not options.has_key('hostname'):
767 ret, host = run('hostname')
769 panic("unable to determine hostname")
770 options['hostname'] = host
772 doHost(dom.childNodes[0], options['hostname'], options.has_key('cleanup') )
776 # try a different traceback style. (dare ya to try this in java)
777 def my_traceback(file=None):
778 """Print the list of tuples as returned by extract_tb() or
779 extract_stack() as a formatted stack trace to the given file."""
781 (t,v, tb) = sys.exc_info()
782 list = traceback.extract_tb(tb)
785 for filename, lineno, name, line in list:
787 print '%s:%04d %-14s %s' % (filename,lineno, name, line.strip())
789 print '%s:%04d %s' % (filename,lineno, name)
790 print '%s: %s' % (t, v)
792 if __name__ == "__main__":