3 # Copyright (C) 2002 Cluster File Systems, Inc.
4 # Author: Robert Read <rread@clusterfs.com>
6 # This file is part of Lustre, http://www.lustre.org.
8 # Lustre is free software; you can redistribute it and/or
9 # modify it under the terms of version 2 of the GNU General Public
10 # License as published by the Free Software Foundation.
12 # Lustre is distributed in the hope that it will be useful,
13 # but WITHOUT ANY WARRANTY; without even the implied warranty of
14 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
15 # GNU General Public License for more details.
17 # You should have received a copy of the GNU General Public License
18 # along with Lustre; if not, write to the Free Software
19 # Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
21 # lconf - lustre configuration tool
23 # lconf is the main driver script for starting and stopping
24 # lustre filesystem services.
26 # Based in part on the XML obdctl modifications done by Brian Behlendorf
29 import string, os, stat, popen2
31 import xml.dom.minidom
34 TCP_ACCEPTOR = 'acceptor'
35 LCTL = './lctl' # fix this...
38 # Maximum number of devices to search for.
39 # (the /dev/loop* nodes need to be created beforehand)
40 MAX_LOOP_DEVICES = 256
44 print """usage: lconf --ldap server | config.xml
46 config.xml Lustre configuration in xml format.
47 --ldap server LDAP server with lustre config database
51 --debug Don't send lctl commenads
52 --reformat Reformat all devices (will confirm)
53 --lustre="src dir" Base directory of lustre sources. Used to search
55 --portals=src Portals source
56 --makeldiff Translate xml source to LDIFF
57 --cleanup Cleans up config. (Shutdown)
60 (SCRIPT STILL UNDER DEVELOPMENT, MOST FUNCTIONALITY UNIMPLEMENTED)
63 # ============================================================
64 # debugging and error funcs
66 def fixme(msg = "this feature"):
67 raise RuntimeError, msg + ' not implmemented yet.'
70 msg = string.join(map(str,args))
72 raise RuntimeError, msg
75 msg = string.join(map(str,args))
83 msg = string.join(map(str,args))
84 if isverbose(): print msg
87 return options.has_key('verbose') and options['verbose'] == 1
90 return options.has_key('debug') and options['debug'] == 1
92 # ============================================================
93 # locally defined exceptions
94 class CommandError (exceptions.Exception):
95 def __init__(self, args=None):
98 # ============================================================
99 # Various system-level functions
100 # (ideally moved to their own module)
102 # Run a command and return the output and status.
103 # stderr is sent to /dev/null, could use popen3 to
104 # save it if necessary
106 cmd = string.join(map(str,args))
108 if isnotouch(): return ([], 0)
109 f = os.popen(cmd + ' 2>&1')
119 # the cmds are written to stdin of lctl
120 # lctl doesn't return errors when run in script mode, so
122 # should modify command line to accept multiple commands, or
123 # create complex command line options
125 debug("+", LCTL, cmds)
126 if isnotouch(): return ([], 0)
127 p = popen2.Popen3(LCTL, 1)
128 p.tochild.write(cmds + "\n")
130 out = p.fromchild.readlines()
132 err = p.childerr.readlines()
134 log (LCTL, "error:", ret)
136 raise CommandError, err
140 # is the path a block device?
147 return stat.S_ISBLK(s[stat.ST_MODE])
149 # build fs according to type
151 def mkfs(fstype, dev):
152 if(fstype == 'ext3'):
153 mkfs = 'mkfs.ext2 -j'
154 elif (fstype == 'extN'):
155 mkfs = 'mkfs.ext2 -j'
157 print 'unsupported fs type: ', fstype
158 if not is_block(dev):
162 run (mkfs, force, dev)
164 # some systems use /dev/loopN, some /dev/loop/N
168 if not os.access(loop + str(0), os.R_OK):
170 if not os.access(loop + str(0), os.R_OK):
171 panic ("can't access loop devices")
174 # find loop device assigned to thefile
177 for n in xrange(0, MAX_LOOP_DEVICES):
179 if os.access(dev, os.R_OK):
180 (stat, out) = run('losetup', dev)
181 if (out and stat == 0):
182 m = re.search(r'\((.*)\)', out[0])
183 if m and file == m.group(1):
189 # create file if necessary and assign the first free loop device
190 def init_loop(file, size, fstype):
191 dev = find_loop(file)
193 print 'WARNING file:', file, 'already mapped to', dev
195 if not os.access(file, os.R_OK | os.W_OK):
196 run("dd if=/dev/zero bs=1k count=0 seek=%d of=%s" %(size, file))
198 # find next free loop
199 for n in xrange(0, MAX_LOOP_DEVICES):
201 if os.access(dev, os.R_OK):
202 (stat, out) = run('losetup', dev)
204 run('losetup', dev, file)
207 print "out of loop devices"
209 print "out of loop devices"
212 # undo loop assignment
213 def clean_loop(file):
214 dev = find_loop(file)
216 ret, out = run('losetup -d', dev)
218 log('unable to clean loop device:', dev, 'for file:', file)
221 # initialize a block device if needed
222 def block_dev(dev, size, fstype, format):
223 if isnotouch(): return dev
224 if not is_block(dev):
225 dev = init_loop(dev, size, fstype)
226 if (format == 'yes'):
230 # create a new device with lctl
231 def lctl_network(net, nid):
238 # create a new connection
239 def lctl_connect(net, nid, port, servuuid):
244 quit""" % (net, nid, port, servuuid, nid)
247 # create a new device with lctl
248 def lctl_disconnect(net, nid, port, servuuid):
255 # create a new device with lctl
256 def lctl_newdev(attach, setup):
261 quit""" % (attach, setup)
265 def lctl_cleanup(name, uuid):
274 def lctl_lovconfig(uuid, mdcuuid, stripe_cnt, stripe_sz, pattern, devlist):
278 lovconfig %s %d %d %s %s
279 quit""" % (mdcuuid, uuid, stripe_cnt, stripe_sz, pattern, devlist)
282 # ============================================================
283 # Functions to prepare the various objects
285 def prepare_ldlm(node):
286 (name, uuid) = getNodeAttr(node)
287 print 'LDLM:', name, uuid
288 lctl_newdev(attach="ldlm %s %s" % (name, uuid),
291 def prepare_lov(node):
292 (name, uuid, mdcuuid, stripe_cnt, strip_sz, pattern, devlist) = getLOVInfo(node)
293 print 'LOV:', name, uuid
294 lctl_lovconfig(uuid, mdcuuid, stripe_cnt, strip_sz, pattern, devlist)
296 def prepare_network(node):
297 (name, uuid, type, nid, port) = getNetworkInfo(node)
298 print 'NETWORK:', type, nid, port
300 run(TCP_ACCEPTOR, port)
301 lctl_network(type, nid)
304 # need to check /proc/mounts and /etc/mtab before
305 # formatting anything.
306 # FIXME: check if device is already formatted.
307 def prepare_obd(obd):
308 (name, uuid, obdtype, dev, size, fstype, format) = getOBDInfo(obd)
309 print "OBD: ", name, obdtype, dev, size, fstype, format
310 dev = block_dev(dev, size, fstype, format)
311 lctl_newdev(attach="%s %s %s" % (obdtype, name, uuid),
312 setup ="%s %s" %(dev, fstype))
315 def prepare_ost(ost):
316 name, uuid, obd = getOSTInfo(ost)
317 print "OST: ", name, uuid, obd
318 lctl_newdev(attach="ost %s %s" % (name, uuid),
319 setup ="$%s" % (obd))
321 def prepare_mds(node):
322 (name, uuid, dev, size, fstype, format) = getMDSInfo(node)
323 print "MDS: ", name, dev, size, fstype
324 # setup network for mds, too
325 dev = block_dev(dev, size, fstype, format)
326 lctl_newdev(attach="mds %s %s" % (name, uuid),
327 setup ="%s %s" %(dev, fstype))
329 def prepare_osc(node):
330 (name, uuid, obduuid, srvuuid) = getOSCInfo(node)
331 print 'OSC:', name, uuid, obduuid, srvuuid
332 net = lookup(node.parentNode, srvuuid)
333 srvname, srvuuid, net, server, port = getNetworkInfo(net)
334 lctl_connect(net, server, port, srvuuid)
335 lctl_newdev(attach="osc %s %s" % (name, uuid),
336 setup ="%s %s" %(obduuid, srvuuid))
338 def prepare_mdc(node):
339 (name, uuid, mdsuuid, netuuid) = getMDCInfo(node)
340 print 'MDC:', name, uuid, mdsuuid, netuuid
341 lctl_newdev(attach="mdc %s %s" % (name, uuid),
342 setup ="%s %s" %(mdsuuid, netuuid))
344 def prepare_mountpoint(node):
347 # ============================================================
348 # Functions to cleanup the various objects
350 def cleanup_ldlm(node):
351 (name, uuid) = getNodeAttr(node)
352 print 'LDLM:', name, uuid
354 lctl_cleanup(name, uuid)
356 print "cleanup failed: ", name
358 def cleanup_lov(node):
359 (name, uuid) = getNodeAttr(node)
360 print 'LOV:', name, uuid
362 #lctl_cleanup(name, uuid)
364 def cleanup_network(node):
365 (name, uuid, type, nid, port) = getNetworkInfo(node)
366 print 'NETWORK:', type, nid, port
367 #lctl_network(type, nid)
369 # need to check /proc/mounts and /etc/mtab before
370 # formatting anything.
371 # FIXME: check if device is already formatted.
372 def cleanup_obd(obd):
373 (name, uuid, obdtype, dev, size, fstype, format) = getOBDInfo(obd)
374 print "OBD: ", name, obdtype, dev, size, fstype, format
376 lctl_cleanup(name, uuid)
378 print "cleanup failed: ", name
381 def cleanup_ost(ost):
382 name, uuid, obd = getOSTInfo(ost)
383 print "OST: ", name, uuid, obd
385 lctl_cleanup(name, uuid)
387 print "cleanup failed: ", name
389 def cleanup_mds(node):
390 (name, uuid, dev, size, fstype, format) = getMDSInfo(node)
391 print "MDS: ", name, dev, size, fstype
393 lctl_cleanup(name, uuid)
395 print "cleanup failed: ", name
399 def cleanup_mdc(node):
400 (name, uuid, mdsuuid, netuuid) = getMDCInfo(node)
401 print 'MDC:', name, uuid, mdsuuid, netuuid
403 lctl_cleanup(name, uuid)
405 print "cleanup failed: ", name
408 def cleanup_osc(node):
409 (name, uuid, obduuid, srvuuid) = getOSCInfo(node)
410 print 'OSC:', name, uuid, obduuid, srvuuid
411 net = lookup(node.parentNode, srvuuid)
412 netname, netuuid, net, server, port = getNetworkInfo(net)
414 lctl_disconnect(net, server, port, srvuuid)
415 lctl_cleanup(name, uuid)
417 print "cleanup failed: ", name
419 def cleanup_mountpoint(node):
422 # ============================================================
423 # XML processing and query
426 dev = obd.getElementsByTagName('device')[0]
429 size = int(dev.getAttribute('size'))
432 return dev.firstChild.data, size
435 def getNetworkInfo(node):
436 name, uuid = getNodeAttr(node);
437 type = node.getAttribute('type')
438 nid = getText(node, 'server', "")
439 port = int(getText(node, 'port', 0))
440 return name, uuid, type, nid, port
442 # extract device attributes for an obd
443 def getNodeAttr(node):
444 name = node.getAttribute('name')
445 uuid = node.getAttribute('uuid')
449 name, uuid = getNodeAttr(obd);
450 obdtype = obd.getAttribute('type')
451 devname, size = getDevice(obd)
452 fstype = getText(obd, 'fstype')
453 format = getText(obd, 'autoformat')
454 return (name, uuid, obdtype, devname, size, fstype, format)
457 def getLOVInfo(node):
458 name, uuid = getNodeAttr(node)
459 devs = node.getElementsByTagName('devices')[0]
460 stripe_sz = int(devs.getAttribute('stripesize'))
461 pattern = int(devs.getAttribute('pattern'))
462 mdcref = node.getElementsByTagName('mdc_ref')[0]
463 mdcuuid = mdcref.getAttribute('uuidref')
464 mdc= lookup(node.parentNode, mdcuuid)
465 mdcname = getName(mdc)
468 for child in devs.childNodes:
469 if child.nodeName == 'obd_ref':
470 devlist = devlist + child.getAttribute('uuidref') + " "
471 strip_cnt = stripe_cnt + 1
472 return (name, uuid, mdcname, stripe_cnt, stripe_sz, pattern, devlist)
474 # extract device attributes for an obd
475 def getMDSInfo(node):
476 name, uuid = getNodeAttr(node)
477 devname, size = getDevice(node)
478 fstype = getText(node, 'fstype')
479 format = getText(node, 'autoformat', "no")
480 return (name, uuid, devname, size, fstype, format)
482 # extract device attributes for an obd
483 def getMDCInfo(node):
484 name, uuid = getNodeAttr(node)
485 ref = node.getElementsByTagName('mds_ref')[0]
486 mdsuuid = ref.getAttribute('uuidref')
487 ref = node.getElementsByTagName('network_ref')[0]
488 netuuid = ref.getAttribute('uuidref')
489 return (name, uuid, mdsuuid, netuuid)
492 # extract device attributes for an obd
493 def getOSTInfo(node):
494 name, uuid = getNodeAttr(node)
495 ref = node.getElementsByTagName('obd_ref')[0]
496 uuid = ref.getAttribute('uuidref')
497 obd = lookup(node.parentNode, uuid)
499 obdname = getOBDInfo(obd)[0]
501 obdname = "OBD NOT FOUND"
502 return (name, uuid, obdname)
504 # extract device attributes for an obd
505 def getOSCInfo(node):
506 name, uuid = getNodeAttr(node)
507 ref = node.getElementsByTagName('obd_ref')[0]
508 obduuid = ref.getAttribute('uuidref')
509 ref = node.getElementsByTagName('network_ref')[0]
510 ostuuid = ref.getAttribute('uuidref')
511 return (name, uuid, obduuid, ostuuid)
514 # Get the text content from the first matching child
515 def getText(node, tag, default=""):
516 list = node.getElementsByTagName(tag)
520 return node.firstChild.data
524 # Recusively search from node for a uuid
525 def lookup(node, uuid):
526 for n in node.childNodes:
527 # this service_id check is ugly. need some other way to
528 # differentiate between definitions and references
529 if n.nodeType == n.ELEMENT_NODE:
530 if getUUID(n) == uuid:
537 # Get name attribute of node
539 return node.getAttribute('name')
542 return node.getAttribute('uuidref')
544 # Get name attribute of node
546 return node.getAttribute('uuid')
548 # the tag name is the service type
549 # fixme: this should do some checks to make sure the node is a service
550 def getServiceType(node):
554 # determine what "level" a particular node is at.
555 # the order of iniitailization is based on level. objects
556 # are assigned a level based on type:
557 # net,devices:1, obd, mdd:2 mds,ost:3 osc,mdc:4 mounts:5
558 def getServiceLevel(node):
559 type = getServiceType(node)
560 if type in ('network', 'device', 'ldlm'):
562 elif type in ('obd', 'mdd'):
564 elif type in ('mds','ost'):
566 elif type in ('mdc','osc'):
568 elif type in ('lov',):
570 elif type in ('mountpoint',):
575 # return list of services in a profile. list is a list of tuples
577 def getServices(lustreNode, profileNode):
579 for n in profileNode.childNodes:
580 if n.nodeType == n.ELEMENT_NODE:
581 servNode = lookup(lustreNode, getRef(n))
583 panic('service not found: ' + getName(n))
584 level = getServiceLevel(servNode)
585 list.append((level, servNode))
589 def getProfile(lustreNode, profile):
590 profList = lustreNode.getElementsByTagName('profile')
591 for prof in profList:
592 if getName(prof) == profile:
596 # ============================================================
597 # lconf type level logic
602 def startService(node):
603 type = getServiceType(node)
604 debug('Starting service:', type, getName(node), getUUID(node))
605 # there must be a more dynamic way of doing this...
610 elif type == 'network':
611 prepare_network(node)
622 elif type == 'mountpoint':
623 prepare_mountpoint(node)
626 # Prepare the system to run lustre using a particular profile
627 # in a the configuration.
628 # * load & the modules
629 # * setup networking for the current node
630 # * make sure partitions are in place and prepared
631 # * initialize devices with lctl
632 # Levels is important, and needs to be enforced.
633 def startProfile(lustreNode, profile):
634 profileNode = getProfile(lustreNode, profile)
636 panic("profile:", profile, "not found.")
637 services = getServices(lustreNode, profileNode)
643 def stopService(node):
644 type = getServiceType(node)
645 debug('Stopping service:', type, getName(node), getUUID(node))
646 # there must be a more dynamic way of doing this...
651 elif type == 'network':
652 cleanup_network(node)
663 elif type == 'mountpoint':
664 cleanup_mountpoint(node)
666 # Shutdown services in reverse order than they were started
667 def cleanupProfile(lustreNode, profile):
668 profileNode = getProfile(lustreNode, profile)
670 panic("profile:", profile, "not found.")
671 services = getServices(lustreNode, profileNode)
677 # Command line processing
679 def parse_cmdline(argv):
681 long_opts = ["ldap", "reformat", "lustre=",
682 "portals=", "makeldiff", "cleanup", "iam=",
688 opts, args = getopt.getopt(argv, short_opts, long_opts)
689 except getopt.GetoptError:
695 if o in ("-h", "--help"):
699 options['cleanup'] = 1
700 if o in ("-v", "--verbose"):
701 options['verbose'] = 1
705 options['portals'] = a
707 options['lustre'] = a
708 if o == "--reformat":
709 options['reformat'] = 1
713 # Initialize or shutdown lustre according to a configuration file
714 # * prepare the system for lustre
715 # * configure devices with lctl
716 # Shutdown does steps in reverse
720 args = parse_cmdline(sys.argv[1:])
722 dom = xml.dom.minidom.parse(args[0])
725 fixme("ldap not implemented yet")
727 if options.has_key('cleanup'):
728 cleanupProfile(dom.childNodes[0], 'local-profile')
730 startProfile(dom.childNodes[0], 'local-profile')
733 # try a different traceback style. (dare ya to try this in java)
734 def my_traceback(file=None):
735 """Print the list of tuples as returned by extract_tb() or
736 extract_stack() as a formatted stack trace to the given file."""
738 (t,v, tb) = sys.exc_info()
739 list = traceback.extract_tb(tb)
742 for filename, lineno, name, line in list:
744 print '%s:%04d %-14s %s' % (filename,lineno, name, line.strip())
746 print '%s:%04d %s' % (filename,lineno, name)
747 print '%s: %s' % (t, v)
749 if __name__ == "__main__":