3 # Copyright (C) 2002 Cluster File Systems, Inc.
4 # Author: Robert Read <rread@clusterfs.com>
6 # This file is part of Lustre, http://www.lustre.org.
8 # Lustre is free software; you can redistribute it and/or
9 # modify it under the terms of version 2 of the GNU General Public
10 # License as published by the Free Software Foundation.
12 # Lustre is distributed in the hope that it will be useful,
13 # but WITHOUT ANY WARRANTY; without even the implied warranty of
14 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
15 # GNU General Public License for more details.
17 # You should have received a copy of the GNU General Public License
18 # along with Lustre; if not, write to the Free Software
19 # Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
21 # lconf - lustre configuration tool
23 # lconf is the main driver script for starting and stopping
24 # lustre filesystem services.
26 # Based in part on the XML obdctl modifications done by Brian Behlendorf
29 import string, os, stat, popen2
31 import xml.dom.minidom
34 TCP_ACCEPTOR = 'acceptor'
38 # Maximum number of devices to search for.
39 # (the /dev/loop* nodes need to be created beforehand)
40 MAX_LOOP_DEVICES = 256
44 print """usage: lconf config.xml
46 config.xml Lustre configuration in xml format.
47 -v | --verbose Print system commands as they are run
48 -d | --debug Print system commands, but does not run them
49 --cleanup Cleans up config. (Shutdown)
52 --ldap server LDAP server with lustre config database
53 --reformat Reformat all devices (will confirm)
54 --lustre="src dir" Base directory of lustre sources. Used to search
56 --portals=src Portals source
57 --makeldiff Translate xml source to LDIFF
62 # ============================================================
63 # debugging and error funcs
65 def fixme(msg = "this feature"):
66 raise RuntimeError, msg + ' not implmemented yet.'
69 msg = string.join(map(str,args))
71 raise RuntimeError, msg
74 msg = string.join(map(str,args))
82 msg = string.join(map(str,args))
83 if isverbose(): print msg
86 return options.has_key('verbose') and options['verbose'] == 1
89 return options.has_key('debug') and options['debug'] == 1
91 # ============================================================
92 # locally defined exceptions
93 class CommandError (exceptions.Exception):
94 def __init__(self, args=None):
97 # ============================================================
98 # handle lctl interface
101 Manage communication with lctl
104 def __init__(self, cmd):
106 Initialize close by finding the lctl binary.
108 syspath = string.split(os.environ['PATH'], ':')
109 syspath.insert(0, "../utils");
112 lctl = os.path.join(d,cmd)
113 if os.access(lctl, os.X_OK):
117 raise RuntimeError, "unable to find lctl binary."
122 the cmds are written to stdin of lctl
123 lctl doesn't return errors when run in script mode, so
125 should modify command line to accept multiple commands, or
126 create complex command line options
128 debug("+", self.lctl, cmds)
129 if isnotouch(): return ([], 0)
130 p = popen2.Popen3(self.lctl, 1)
131 p.tochild.write(cmds + "\n")
133 out = p.fromchild.readlines()
135 err = p.childerr.readlines()
137 log (self.lctl, "error:", ret)
139 raise CommandError, err
143 # create a new device with lctl
144 def network(self, net, nid):
151 # create a new connection
152 def connect(self, net, nid, port, servuuid):
157 quit""" % (net, nid, port, servuuid, nid)
160 # create a new device with lctl
161 def disconnect(self, net, nid, port, servuuid):
168 # create a new device with lctl
169 def newdev(self, attach, setup):
174 quit""" % (attach, setup)
178 def cleanup(self, name, uuid):
187 def lovconfig(self, uuid, mdcuuid, stripe_cnt, stripe_sz, pattern, devlist):
191 lovconfig %s %d %d %s %s
192 quit""" % (mdcuuid, uuid, stripe_cnt, stripe_sz, pattern, devlist)
195 # ============================================================
196 # Various system-level functions
197 # (ideally moved to their own module)
199 # Run a command and return the output and status.
200 # stderr is sent to /dev/null, could use popen3 to
201 # save it if necessary
203 cmd = string.join(map(str,args))
205 if isnotouch(): return ([], 0)
206 f = os.popen(cmd + ' 2>&1')
216 # is the path a block device?
223 return stat.S_ISBLK(s[stat.ST_MODE])
225 # build fs according to type
227 def mkfs(fstype, dev):
228 if(fstype == 'ext3'):
229 mkfs = 'mkfs.ext2 -j'
230 elif (fstype == 'extN'):
231 mkfs = 'mkfs.ext2 -j'
233 print 'unsupported fs type: ', fstype
234 if not is_block(dev):
238 run (mkfs, force, dev)
240 # some systems use /dev/loopN, some /dev/loop/N
244 if not os.access(loop + str(0), os.R_OK):
246 if not os.access(loop + str(0), os.R_OK):
247 panic ("can't access loop devices")
250 # find loop device assigned to thefile
253 for n in xrange(0, MAX_LOOP_DEVICES):
255 if os.access(dev, os.R_OK):
256 (stat, out) = run('losetup', dev)
257 if (out and stat == 0):
258 m = re.search(r'\((.*)\)', out[0])
259 if m and file == m.group(1):
265 # create file if necessary and assign the first free loop device
266 def init_loop(file, size, fstype):
267 dev = find_loop(file)
269 print 'WARNING file:', file, 'already mapped to', dev
271 if not os.access(file, os.R_OK | os.W_OK):
272 run("dd if=/dev/zero bs=1k count=0 seek=%d of=%s" %(size, file))
274 # find next free loop
275 for n in xrange(0, MAX_LOOP_DEVICES):
277 if os.access(dev, os.R_OK):
278 (stat, out) = run('losetup', dev)
280 run('losetup', dev, file)
283 print "out of loop devices"
285 print "out of loop devices"
288 # undo loop assignment
289 def clean_loop(file):
290 dev = find_loop(file)
292 ret, out = run('losetup -d', dev)
294 log('unable to clean loop device:', dev, 'for file:', file)
297 # initialize a block device if needed
298 def block_dev(dev, size, fstype, format):
299 if isnotouch(): return dev
300 if not is_block(dev):
301 dev = init_loop(dev, size, fstype)
302 if (format == 'yes'):
306 # ============================================================
307 # Functions to prepare the various objects
309 def prepare_ldlm(node):
310 (name, uuid) = getNodeAttr(node)
311 print 'LDLM:', name, uuid
312 lctl.newdev(attach="ldlm %s %s" % (name, uuid),
315 def prepare_lov(node):
316 (name, uuid, mdcuuid, stripe_cnt, strip_sz, pattern, devlist) = getLOVInfo(node)
317 print 'LOV:', name, uuid
318 lctl.lovconfig(uuid, mdcuuid, stripe_cnt, strip_sz, pattern, devlist)
320 def prepare_network(node):
321 (name, uuid, type, nid, port) = getNetworkInfo(node)
322 print 'NETWORK:', type, nid, port
324 run(TCP_ACCEPTOR, port)
325 lctl.network(type, nid)
328 # need to check /proc/mounts and /etc/mtab before
329 # formatting anything.
330 # FIXME: check if device is already formatted.
331 def prepare_obd(obd):
332 (name, uuid, obdtype, dev, size, fstype, format) = getOBDInfo(obd)
333 print "OBD: ", name, obdtype, dev, size, fstype, format
334 dev = block_dev(dev, size, fstype, format)
335 lctl.newdev(attach="%s %s %s" % (obdtype, name, uuid),
336 setup ="%s %s" %(dev, fstype))
339 def prepare_ost(ost):
340 name, uuid, obd = getOSTInfo(ost)
341 print "OST: ", name, uuid, obd
342 lctl.newdev(attach="ost %s %s" % (name, uuid),
343 setup ="$%s" % (obd))
345 def prepare_mds(node):
346 (name, uuid, dev, size, fstype, format) = getMDSInfo(node)
347 print "MDS: ", name, dev, size, fstype
348 # setup network for mds, too
349 dev = block_dev(dev, size, fstype, format)
350 lctl.newdev(attach="mds %s %s" % (name, uuid),
351 setup ="%s %s" %(dev, fstype))
353 def prepare_osc(node):
354 (name, uuid, obduuid, srvuuid) = getOSCInfo(node)
355 print 'OSC:', name, uuid, obduuid, srvuuid
356 net = lookup(node.parentNode, srvuuid)
357 srvname, srvuuid, net, server, port = getNetworkInfo(net)
358 lctl.connect(net, server, port, srvuuid)
359 lctl.newdev(attach="osc %s %s" % (name, uuid),
360 setup ="%s %s" %(obduuid, srvuuid))
362 def prepare_mdc(node):
363 (name, uuid, mdsuuid, netuuid) = getMDCInfo(node)
364 print 'MDC:', name, uuid, mdsuuid, netuuid
365 lctl.newdev(attach="mdc %s %s" % (name, uuid),
366 setup ="%s %s" %(mdsuuid, netuuid))
368 def prepare_mountpoint(node):
371 # ============================================================
372 # Functions to cleanup the various objects
374 def cleanup_ldlm(node):
375 (name, uuid) = getNodeAttr(node)
376 print 'LDLM:', name, uuid
378 lctl.cleanup(name, uuid)
380 print "cleanup failed: ", name
382 def cleanup_lov(node):
383 (name, uuid) = getNodeAttr(node)
384 print 'LOV:', name, uuid
386 #lctl.cleanup(name, uuid)
388 def cleanup_network(node):
389 (name, uuid, type, nid, port) = getNetworkInfo(node)
390 print 'NETWORK:', type, nid, port
391 #lctl.network(type, nid)
393 # need to check /proc/mounts and /etc/mtab before
394 # formatting anything.
395 # FIXME: check if device is already formatted.
396 def cleanup_obd(obd):
397 (name, uuid, obdtype, dev, size, fstype, format) = getOBDInfo(obd)
398 print "OBD: ", name, obdtype, dev, size, fstype, format
400 lctl.cleanup(name, uuid)
402 print "cleanup failed: ", name
405 def cleanup_ost(ost):
406 name, uuid, obd = getOSTInfo(ost)
407 print "OST: ", name, uuid, obd
409 lctl.cleanup(name, uuid)
411 print "cleanup failed: ", name
413 def cleanup_mds(node):
414 (name, uuid, dev, size, fstype, format) = getMDSInfo(node)
415 print "MDS: ", name, dev, size, fstype
417 lctl.cleanup(name, uuid)
419 print "cleanup failed: ", name
423 def cleanup_mdc(node):
424 (name, uuid, mdsuuid, netuuid) = getMDCInfo(node)
425 print 'MDC:', name, uuid, mdsuuid, netuuid
427 lctl.cleanup(name, uuid)
429 print "cleanup failed: ", name
432 def cleanup_osc(node):
433 (name, uuid, obduuid, srvuuid) = getOSCInfo(node)
434 print 'OSC:', name, uuid, obduuid, srvuuid
435 net = lookup(node.parentNode, srvuuid)
436 netname, netuuid, net, server, port = getNetworkInfo(net)
438 lctl.disconnect(net, server, port, srvuuid)
439 lctl.cleanup(name, uuid)
441 print "cleanup failed: ", name
443 def cleanup_mountpoint(node):
446 # ============================================================
447 # XML processing and query
450 dev = obd.getElementsByTagName('device')[0]
453 size = int(dev.getAttribute('size'))
456 return dev.firstChild.data, size
459 def getNetworkInfo(node):
460 name, uuid = getNodeAttr(node);
461 type = node.getAttribute('type')
462 nid = getText(node, 'server', "")
463 port = int(getText(node, 'port', 0))
464 return name, uuid, type, nid, port
466 # extract device attributes for an obd
467 def getNodeAttr(node):
468 name = node.getAttribute('name')
469 uuid = node.getAttribute('uuid')
473 name, uuid = getNodeAttr(obd);
474 obdtype = obd.getAttribute('type')
475 devname, size = getDevice(obd)
476 fstype = getText(obd, 'fstype')
477 format = getText(obd, 'autoformat')
478 return (name, uuid, obdtype, devname, size, fstype, format)
481 def getLOVInfo(node):
482 name, uuid = getNodeAttr(node)
483 devs = node.getElementsByTagName('devices')[0]
484 stripe_sz = int(devs.getAttribute('stripesize'))
485 pattern = int(devs.getAttribute('pattern'))
486 mdcref = node.getElementsByTagName('mdc_ref')[0]
487 mdcuuid = mdcref.getAttribute('uuidref')
488 mdc= lookup(node.parentNode, mdcuuid)
489 mdcname = getName(mdc)
492 for child in devs.childNodes:
493 if child.nodeName == 'obd_ref':
494 devlist = devlist + child.getAttribute('uuidref') + " "
495 strip_cnt = stripe_cnt + 1
496 return (name, uuid, mdcname, stripe_cnt, stripe_sz, pattern, devlist)
498 # extract device attributes for an obd
499 def getMDSInfo(node):
500 name, uuid = getNodeAttr(node)
501 devname, size = getDevice(node)
502 fstype = getText(node, 'fstype')
503 format = getText(node, 'autoformat', "no")
504 return (name, uuid, devname, size, fstype, format)
506 # extract device attributes for an obd
507 def getMDCInfo(node):
508 name, uuid = getNodeAttr(node)
509 ref = node.getElementsByTagName('mds_ref')[0]
510 mdsuuid = ref.getAttribute('uuidref')
511 ref = node.getElementsByTagName('network_ref')[0]
512 netuuid = ref.getAttribute('uuidref')
513 return (name, uuid, mdsuuid, netuuid)
516 # extract device attributes for an obd
517 def getOSTInfo(node):
518 name, uuid = getNodeAttr(node)
519 ref = node.getElementsByTagName('obd_ref')[0]
520 uuid = ref.getAttribute('uuidref')
521 obd = lookup(node.parentNode, uuid)
523 obdname = getOBDInfo(obd)[0]
525 obdname = "OBD NOT FOUND"
526 return (name, uuid, obdname)
528 # extract device attributes for an obd
529 def getOSCInfo(node):
530 name, uuid = getNodeAttr(node)
531 ref = node.getElementsByTagName('obd_ref')[0]
532 obduuid = ref.getAttribute('uuidref')
533 ref = node.getElementsByTagName('network_ref')[0]
534 ostuuid = ref.getAttribute('uuidref')
535 return (name, uuid, obduuid, ostuuid)
538 # Get the text content from the first matching child
539 def getText(node, tag, default=""):
540 list = node.getElementsByTagName(tag)
544 return node.firstChild.data
548 # Recusively search from node for a uuid
549 def lookup(node, uuid):
550 for n in node.childNodes:
551 # this service_id check is ugly. need some other way to
552 # differentiate between definitions and references
553 if n.nodeType == n.ELEMENT_NODE:
554 if getUUID(n) == uuid:
561 # Get name attribute of node
563 return node.getAttribute('name')
566 return node.getAttribute('uuidref')
568 # Get name attribute of node
570 return node.getAttribute('uuid')
572 # the tag name is the service type
573 # fixme: this should do some checks to make sure the node is a service
574 def getServiceType(node):
578 # determine what "level" a particular node is at.
579 # the order of iniitailization is based on level. objects
580 # are assigned a level based on type:
581 # net,devices:1, obd, mdd:2 mds,ost:3 osc,mdc:4 mounts:5
582 def getServiceLevel(node):
583 type = getServiceType(node)
584 if type in ('network', 'device', 'ldlm'):
586 elif type in ('obd', 'mdd'):
588 elif type in ('mds','ost'):
590 elif type in ('mdc','osc'):
592 elif type in ('lov',):
594 elif type in ('mountpoint',):
599 # return list of services in a profile. list is a list of tuples
601 def getServices(lustreNode, profileNode):
603 for n in profileNode.childNodes:
604 if n.nodeType == n.ELEMENT_NODE:
605 servNode = lookup(lustreNode, getRef(n))
607 panic('service not found: ' + getName(n))
608 level = getServiceLevel(servNode)
609 list.append((level, servNode))
613 def getByName(lustreNode, tag, name):
614 ndList = lustreNode.getElementsByTagName(tag)
616 if getName(nd) == name:
621 # ============================================================
622 # lconf type level logic
627 def startService(node):
628 type = getServiceType(node)
629 debug('Starting service:', type, getName(node), getUUID(node))
630 # there must be a more dynamic way of doing this...
635 elif type == 'network':
636 prepare_network(node)
647 elif type == 'mountpoint':
648 prepare_mountpoint(node)
651 # Prepare the system to run lustre using a particular profile
652 # in a the configuration.
653 # * load & the modules
654 # * setup networking for the current node
655 # * make sure partitions are in place and prepared
656 # * initialize devices with lctl
657 # Levels is important, and needs to be enforced.
658 def startProfile(lustreNode, profileNode):
660 panic("profile:", profile, "not found.")
661 services = getServices(lustreNode, profileNode)
667 def stopService(node):
668 type = getServiceType(node)
669 debug('Stopping service:', type, getName(node), getUUID(node))
670 # there must be a more dynamic way of doing this...
675 elif type == 'network':
676 cleanup_network(node)
687 elif type == 'mountpoint':
688 cleanup_mountpoint(node)
690 # Shutdown services in reverse order than they were started
691 def cleanupProfile(lustreNode, profileNode):
693 panic("profile:", profile, "not found.")
694 services = getServices(lustreNode, profileNode)
700 def doHost(lustreNode, hostname, cleanFlag):
701 node = getByName(lustreNode, 'node', hostname)
703 node = getByName(lustreNode, 'node', 'localhost')
705 panic("no node for ", hostname)
706 reflist = node.getElementsByTagName('profile_ref')
709 cleanupProfile(lustreNode, lookup(lustreNode, getRef(r)))
711 startProfile(lustreNode, lookup(lustreNode, getRef(r)))
714 # Command line processing
716 def parse_cmdline(argv):
718 long_opts = ["ldap", "reformat", "lustre=",
719 "portals=", "makeldiff", "cleanup", "iam=",
725 opts, args = getopt.getopt(argv, short_opts, long_opts)
726 except getopt.GetoptError:
731 if o in ("-h", "--help"):
734 options['cleanup'] = 1
735 if o in ("-v", "--verbose"):
736 options['verbose'] = 1
737 if o in ("-d", "--debug"):
739 options['verbose'] = 1
741 options['portals'] = a
743 options['lustre'] = a
744 if o == "--reformat":
745 options['reformat'] = 1
749 # Initialize or shutdown lustre according to a configuration file
750 # * prepare the system for lustre
751 # * configure devices with lctl
752 # Shutdown does steps in reverse
754 lctl = LCTLInterface('lctl')
757 args = parse_cmdline(sys.argv[1:])
759 dom = xml.dom.minidom.parse(args[0])
763 if not options.has_key('hostname'):
764 ret, host = run('hostname')
766 panic("unable to determine hostname")
767 options['hostname'] = host
768 doHost(dom.childNodes[0], options['hostname'], options.has_key('cleanup') )
770 if __name__ == "__main__":