3 # Copyright (C) 2002 Cluster File Systems, Inc.
4 # Author: Robert Read <rread@clusterfs.com>
6 # This file is part of Lustre, http://www.lustre.org.
8 # Lustre is free software; you can redistribute it and/or
9 # modify it under the terms of version 2 of the GNU General Public
10 # License as published by the Free Software Foundation.
12 # Lustre is distributed in the hope that it will be useful,
13 # but WITHOUT ANY WARRANTY; without even the implied warranty of
14 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
15 # GNU General Public License for more details.
17 # You should have received a copy of the GNU General Public License
18 # along with Lustre; if not, write to the Free Software
19 # Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
21 # lconf - lustre configuration tool
23 # lconf is the main driver script for starting and stopping
24 # lustre filesystem services.
26 # Based in part on the XML obdctl modifications done by Brian Behlendorf
29 import string, os, stat, popen2
31 import xml.dom.minidom
34 TCP_ACCEPTOR = 'acceptor'
38 # Maximum number of devices to search for.
39 # (the /dev/loop* nodes need to be created beforehand)
40 MAX_LOOP_DEVICES = 256
44 print """usage: lconf config.xml
46 config.xml Lustre configuration in xml format.
47 -v | --verbose Print system commands as they are run
48 -d | --debug Print system commands, but does not run them
49 --host <hostname> Load config for <hostname>
50 --cleanup Cleans up config. (Shutdown)
51 -h | --help Print this help
54 --ldap server LDAP server with lustre config database
55 --reformat Reformat all devices (will confirm)
56 --lustre="src dir" Base directory of lustre sources. Used to search
58 --portals=src Portals source
59 --makeldiff Translate xml source to LDIFF
64 # ============================================================
65 # debugging and error funcs
67 def fixme(msg = "this feature"):
68 raise RuntimeError, msg + ' not implmemented yet.'
71 msg = string.join(map(str,args))
73 raise RuntimeError, msg
76 msg = string.join(map(str,args))
84 msg = string.join(map(str,args))
85 if isverbose(): print msg
88 return options.has_key('verbose') and options['verbose'] == 1
91 return options.has_key('debug') and options['debug'] == 1
93 # ============================================================
94 # locally defined exceptions
95 class CommandError (exceptions.Exception):
96 def __init__(self, args=None):
99 # ============================================================
100 # handle lctl interface
103 Manage communication with lctl
106 def __init__(self, cmd):
108 Initialize close by finding the lctl binary.
110 syspath = string.split(os.environ['PATH'], ':')
111 syspath.insert(0, "../utils");
114 lctl = os.path.join(d,cmd)
115 if os.access(lctl, os.X_OK):
119 raise RuntimeError, "unable to find lctl binary."
124 the cmds are written to stdin of lctl
125 lctl doesn't return errors when run in script mode, so
127 should modify command line to accept multiple commands, or
128 create complex command line options
130 debug("+", self.lctl, cmds)
131 if isnotouch(): return ([], 0)
132 p = popen2.Popen3(self.lctl, 1)
133 p.tochild.write(cmds + "\n")
135 out = p.fromchild.readlines()
137 err = p.childerr.readlines()
139 log (self.lctl, "error:", ret)
141 raise CommandError, err
145 # create a new device with lctl
146 def network(self, net, nid):
153 # create a new connection
154 def connect(self, net, nid, port, servuuid):
159 quit""" % (net, nid, port, servuuid, nid)
162 # create a new device with lctl
163 def disconnect(self, net, nid, port, servuuid):
170 # create a new device with lctl
171 def newdev(self, attach, setup):
176 quit""" % (attach, setup)
180 def cleanup(self, name, uuid):
189 def lovconfig(self, uuid, mdcuuid, stripe_cnt, stripe_sz, pattern, devlist):
193 lovconfig %s %d %d %s %s
194 quit""" % (mdcuuid, uuid, stripe_cnt, stripe_sz, pattern, devlist)
197 # ============================================================
198 # Various system-level functions
199 # (ideally moved to their own module)
201 # Run a command and return the output and status.
202 # stderr is sent to /dev/null, could use popen3 to
203 # save it if necessary
205 cmd = string.join(map(str,args))
207 if isnotouch(): return ([], 0)
208 f = os.popen(cmd + ' 2>&1')
218 # is the path a block device?
225 return stat.S_ISBLK(s[stat.ST_MODE])
227 # build fs according to type
229 def mkfs(fstype, dev):
230 if(fstype == 'ext3'):
231 mkfs = 'mkfs.ext2 -j -b 4096'
232 elif (fstype == 'extN'):
233 mkfs = 'mkfs.ext2 -j -b 4096'
235 print 'unsupported fs type: ', fstype
236 if not is_block(dev):
240 run (mkfs, force, dev)
242 # some systems use /dev/loopN, some /dev/loop/N
246 if not os.access(loop + str(0), os.R_OK):
248 if not os.access(loop + str(0), os.R_OK):
249 panic ("can't access loop devices")
252 # find loop device assigned to thefile
255 for n in xrange(0, MAX_LOOP_DEVICES):
257 if os.access(dev, os.R_OK):
258 (stat, out) = run('losetup', dev)
259 if (out and stat == 0):
260 m = re.search(r'\((.*)\)', out[0])
261 if m and file == m.group(1):
267 # create file if necessary and assign the first free loop device
268 def init_loop(file, size, fstype):
269 dev = find_loop(file)
271 print 'WARNING file:', file, 'already mapped to', dev
273 if not os.access(file, os.R_OK | os.W_OK):
274 run("dd if=/dev/zero bs=1k count=0 seek=%d of=%s" %(size, file))
276 # find next free loop
277 for n in xrange(0, MAX_LOOP_DEVICES):
279 if os.access(dev, os.R_OK):
280 (stat, out) = run('losetup', dev)
282 run('losetup', dev, file)
285 print "out of loop devices"
287 print "out of loop devices"
290 # undo loop assignment
291 def clean_loop(file):
292 dev = find_loop(file)
294 ret, out = run('losetup -d', dev)
296 log('unable to clean loop device:', dev, 'for file:', file)
299 # initialize a block device if needed
300 def block_dev(dev, size, fstype, format):
301 if isnotouch(): return dev
302 if not is_block(dev):
303 dev = init_loop(dev, size, fstype)
304 if (format == 'yes'):
308 # ============================================================
309 # Functions to prepare the various objects
311 def prepare_ldlm(node):
312 (name, uuid) = getNodeAttr(node)
313 print 'LDLM:', name, uuid
314 lctl.newdev(attach="ldlm %s %s" % (name, uuid),
317 def prepare_lov(node):
318 (name, uuid, mdcuuid, stripe_cnt, strip_sz, pattern, devlist, mdsname) = getLOVInfo(node)
319 print 'LOV:', name, uuid, mdcuuid, stripe_cnt, strip_sz, pattern, devlist, mdsname
320 lctl.lovconfig(uuid, mdsname, stripe_cnt, strip_sz, pattern, devlist)
321 lctl.newdev(attach="lov %s %s" % (name, uuid),
322 setup ="%s" % (mdcuuid))
324 def prepare_network(node):
325 (name, uuid, type, nid, port) = getNetworkInfo(node)
326 print 'NETWORK:', name, uuid, type, nid, port
328 run(TCP_ACCEPTOR, port)
329 lctl.network(type, nid)
332 # need to check /proc/mounts and /etc/mtab before
333 # formatting anything.
334 # FIXME: check if device is already formatted.
335 def prepare_obd(obd):
336 (name, uuid, obdtype, dev, size, fstype, format) = getOBDInfo(obd)
337 print 'OBD:', name, uuid, obdtype, dev, size, fstype, format
338 dev = block_dev(dev, size, fstype, format)
339 lctl.newdev(attach="%s %s %s" % (obdtype, name, uuid),
340 setup ="%s %s" %(dev, fstype))
343 def prepare_ost(ost):
344 name, uuid, obd = getOSTInfo(ost)
345 print 'OST:', name, uuid, obd
346 lctl.newdev(attach="ost %s %s" % (name, uuid),
347 setup ="$%s" % (obd))
349 def prepare_mds(node):
350 (name, uuid, dev, size, fstype, format) = getMDSInfo(node)
351 print 'MDS:', name, uuid, dev, size, fstype
352 # setup network for mds, too
353 dev = block_dev(dev, size, fstype, format)
354 lctl.newdev(attach="mds %s %s" % (name, uuid),
355 setup ="%s %s" %(dev, fstype))
357 def prepare_osc(node):
358 (name, uuid, obduuid, ostuuid) = getOSCInfo(node)
359 print 'OSC:', name, uuid, obduuid, ostuuid
360 net = lookup(node.parentNode, ostuuid)
361 srvname, srvuuid, net, server, port = getNetworkInfo(net)
362 lctl.connect(net, server, port, ostuuid)
363 lctl.newdev(attach="osc %s %s" % (name, uuid),
364 setup ="%s %s" %(obduuid, ostuuid))
366 def prepare_mdc(node):
367 (name, uuid, mdsuuid, netuuid) = getMDCInfo(node)
368 print 'MDC:', name, uuid, mdsuuid, netuuid
369 net = lookup(node.parentNode, netuuid)
370 srvname, srvuuid, net, server, port = getNetworkInfo(net)
371 lctl.connect(net, server, port, netuuid)
372 lctl.newdev(attach="mdc %s %s" % (name, uuid),
373 setup ="%s %s" %(mdsuuid, netuuid))
375 def prepare_mountpoint(node):
376 name, uuid, oscuuid, mdcuuid, mtpt = getMTPTInfo(node)
377 print 'MTPT:', name, uuid, oscuuid, mdcuuid, mtpt
378 cmd = "mount -t lustre_lite -o ost=%s,mds=%s none %s" % \
379 (oscuuid, mdcuuid, mtpt)
382 # ============================================================
383 # Functions to cleanup the various objects
385 def cleanup_ldlm(node):
386 (name, uuid) = getNodeAttr(node)
387 print 'LDLM:', name, uuid
389 lctl.cleanup(name, uuid)
391 print "cleanup failed: ", name
393 def cleanup_lov(node):
394 (name, uuid, mdcuuid, stripe_cnt, strip_sz, pattern, devlist, mdsname) = getLOVInfo(node)
395 print 'LOV:', name, uuid, mdcuuid, stripe_cnt, strip_sz, pattern, devlist, mdsname
397 lctl.cleanup(name, uuid)
399 print "cleanup failed: ", name
401 def cleanup_network(node):
402 (name, uuid, type, nid, port) = getNetworkInfo(node)
403 print 'NETWORK:', name, uuid, type, nid, port
404 #lctl.network(type, nid)
406 # need to check /proc/mounts and /etc/mtab before
407 # formatting anything.
408 # FIXME: check if device is already formatted.
409 def cleanup_obd(obd):
410 (name, uuid, obdtype, dev, size, fstype, format) = getOBDInfo(obd)
411 print "OBD: ", name, obdtype, dev, size, fstype, format
413 lctl.cleanup(name, uuid)
415 print "cleanup failed: ", name
418 def cleanup_ost(ost):
419 name, uuid, obd = getOSTInfo(ost)
420 print "OST: ", name, uuid, obd
422 lctl.cleanup(name, uuid)
424 print "cleanup failed: ", name
426 def cleanup_mds(node):
427 (name, uuid, dev, size, fstype, format) = getMDSInfo(node)
428 print "MDS: ", name, dev, size, fstype
430 lctl.cleanup(name, uuid)
432 print "cleanup failed: ", name
436 def cleanup_mdc(node):
437 (name, uuid, mdsuuid, netuuid) = getMDCInfo(node)
438 print 'MDC:', name, uuid, mdsuuid, netuuid
439 net = lookup(node.parentNode, netuuid)
440 srvname, srvuuid, net, server, port = getNetworkInfo(net)
442 lctl.disconnect(net, server, port, netuuid)
443 lctl.cleanup(name, uuid)
445 print "cleanup failed: ", name
448 def cleanup_osc(node):
449 (name, uuid, obduuid, ostuuid) = getOSCInfo(node)
450 print 'OSC:', name, uuid, obduuid, ostuuid
451 net = lookup(node.parentNode, ostuuid)
452 srvname, srvuuid, net, server, port = getNetworkInfo(net)
454 lctl.disconnect(net, server, port, ostuuid)
455 lctl.cleanup(name, uuid)
457 print "cleanup failed: ", name
459 def cleanup_mountpoint(node):
460 name, uuid, oscuuid, mdcuuid, mtpt = getMTPTInfo(node)
461 print 'MTPT:', name, uuid, oscuuid, mdcuuid, mtpt
464 # ============================================================
465 # XML processing and query
468 dev = obd.getElementsByTagName('device')[0]
471 size = int(dev.getAttribute('size'))
474 return dev.firstChild.data, size
477 def getNetworkInfo(node):
478 name, uuid = getNodeAttr(node);
479 type = node.getAttribute('type')
480 nid = getText(node, 'server', "")
481 port = int(getText(node, 'port', 0))
482 return name, uuid, type, nid, port
484 # extract device attributes for an obd
485 def getNodeAttr(node):
486 name = node.getAttribute('name')
487 uuid = node.getAttribute('uuid')
491 name, uuid = getNodeAttr(obd);
492 obdtype = obd.getAttribute('type')
493 devname, size = getDevice(obd)
494 fstype = getText(obd, 'fstype')
495 format = getText(obd, 'autoformat')
496 return (name, uuid, obdtype, devname, size, fstype, format)
499 def getLOVInfo(node):
500 name, uuid = getNodeAttr(node)
501 devs = node.getElementsByTagName('devices')[0]
502 stripe_sz = int(devs.getAttribute('stripesize'))
503 pattern = int(devs.getAttribute('pattern'))
504 mdcref = node.getElementsByTagName('mdc_ref')[0]
505 mdcuuid = mdcref.getAttribute('uuidref')
506 mdc= lookup(node.parentNode, mdcuuid)
507 mdsref = mdc.getElementsByTagName('mds_ref')[0]
508 mdsuuid = mdsref.getAttribute('uuidref')
509 mds= lookup(node.parentNode, mdsuuid)
510 mdsname = getName(mds)
513 for child in devs.childNodes:
514 if child.nodeName == 'osc_ref':
515 devlist = devlist + child.getAttribute('uuidref') + " "
516 strip_cnt = stripe_cnt + 1
517 return (name, uuid, mdcuuid, stripe_cnt, stripe_sz, pattern, devlist, mdsname)
519 # extract device attributes for an obd
520 def getMDSInfo(node):
521 name, uuid = getNodeAttr(node)
522 devname, size = getDevice(node)
523 fstype = getText(node, 'fstype')
524 format = getText(node, 'autoformat', "no")
525 return (name, uuid, devname, size, fstype, format)
527 # extract device attributes for an obd
528 def getMDCInfo(node):
529 name, uuid = getNodeAttr(node)
530 ref = node.getElementsByTagName('mds_ref')[0]
531 mdsuuid = ref.getAttribute('uuidref')
532 ref = node.getElementsByTagName('network_ref')[0]
533 netuuid = ref.getAttribute('uuidref')
534 return (name, uuid, mdsuuid, netuuid)
537 # extract device attributes for an obd
538 def getOSTInfo(node):
539 name, uuid = getNodeAttr(node)
540 ref = node.getElementsByTagName('obd_ref')[0]
541 uuid = ref.getAttribute('uuidref')
542 obd = lookup(node.parentNode, uuid)
544 obdname = getOBDInfo(obd)[0]
546 obdname = "OBD NOT FOUND"
547 return (name, uuid, obdname)
549 # extract device attributes for an obd
550 def getOSCInfo(node):
551 name, uuid = getNodeAttr(node)
552 ref = node.getElementsByTagName('obd_ref')[0]
553 obduuid = ref.getAttribute('uuidref')
554 ref = node.getElementsByTagName('network_ref')[0]
555 ostuuid = ref.getAttribute('uuidref')
556 return (name, uuid, obduuid, ostuuid)
558 # extract device attributes for an obd
559 def getMTPTInfo(node):
560 name, uuid = getNodeAttr(node)
561 path = getText(node, 'path')
562 ref = node.getElementsByTagName('mdc_ref')[0]
563 mdcuuid = ref.getAttribute('uuidref')
564 ref = node.getElementsByTagName('osc_ref')[0]
565 lovuuid = ref.getAttribute('uuidref')
566 return (name, uuid, lovuuid, mdcuuid, path)
569 # Get the text content from the first matching child
570 def getText(node, tag, default=""):
571 list = node.getElementsByTagName(tag)
575 return node.firstChild.data
579 # Recusively search from node for a uuid
580 def lookup(node, uuid):
581 for n in node.childNodes:
582 # this service_id check is ugly. need some other way to
583 # differentiate between definitions and references
584 if n.nodeType == n.ELEMENT_NODE:
585 if getUUID(n) == uuid:
592 # Get name attribute of node
594 return node.getAttribute('name')
597 return node.getAttribute('uuidref')
599 # Get name attribute of node
601 return node.getAttribute('uuid')
603 # the tag name is the service type
604 # fixme: this should do some checks to make sure the node is a service
605 def getServiceType(node):
609 # determine what "level" a particular node is at.
610 # the order of iniitailization is based on level. objects
611 # are assigned a level based on type:
612 # net,devices,ldlm:1, obd, mdd:2 mds,ost:3 osc,mdc:4 mounts:5
613 def getServiceLevel(node):
614 type = getServiceType(node)
615 if type in ('network', 'device', 'ldlm'):
617 elif type in ('obd', 'mdd'):
619 elif type in ('mds','ost'):
621 elif type in ('mdc','osc'):
623 elif type in ('lov',):
625 elif type in ('mountpoint',):
630 # return list of services in a profile. list is a list of tuples
632 def getServices(lustreNode, profileNode):
634 for n in profileNode.childNodes:
635 if n.nodeType == n.ELEMENT_NODE:
636 servNode = lookup(lustreNode, getRef(n))
638 panic('service not found: ' + getName(n))
639 level = getServiceLevel(servNode)
640 list.append((level, servNode))
644 def getByName(lustreNode, tag, name):
645 ndList = lustreNode.getElementsByTagName(tag)
647 if getName(nd) == name:
652 # ============================================================
653 # lconf type level logic
658 def startService(node):
659 type = getServiceType(node)
660 debug('Starting service:', type, getName(node), getUUID(node))
661 # there must be a more dynamic way of doing this...
666 elif type == 'network':
667 prepare_network(node)
678 elif type == 'mountpoint':
679 prepare_mountpoint(node)
682 # Prepare the system to run lustre using a particular profile
683 # in a the configuration.
684 # * load & the modules
685 # * setup networking for the current node
686 # * make sure partitions are in place and prepared
687 # * initialize devices with lctl
688 # Levels is important, and needs to be enforced.
689 def startProfile(lustreNode, profileNode):
691 panic("profile:", profile, "not found.")
692 services = getServices(lustreNode, profileNode)
698 def stopService(node):
699 type = getServiceType(node)
700 debug('Stopping service:', type, getName(node), getUUID(node))
701 # there must be a more dynamic way of doing this...
706 elif type == 'network':
707 cleanup_network(node)
718 elif type == 'mountpoint':
719 cleanup_mountpoint(node)
721 # Shutdown services in reverse order than they were started
722 def cleanupProfile(lustreNode, profileNode):
724 panic("profile:", profile, "not found.")
725 services = getServices(lustreNode, profileNode)
733 def doHost(lustreNode, hosts, cleanFlag):
735 node = getByName(lustreNode, 'node', h)
739 reflist = node.getElementsByTagName('profile_ref')
742 cleanupProfile(lustreNode, lookup(lustreNode, getRef(r)))
744 startProfile(lustreNode, lookup(lustreNode, getRef(r)))
747 # Command line processing
749 def parse_cmdline(argv):
751 long_opts = ["ldap", "reformat", "lustre=",
752 "portals=", "makeldiff", "cleanup", "iam=",
753 "help", "debug", "host="]
758 opts, args = getopt.getopt(argv, short_opts, long_opts)
759 except getopt.GetoptError:
764 if o in ("-h", "--help"):
767 options['cleanup'] = 1
768 if o in ("-v", "--verbose"):
769 options['verbose'] = 1
770 if o in ("-d", "--debug"):
772 options['verbose'] = 1
774 options['portals'] = a
776 options['lustre'] = a
777 if o == "--reformat":
778 options['reformat'] = 1
780 options['hostname'] = [a]
784 # Initialize or shutdown lustre according to a configuration file
785 # * prepare the system for lustre
786 # * configure devices with lctl
787 # Shutdown does steps in reverse
789 lctl = LCTLInterface('lctl')
792 args = parse_cmdline(sys.argv[1:])
794 dom = xml.dom.minidom.parse(args[0])
798 if not options.has_key('hostname'):
799 ret, host = run('hostname')
801 print "unable to determine hostname"
803 options['hostname'] = [host]
804 options['hostname'].append('localhost')
805 doHost(dom.childNodes[0], options['hostname'], options.has_key('cleanup') )
807 if __name__ == "__main__":