3 # Copyright (C) 2002 Cluster File Systems, Inc.
4 # Author: Robert Read <rread@clusterfs.com>
6 # This file is part of Lustre, http://www.lustre.org.
8 # Lustre is free software; you can redistribute it and/or
9 # modify it under the terms of version 2 of the GNU General Public
10 # License as published by the Free Software Foundation.
12 # Lustre is distributed in the hope that it will be useful,
13 # but WITHOUT ANY WARRANTY; without even the implied warranty of
14 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
15 # GNU General Public License for more details.
17 # You should have received a copy of the GNU General Public License
18 # along with Lustre; if not, write to the Free Software
19 # Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
21 # lconf - lustre configuration tool
23 # lconf is the main driver script for starting and stopping
24 # lustre filesystem services.
26 # Based in part on the XML obdctl modifications done by Brian Behlendorf
29 import string, os, stat, popen2
31 import xml.dom.minidom
34 TCP_ACCEPTOR = 'acceptor'
35 LCTL = './lctl' # fix this...
38 # Maximum number of devices to search for.
39 # (the /dev/loop* nodes need to be created beforehand)
40 MAX_LOOP_DEVICES = 256
44 print """usage: lconf --ldap server | config.xml
46 config.xml Lustre configuration in xml format.
47 --ldap server LDAP server with lustre config database
51 --debug Don't send lctl commenads
52 --reformat Reformat all devices (will confirm)
53 --lustre="src dir" Base directory of lustre sources. Used to search
55 --portals=src Portals source
56 --makeldiff Translate xml source to LDIFF
57 --cleanup Cleans up config. (Shutdown)
60 (SCRIPT STILL UNDER DEVELOPMENT, MOST FUNCTIONALITY UNIMPLEMENTED)
63 # ============================================================
64 # debugging and error funcs
66 def fixme(msg = "this feature"):
67 raise RuntimeError, msg + ' not implmemented yet.'
70 msg = string.join(map(str,args))
72 raise RuntimeError, msg
75 msg = string.join(map(str,args))
83 msg = string.join(map(str,args))
84 if isverbose(): print msg
87 return options.has_key('verbose') and options['verbose'] == 1
90 return options.has_key('debug') and options['debug'] == 1
92 # ============================================================
93 # locally defined exceptions
94 class CommandError (exceptions.Exception):
95 def __init__(self, args=None):
98 # ============================================================
99 # Various system-level functions
100 # (ideally moved to their own module)
102 # Run a command and return the output and status.
103 # stderr is sent to /dev/null, could use popen3 to
104 # save it if necessary
106 cmd = string.join(map(str,args))
108 if isnotouch(): return ([], 0)
109 f = os.popen(cmd + ' 2>&1')
119 # the cmds are written to stdin of lctl
121 debug("+", LCTL, cmds)
122 if isnotouch(): return ([], 0)
123 p = popen2.Popen3(LCTL, 1)
124 p.tochild.write(cmds + "\n")
126 out = p.fromchild.readlines()
129 err = p.childerr.readlines()
130 log (LCTL, "error:", ret)
132 raise CommandError, err
136 # is the path a block device?
143 return stat.S_ISBLK(s[stat.ST_MODE])
145 # build fs according to type
147 def mkfs(fstype, dev):
148 if(fstype == 'ext3'):
149 mkfs = 'mkfs.ext2 -j'
150 elif (fstype == 'extN'):
151 mkfs = 'mkfs.ext2 -j'
153 print 'unsupported fs type: ', fstype
154 if not is_block(dev):
158 run (mkfs, force, dev)
160 # some systems use /dev/loopN, some /dev/loop/N
164 if not os.access(loop + str(0), os.R_OK):
166 if not os.access(loop + str(0), os.R_OK):
167 panic ("can't access loop devices")
170 # find loop device assigned to thefile
173 for n in xrange(0, MAX_LOOP_DEVICES):
175 if os.access(dev, os.R_OK):
176 (stat, out) = run('losetup', dev)
177 if (out and stat == 0):
178 m = re.search(r'\((.*)\)', out[0])
179 if m and file == m.group(1):
185 # create file if necessary and assign the first free loop device
186 def init_loop(file, size, fstype):
187 dev = find_loop(file)
189 print 'WARNING file:', file, 'already mapped to', dev
191 if not os.access(file, os.R_OK | os.W_OK):
192 run("dd if=/dev/zero bs=1k count=0 seek=%d of=%s" %(size, file))
194 # find next free loop
195 for n in xrange(0, MAX_LOOP_DEVICES):
197 if os.access(dev, os.R_OK):
198 (stat, out) = run('losetup', dev)
200 run('losetup', dev, file)
203 print "out of loop devices"
205 print "out of loop devices"
208 # undo loop assignment
209 def clean_loop(file):
210 dev = find_loop(file)
212 ret, out = run('losetup -d', dev)
214 log('unable to clean loop device:', dev, 'for file:', file)
217 # initialize a block device if needed
218 def block_dev(dev, size, fstype, format):
219 if isnotouch(): return dev
220 if not is_block(dev):
221 dev = init_loop(dev, size, fstype)
222 if (format == 'yes'):
226 # create a new device with lctl
227 def lctl_network(net, nid):
234 # create a new device with lctl
235 def lctl_connect(net, server, port, servuuid):
240 quit""" % (net, server, port, servuuid)
243 # create a new device with lctl
244 def lctl_newdev(attach, setup):
249 quit""" % (attach, setup)
252 # ============================================================
253 # Functions to prepare the various objects
255 def prepare_ldlm(node):
256 (name, uuid) = getNodeAttr(node)
257 print 'LDLM:', name, uuid
258 lctl_newdev(attach="ldlm %s %s" % (name, uuid),
262 def prepare_network(node):
263 (type, nid, port) = getNetworkInfo(node)
264 print 'NETWORK:', type, nid, port
266 run(TCP_ACCEPTOR, port)
267 lctl_network(type, nid)
270 # need to check /proc/mounts and /etc/mtab before
271 # formatting anything.
272 # FIXME: check if device is already formatted.
273 def prepare_obd(obd):
274 (name, uuid, obdtype, dev, size, fstype, format) = getOBDInfo(obd)
275 print "OBD: ", name, obdtype, dev, size, fstype, format
276 dev = block_dev(dev, size, fstype, format)
277 lctl_newdev(attach="%s %s %s" % (obdtype, name, uuid),
278 setup ="%s %s" %(dev, fstype))
281 def prepare_ost(ost):
282 name, uuid, obd = getOSTInfo(ost)
283 print "OST: ", name, uuid, obd
284 lctl_newdev(attach="ost %s %s" % (name, uuid),
285 setup ="$%s" % (obd))
287 def prepare_mds(node):
288 (name, uuid, dev, size, fstype, format) = getMDSInfo(node)
289 print "MDS: ", name, dev, size, fstype
290 dev = block_dev(dev, size, fstype, format)
291 lctl_newdev(attach="mds %s %s" % (name, uuid),
292 setup ="%s %s" %(dev, fstype))
294 def prepare_osc(node):
295 (name, uuid, obduuid, srvuuid) = getOSCInfo(node)
296 print 'OSC:', name, uuid, obduuid, srvuuid
297 net = lookup(node.parentNode, srvuuid)
298 net, server, port = getNetworkInfo(net)
299 lctl_connect(net, server, port, srvuuid)
300 lctl_newdev(attach="osc %s %s" % (name, uuid),
301 setup ="%s %s" %(obduuid, srvuuid))
303 def prepare_mdc(node):
306 def prepare_mountpoint(node):
309 # ============================================================
310 # Functions to cleanup the various objects
312 def cleanup_ldlm(node):
313 (name, uuid) = getNodeAttr(node)
314 print 'LDLM:', name, uuid
315 #lctl_newdev(attach="ldlm %s %s" % (name, uuid),
319 def cleanup_network(node):
320 (type, nid, port) = getNetworkInfo(node)
321 print 'NETWORK:', type, nid, port
323 run(TCP_ACCEPTOR, port)
324 #lctl_network(type, nid)
327 # need to check /proc/mounts and /etc/mtab before
328 # formatting anything.
329 # FIXME: check if device is already formatted.
330 def cleanup_obd(obd):
331 (name, uuid, obdtype, dev, size, fstype, format) = getOBDInfo(obd)
332 print "OBD: ", name, obdtype, dev, size, fstype, format
333 #lctl_newdev(attach="%s %s %s" % (obdtype, name, uuid),
334 # setup ="%s %s" %(dev, fstype))
337 def cleanup_ost(ost):
338 name, uuid, obd = getOSTInfo(ost)
339 print "OST: ", name, uuid, obd
340 #lctl_newdev(attach="ost %s %s" % (name, uuid),
341 # setup ="$%s" % (obd))
343 def cleanup_mds(node):
344 (name, uuid, dev, size, fstype, format) = getMDSInfo(node)
345 print "MDS: ", name, dev, size, fstype
346 dev = block_dev(dev, size, fstype, format)
347 #lctl_newdev(attach="mds %s %s" % (name, uuid),
348 # setup ="%s %s" %(dev, fstype))
350 def cleanup_osc(node):
351 (name, uuid, obduuid, srvuuid) = getOSCInfo(node)
352 print 'OSC:', name, uuid, obduuid, srvuuid
353 net = lookup(node.parentNode, srvuuid)
354 net, server, port = getNetworkInfo(net)
355 #lctl_connect(net, server, port, srvuuid)
356 #lctl_newdev(attach="osc %s %s" % (name, uuid),
357 # setup ="%s %s" %(obduuid, srvuuid))
359 def cleanup_mdc(node):
362 def cleanup_mountpoint(node):
365 # ============================================================
366 # XML processing and query
369 dev = obd.getElementsByTagName('device')[0]
372 size = int(dev.getAttribute('size'))
375 return dev.firstChild.data, size
378 def getNetworkInfo(node):
379 type = node.getAttribute('type')
380 nid = getText(node, 'server', "")
381 port = int(getText(node, 'port', 0))
382 return type, nid, port
384 # extract device attributes for an obd
385 def getNodeAttr(node):
386 name = node.getAttribute('name')
387 uuid = node.getAttribute('uuid')
391 name, uuid = getNodeAttr(obd);
392 obdtype = obd.getAttribute('type')
393 devname, size = getDevice(obd)
394 fstype = getText(obd, 'fstype')
395 format = getText(obd, 'autoformat')
396 return (name, uuid, obdtype, devname, size, fstype, format)
398 # extract device attributes for an obd
399 def getMDSInfo(node):
400 name, uuid = getNodeAttr(node)
401 devname, size = getDevice(node)
402 fstype = getText(node, 'fstype')
403 format = getText(node, 'autoformat', "no")
404 return (name, uuid, devname, size, fstype, format)
406 # extract device attributes for an obd
407 def getOSTInfo(node):
408 name, uuid = getNodeAttr(node)
409 ref = node.getElementsByTagName('obd_ref')[0]
410 uuid = ref.getAttribute('uuidref')
411 obd = lookup(node.parentNode, uuid)
413 obdname = getOBDInfo(obd)[0]
415 obdname = "OBD NOT FOUND"
416 return (name, uuid, obdname)
418 # extract device attributes for an obd
419 def getOSCInfo(node):
420 name, uuid = getNodeAttr(node)
421 ref = node.getElementsByTagName('obd_ref')[0]
422 obduuid = ref.getAttribute('uuidref')
423 ref = node.getElementsByTagName('network_ref')[0]
424 ostuuid = ref.getAttribute('uuidref')
425 return (name, uuid, obduuid, ostuuid)
428 # Get the text content from the first matching child
429 def getText(node, tag, default=""):
430 list = node.getElementsByTagName(tag)
434 return node.firstChild.data
438 # Recusively search from node for a uuid
439 def lookup(node, uuid):
440 for n in node.childNodes:
441 # this service_id check is ugly. need some other way to
442 # differentiate between definitions and references
443 if n.nodeType == n.ELEMENT_NODE:
444 if getUUID(n) == uuid:
451 # Get name attribute of node
453 return node.getAttribute('name')
456 return node.getAttribute('uuidref')
458 # Get name attribute of node
460 return node.getAttribute('uuid')
462 # the tag name is the service type
463 # fixme: this should do some checks to make sure the node is a service
464 def getServiceType(node):
468 # determine what "level" a particular node is at.
469 # the order of iniitailization is based on level. objects
470 # are assigned a level based on type:
471 # net,devices:1, obd, mdd:2 mds,ost:3 osc,mdc:4 mounts:5
472 def getServiceLevel(node):
473 type = getServiceType(node)
474 if type in ('network', 'device', 'ldlm'):
476 elif type in ('obd', 'mdd'):
478 elif type in ('mds','ost'):
480 elif type in ('mdc','osc'):
482 elif type in ('mountpoint',):
487 # return list of services in a profile. list is a list of tuples
489 def getServices(lustreNode, profileNode):
491 for n in profileNode.childNodes:
492 if n.nodeType == n.ELEMENT_NODE:
493 servNode = lookup(lustreNode, getRef(n))
495 panic('service not found: ' + getName(n))
496 level = getServiceLevel(servNode)
497 list.append((level, servNode))
501 def getProfile(lustreNode, profile):
502 profList = lustreNode.getElementsByTagName('profile')
503 for prof in profList:
504 if getName(prof) == profile:
508 # ============================================================
509 # lconf type level logic
514 def startService(node):
515 type = getServiceType(node)
516 debug('Starting service:', type, getName(node), getUUID(node))
517 # there must be a more dynamic way of doing this...
520 elif type == 'network':
521 prepare_network(node)
532 elif type == 'mountpoint':
533 prepare_mountpoint(node)
536 # Prepare the system to run lustre using a particular profile
537 # in a the configuration.
538 # * load & the modules
539 # * setup networking for the current node
540 # * make sure partitions are in place and prepared
541 # * initialize devices with lctl
542 # Levels is important, and needs to be enforced.
543 def startProfile(lustreNode, profile):
544 profileNode = getProfile(lustreNode, profile)
546 panic("profile:", profile, "not found.")
547 services = getServices(lustreNode, profileNode)
553 def stopService(node):
554 type = getServiceType(node)
555 debug('Stopping service:', type, getName(node), getUUID(node))
556 # there must be a more dynamic way of doing this...
559 elif type == 'network':
560 cleanup_network(node)
571 elif type == 'mountpoint':
572 cleanup_mountpoint(node)
574 # Shutdown services in reverse order than they were started
575 def cleanupProfile(lustreNode, profile):
576 profileNode = getProfile(lustreNode, profile)
578 panic("profile:", profile, "not found.")
579 services = getServices(lustreNode, profileNode)
585 # Command line processing
587 def parse_cmdline(argv):
589 long_opts = ["ldap", "reformat", "lustre=",
590 "portals=", "makeldiff", "cleanup", "iam=",
596 opts, args = getopt.getopt(argv, short_opts, long_opts)
597 except getopt.GetoptError:
603 if o in ("-h", "--help"):
607 options['cleanup'] = 1
608 if o in ("-v", "--verbose"):
609 options['verbose'] = 1
613 options['portals'] = a
615 options['lustre'] = a
616 if o == "--reformat":
617 options['reformat'] = 1
621 # Initialize or shutdown lustre according to a configuration file
622 # * prepare the system for lustre
623 # * configure devices with lctl
624 # Shutdown does steps in reverse
628 args = parse_cmdline(sys.argv[1:])
630 dom = xml.dom.minidom.parse(args[0])
633 fixme("ldap not implemented yet")
635 if options.has_key('cleanup'):
636 cleanupProfile(dom.childNodes[0], 'local-profile')
638 startProfile(dom.childNodes[0], 'local-profile')
641 # try a different traceback style. (dare ya to try this in java)
642 def my_traceback(file=None):
643 """Print the list of tuples as returned by extract_tb() or
644 extract_stack() as a formatted stack trace to the given file."""
646 (t,v, tb) = sys.exc_info()
647 list = traceback.extract_tb(tb)
650 for filename, lineno, name, line in list:
652 print '%s:%04d %-14s %s' % (filename,lineno, name, line.strip())
654 print '%s:%04d %s' % (filename,lineno, name)
655 print '%s: %s' % (t, v)
657 if __name__ == "__main__":