2 # -*- mode: c; c-basic-offset: 8; indent-tabs-mode: nil; -*-
3 # vim:expandtab:shiftwidth=8:tabstop=8:
5 # Copyright (C) 2002-2003 Cluster File Systems, Inc.
6 # Authors: Robert Read <rread@clusterfs.com>
7 # Mike Shaver <shaver@clusterfs.com>
8 # This file is part of Lustre, http://www.lustre.org.
10 # Lustre is free software; you can redistribute it and/or
11 # modify it under the terms of version 2 of the GNU General Public
12 # License as published by the Free Software Foundation.
14 # Lustre is distributed in the hope that it will be useful,
15 # but WITHOUT ANY WARRANTY; without even the implied warranty of
16 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17 # GNU General Public License for more details.
19 # You should have received a copy of the GNU General Public License
20 # along with Lustre; if not, write to the Free Software
21 # Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
23 # lconf - lustre configuration tool
25 # lconf is the main driver script for starting and stopping
26 # lustre filesystem services.
28 # Based in part on the XML obdctl modifications done by Brian Behlendorf
30 import sys, getopt, types, errno
31 import string, os, stat, popen2, socket, time, random, fcntl, select
32 import re, exceptions, signal, traceback
33 import xml.dom.minidom
35 if sys.version[0] == '1':
36 from FCNTL import F_GETFL, F_SETFL
38 from fcntl import F_GETFL, F_SETFL
40 PYMOD_DIR = ["/usr/lib64/lustre/python", "/usr/lib/lustre/python"]
43 if string.find(sys.platform, 'linux') != -1:
45 elif string.find(sys.platform, 'darwin') != -1:
47 KEXTPATH='/System/Library/Extensions/'
49 PLATFORM='Unsupported'
51 def development_mode():
52 base = os.path.dirname(sys.argv[0])
53 if os.access(base+"/Makefile", os.R_OK):
57 if development_mode():
58 sys.path.append('../utils')
60 sys.path.extend(PYMOD_DIR)
67 # Maximum number of devices to search for.
68 # (the /dev/loop* nodes need to be created beforehand)
69 MAX_LOOP_DEVICES = 256
70 PORTALS_DIR = '../lnet'
72 # Needed to call lconf --record
75 # Please keep these in sync with the values in lnet/include/libcfs/libcfs.h
87 "warning" : (1 << 10),
91 "portals" : (1 << 14), # deprecated
94 "dlmtrace" : (1 << 16),
98 "rpctrace" : (1 << 20),
99 "vfstrace" : (1 << 21),
102 "config" : (1 << 24),
103 "console" : (1 << 25),
109 "undefined" : (1 << 0),
119 "portals" : (1 << 10), # deprecated
121 "nal" : (1 << 11), # deprecated
122 "pinger" : (1 << 12),
123 "filter" : (1 << 13),
124 "ptlbd" : (1 << 14), # deprecated
128 "ptlrouter" : (1 << 18), # deprecated
132 "confobd" : (1 << 22), # deprecated
144 first_cleanup_error = 0
145 def cleanup_error(rc):
146 global first_cleanup_error
147 if not first_cleanup_error:
148 first_cleanup_error = rc
150 # ============================================================
151 # debugging and error funcs
153 def fixme(msg = "this feature"):
154 raise Lustre.LconfError, msg + ' not implemented yet.'
157 msg = string.join(map(str,args))
158 if not config.noexec:
159 raise Lustre.LconfError(msg)
164 msg = string.join(map(str,args))
169 print string.strip(s)
172 # apparently, (non)execution of the following line affects mds device
173 # startup order (e.g. two mds's using loopback devices), so always do it.
174 msg = string.join(map(str,args))
178 # ack, python's builtin int() does not support '0x123' syntax.
179 # eval can do it, although what a hack!
182 if type(s) is types.IntType:
185 if (s[0:2] == '0x') or (s[0:1] == '0'):
186 return eval(s, {}, {})
189 except SyntaxError, e:
190 raise ValueError("not a number")
192 raise ValueError("not a number")
194 raise ValueError("not a number")
196 # ============================================================
197 # locally defined exceptions
198 class CommandError (exceptions.Exception):
199 def __init__(self, cmd_name, cmd_err, rc=None):
200 self.cmd_name = cmd_name
201 self.cmd_err = cmd_err
206 if type(self.cmd_err) == types.StringType:
208 print "! %s (%d): %s" % (self.cmd_name, self.rc, self.cmd_err)
210 print "! %s: %s" % (self.cmd_name, self.cmd_err)
211 elif type(self.cmd_err) == types.ListType:
213 print "! %s (error %d):" % (self.cmd_name, self.rc)
215 print "! %s:" % (self.cmd_name)
216 for s in self.cmd_err:
217 print "> %s" %(string.strip(s))
221 # ============================================================
222 # handle lctl interface
225 Manage communication with lctl
228 def __init__(self, cmd):
230 Initialize close by finding the lctl binary.
232 self.lctl = find_prog(cmd)
234 self.record_device = ''
237 debug('! lctl not found')
240 raise CommandError('lctl', "unable to find lctl binary.")
242 def use_save_file(self, file):
243 self.save_file = file
245 def record(self, dev_name, logname):
246 log("Recording log", logname, "on", dev_name)
247 self.record_device = dev_name
248 self.record_log = logname
250 def end_record(self):
251 log("End recording log", self.record_log, "on", self.record_device)
252 self.record_device = None
253 self.record_log = None
255 def set_nonblock(self, fd):
256 fl = fcntl.fcntl(fd, F_GETFL)
257 fcntl.fcntl(fd, F_SETFL, fl | os.O_NDELAY)
262 the cmds are written to stdin of lctl
263 lctl doesn't return errors when run in script mode, so
265 should modify command line to accept multiple commands, or
266 create complex command line options
270 cmds = '\n dump ' + self.save_file + '\n' + cmds
271 elif self.record_device:
275 %s""" % (self.record_device, self.record_log, cmds)
277 debug("+", cmd_line, cmds)
278 if config.noexec: return (0, [])
280 child = popen2.Popen3(cmd_line, 1) # Capture stdout and stderr from command
281 child.tochild.write(cmds + "\nq\n")
282 child.tochild.close()
284 # From "Python Cookbook" from O'Reilly
285 outfile = child.fromchild
286 outfd = outfile.fileno()
287 self.set_nonblock(outfd)
288 errfile = child.childerr
289 errfd = errfile.fileno()
290 self.set_nonblock(errfd)
292 outdata = errdata = ''
295 ready = select.select([outfd,errfd],[],[]) # Wait for input
296 if outfd in ready[0]:
297 outchunk = outfile.read()
298 if outchunk == '': outeof = 1
299 outdata = outdata + outchunk
300 if errfd in ready[0]:
301 errchunk = errfile.read()
302 if errchunk == '': erreof = 1
303 errdata = errdata + errchunk
304 if outeof and erreof: break
305 # end of "borrowed" code
308 if os.WIFEXITED(ret):
309 rc = os.WEXITSTATUS(ret)
312 if rc or len(errdata):
313 raise CommandError(self.lctl, errdata, rc)
316 def runcmd(self, *args):
318 run lctl using the command line
320 cmd = string.join(map(str,args))
321 debug("+", self.lctl, cmd)
322 rc, out = run(self.lctl, cmd)
324 raise CommandError(self.lctl, out, rc)
327 def unconfigure_network(self):
328 """get lnet to unreference itself"""
330 network unconfigure"""
333 def clear_log(self, dev, log):
334 """ clear an existing log """
339 quit """ % (dev, log)
342 # create a new connection
343 def add_uuid(self, net_type, uuid, nid):
344 if net_type != 'lnet' and string.find(nid,'@') < 0:
345 nidstr = nid + "@" + net_type
348 cmds = "\n add_uuid %s %s" %(uuid, nidstr)
351 def connect(self, srv):
353 panic('nid_uuid not set for ', srv.net_type, srv.nid)
354 hostaddr = srv.db.get_hostaddr()
355 if len(hostaddr) > 1:
356 panic('multiple --hostaddr for ', srv.nid_uuid, ' not supported')
357 elif len(hostaddr) == 1 and hostaddr[0] != srv.nid:
358 panic('different --hostaddr and --nid for ', srv.nid_uuid, ' not supported')
360 self.add_uuid(srv.net_type, srv.nid_uuid, srv.nid)
363 def recover(self, dev_name, new_conn):
366 recover %s""" %(dev_name, new_conn)
369 # disconnect one connection
370 def disconnect(self, srv):
372 panic('nid_uuid not set for ', srv.net_type, srv.nid)
373 self.del_uuid(srv.nid_uuid)
375 def del_uuid(self, uuid):
382 def attach(self, type, name, uuid):
385 quit""" % (type, name, uuid)
388 def setup(self, name, setup = ""):
392 quit""" % (name, setup)
395 def abort_recovery(self, name):
403 def add_conn(self, name, conn_uuid):
407 quit""" % (name, conn_uuid)
410 # create a new device with lctl
411 def newdev(self, type, name, uuid, setup = ""):
412 self.attach(type, name, uuid);
414 self.setup(name, setup)
415 except CommandError, e:
416 self.cleanup(name, uuid, 0)
418 if (config.abort_recovery):
419 if (type == 'obdfilter' or type == 'mds'):
420 self.abort_recovery(name)
423 def cleanup(self, name, uuid, force, failover = 0):
424 if failover: force = 1
430 quit""" % (name, ('', 'force')[force],
431 ('', 'failover')[failover])
435 def lov_setup(self, name, uuid, desc_uuid, mdsuuid, stripe_cnt,
436 stripe_sz, stripe_off, pattern):
439 lov_setup %s %d %d %d %s
440 quit""" % (name, uuid, desc_uuid, stripe_cnt, stripe_sz, stripe_off, pattern)
443 # add an OBD to a LOV
444 def lov_add_obd(self, name, uuid, obd_uuid, index, gen):
447 lov_modify_tgts add %s %s %s %s
448 quit""" % (name, name, obd_uuid, index, gen)
451 # delete an OBD from a LOV
452 def lov_del_obd(self, name, uuid, obd_uuid, index, gen):
455 lov_modify_tgts del %s %s %s %s
456 quit""" % (name, name, obd_uuid, index, gen)
460 def deactivate(self, name):
468 def dump(self, dump_file):
471 quit""" % (dump_file)
474 # get list of devices
475 def device_list(self):
477 if PLATFORM == 'LINUX':
478 devices = '/proc/fs/lustre/devices'
479 if os.access(devices, os.R_OK):
481 fp = open(devices, 'r')
486 elif PLATFORM == 'DARWIN':
487 rc, out = self.run("device_list")
488 ret = out.split("\n")
493 # remove the last empty line
498 def lustre_version(self):
499 rc, out = self.runcmd('version')
503 def mount_option(self, profile, osc, mdc):
505 mount_option %s %s %s
506 quit""" % (profile, osc, mdc)
509 # delete mount options
510 def del_mount_option(self, profile):
516 def set_timeout(self, timeout):
523 def set_lustre_upcall(self, upcall):
528 # ============================================================
529 # Various system-level functions
530 # (ideally moved to their own module)
532 # Run a command and return the output and status.
533 # stderr is sent to /dev/null, could use popen3 to
534 # save it if necessary
537 if config.noexec: return (0, [])
538 f = os.popen(cmd + ' 2>&1')
548 cmd = string.join(map(str,args))
551 # Run a command in the background.
552 def run_daemon(*args):
553 cmd = string.join(map(str,args))
555 if config.noexec: return 0
556 f = os.popen(cmd + ' 2>&1')
564 # Determine full path to use for an external command
565 # searches dirname(argv[0]) first, then PATH
567 syspath = string.split(os.environ['PATH'], ':')
568 cmdpath = os.path.dirname(sys.argv[0])
569 syspath.insert(0, cmdpath);
571 syspath.insert(0, os.path.join(config.portals, 'utils/'))
573 prog = os.path.join(d,cmd)
574 if os.access(prog, os.X_OK):
578 # Recursively look for file starting at base dir
579 def do_find_file(base, mod):
580 fullname = os.path.join(base, mod)
581 if os.access(fullname, os.R_OK):
583 for d in os.listdir(base):
584 dir = os.path.join(base,d)
585 if os.path.isdir(dir):
586 module = do_find_file(dir, mod)
590 def find_module(src_dir, dev_dir, modname):
591 modbase = src_dir +'/'+ dev_dir +'/'+ modname
592 for modext in '.ko', '.o':
593 module = modbase + modext
595 if os.access(module, os.R_OK):
601 # is the path a block device?
608 return stat.S_ISBLK(s[stat.ST_MODE])
610 def my_realpath(path):
612 if os.path.islink(path):
613 # get the realpath of the mount point path
614 if 'realpath' in dir(os.path):
615 real_path = os.path.realpath(path)
619 while os.path.islink(real_path) and (link_count < 20):
620 link_count = link_count + 1
621 path_link = os.readlink(real_path)
622 if os.path.isabs(path_link):
623 real_path = path_link
625 real_path = os.path.join(os.path.dirname(real_path), path_link)
627 panic("Encountered too many symbolic links resolving path:", path)
633 panic("Fatal error realpath()ing path:", path)
636 # build fs according to type
638 def mkfs(dev, devsize, fstype, jsize, isize, mkfsoptions, isblock=1):
644 panic("size of filesystem on '%s' must be larger than 8MB, but is set to %s"%
646 # devsize is in 1k, and fs block count is in 4k
647 block_cnt = devsize/4
649 if fstype in ('ext3', 'ldiskfs'):
650 # ext3 journal size is in megabytes
653 if not is_block(dev):
654 ret, out = runcmd("ls -l %s" %dev)
655 devsize = int(string.split(out[0])[4]) / 1024
657 # sfdisk works for symlink, hardlink, and realdev
658 ret, out = runcmd("sfdisk -s %s" %dev)
660 devsize = int(out[0])
662 # sfdisk -s will fail for too large block device,
663 # then, read the size of partition from /proc/partitions
665 # get the realpath of the device
666 # it may be the real device, such as /dev/hda7
667 # or the hardlink created via mknod for a device
668 real_dev = my_realpath(dev)
670 # get the major and minor number of the realpath via ls
671 # it seems python(os.stat) does not return
672 # the st_rdev member of the stat structure
673 ret, out = runcmd("ls -l %s" %real_dev)
674 major = string.split(string.split(out[0])[4], ",")[0]
675 minor = string.split(out[0])[5]
677 # get the devsize from /proc/partitions with the major and minor number
678 ret, out = runcmd("cat /proc/partitions")
681 if string.split(line)[0] == major and string.split(line)[1] == minor:
682 devsize = int(string.split(line)[2])
685 if devsize > 1024 * 1024:
686 jsize = ((devsize / 102400) * 4)
689 if jsize: jopt = "-J size=%d" %(jsize,)
690 if isize: iopt = "-I %d" %(isize,)
691 mkfs = 'mkfs.ext2 -j -b 4096 '
692 if not isblock or config.force:
694 elif fstype == 'reiserfs':
695 # reiserfs journal size is in blocks
696 if jsize: jopt = "--journal_size %d" %(jsize,)
697 mkfs = 'mkreiserfs -ff'
699 panic('unsupported fs type: ', fstype)
701 if config.mkfsoptions != None:
702 mkfs = mkfs + ' ' + config.mkfsoptions
703 if mkfsoptions != None:
704 mkfs = mkfs + ' ' + mkfsoptions
705 (ret, out) = run (mkfs, jopt, iopt, dev, block_cnt)
707 panic("Unable to build fs:", dev, string.join(out))
708 # enable hash tree indexing on fsswe
709 if fstype in ('ext3', 'ldiskfs'):
710 htree = 'tune2fs -O dir_index'
711 (ret, out) = run (htree, dev)
713 panic("Unable to enable htree:", dev)
715 # some systems use /dev/loopN, some /dev/loop/N
719 if not os.access(loop + str(0), os.R_OK):
721 if not os.access(loop + str(0), os.R_OK):
725 # find loop device assigned to the file
728 for n in xrange(0, MAX_LOOP_DEVICES):
730 if os.access(dev, os.R_OK):
731 (stat, out) = run('losetup', dev)
732 if out and stat == 0:
733 m = re.search(r'\((.*)\)', out[0])
734 if m and file == m.group(1):
740 # create file if necessary and assign the first free loop device
741 def init_loop(file, size, fstype, journal_size, inode_size, mkfsoptions, reformat):
742 dev = find_loop(file)
744 print 'WARNING file:', file, 'already mapped to', dev
746 if reformat or not os.access(file, os.R_OK | os.W_OK):
748 panic("size of loopback file '%s' must be larger than 8MB, but is set to %s" % (file,size))
749 (ret, out) = run("dd if=/dev/zero bs=1k count=0 seek=%d of=%s" %(size,
752 panic("Unable to create backing store:", file)
753 mkfs(file, size, fstype, journal_size, inode_size, mkfsoptions, isblock=0)
756 # find next free loop
757 for n in xrange(0, MAX_LOOP_DEVICES):
759 if os.access(dev, os.R_OK):
760 (stat, out) = run('losetup', dev)
762 (stat, out) = run('losetup', dev, file)
764 panic("losetup failed: (%s) %s" % (stat, out[0].strip()))
767 print "out of loop devices"
769 print "out of loop devices"
772 # undo loop assignment
773 def clean_loop(file):
774 dev = find_loop(file)
776 ret, out = run('losetup -d', dev)
778 log('unable to clean loop device:', dev, 'for file:', file)
781 # determine if dev is formatted as a <fstype> filesystem
782 def need_format(fstype, dev):
783 # FIXME don't know how to implement this
786 # initialize a block device if needed
787 def block_dev(dev, size, fstype, reformat, autoformat, journal_size,
788 inode_size, mkfsoptions):
789 if config.noexec: return dev
790 if not is_block(dev):
791 dev = init_loop(dev, size, fstype, journal_size, inode_size,
792 mkfsoptions, reformat)
793 elif reformat or (need_format(fstype, dev) and autoformat == 'yes'):
794 mkfs(dev, size, fstype, journal_size, inode_size, mkfsoptions,
797 # panic("device:", dev,
798 # "not prepared, and autoformat is not set.\n",
799 # "Rerun with --reformat option to format ALL filesystems")
803 """lookup IP address for an interface"""
804 rc, out = run("/sbin/ifconfig", iface)
807 addr = string.split(out[1])[1]
808 ip = string.split(addr, ':')[1]
811 def def_mount_options(fstype, target, blkdev):
812 """returns deafult mount options for passed fstype and target (mds, ost)"""
813 if fstype == 'ext3' or fstype == 'ldiskfs':
814 mountfsoptions = "errors=remount-ro"
816 if sys_get_branch() == '2.4':
817 mountfsoptions = "%s,asyncdel" % (mountfsoptions)
819 # mountfsoptions = "%s,extents,mballoc" % (mountfsoptions)
820 elif target == 'mds':
821 if config.user_xattr:
822 mountfsoptions = "%s,user_xattr" % (mountfsoptions)
824 mountfsoptions = "%s,acl" % (mountfsoptions)
827 # grab superblock info
828 dumpe2fs="dumpe2fs -f -h"
829 (ret, sb) = run(dumpe2fs, blkdev)
831 panic("unable to get superblock for ", blkdev)
833 # extract journal UUID
837 lst = string.split(line, ":")
838 if lst[0] == 'Journal UUID':
840 panic("cannot retrieve journal UUID for ", blkdev)
841 if string.split(lst[1])[0] != '<none>':
842 journal_UUID = string.split(lst[1])[0]
843 debug(blkdev, 'has journal UUID', journal_UUID)
844 if lst[0] == 'Journal device':
846 panic("cannot retrieve journal device for ", blkdev)
847 if string.split(lst[1])[0] != '0x0000':
848 journal_DEV = string.split(lst[1])[0]
849 debug(blkdev, 'has journal device', journal_DEV)
852 if len(journal_UUID) == 0 or len(journal_DEV) == 0:
853 debug('no external journal found for', blkdev)
854 # use internal journal
855 return mountfsoptions
857 # run blkid, lookup highest-priority device with matching UUID
858 blkid = "blkid -o device -l -t UUID='%s'" % (journal_UUID)
859 (ret, devname) = run(blkid)
860 if ret or len(devname) == 0:
861 panic("cannot find external journal for ", blkdev)
862 debug('found', blkdev, 'journal UUID', journal_UUID, 'on',
863 string.replace(devname[0], '\n', ''))
865 try: # sigh, python 1.5 does not support os.stat().st_rdev
866 jdevpath = my_realpath(string.replace(devname[0], '\n', ''))
867 ret, out = runcmd("ls -l %s" %jdevpath)
869 major = int(string.split(string.split(out[0])[4], ',')[0])
870 minor = int(string.split(out[0])[5])
871 debug('major', major, 'minor', minor)
872 rdev = major << 8 | minor
874 panic("cannot stat ", devname[0])
876 debug('found', blkdev, 'journal UUID', journal_UUID, 'on',
877 jdevpath, 'rdev', rdev)
880 if string.atoi(journal_DEV, 0) != rdev:
881 mountfsoptions = "%s,journal_dev=%#x" % (mountfsoptions,rdev)
883 return mountfsoptions
886 def sys_get_branch():
887 """Returns kernel release"""
888 return os.uname()[2][:3]
890 def mod_loaded(modname):
891 """Check if a module is already loaded. Look in /proc/modules for it."""
892 if PLATFORM == 'LINUX':
894 fp = open('/proc/modules')
895 lines = fp.readlines()
897 # please forgive my tired fingers for this one
898 ret = filter(lambda word, mod=modname: word == mod,
899 map(lambda line: string.split(line)[0], lines))
903 elif PLATFORM == 'DARWIN':
904 ret, out = run('/usr/sbin/kextstat | /usr/bin/grep', modname)
912 # XXX: instead of device_list, ask for $name and see what we get
913 def is_prepared(name):
914 """Return true if a device exists for the name"""
917 if (config.noexec or config.record) and config.cleanup:
920 # expect this format:
921 # 1 UP ldlm ldlm ldlm_UUID 2
922 out = lctl.device_list()
924 if name == string.split(s)[3]:
926 except CommandError, e:
930 def is_network_prepared():
931 """If the any device exists, then assume that all networking
932 has been configured"""
933 out = lctl.device_list()
936 def fs_is_mounted(path):
937 """Return true if path is a mounted lustre filesystem"""
939 real_path = my_realpath(path)
941 fp = open('/proc/mounts')
942 lines = fp.readlines()
946 if a[1] == real_path and a[2] == 'lustre_lite':
953 """Manage kernel modules"""
954 def __init__(self, lustre_dir, portals_dir):
955 self.lustre_dir = lustre_dir
956 self.portals_dir = portals_dir
957 self.kmodule_list = []
959 def add_portals_module(self, dev_dir, modname):
960 """Append a module to list of modules to load."""
961 self.kmodule_list.append((self.portals_dir, dev_dir, modname))
963 def add_lustre_module(self, dev_dir, modname):
964 """Append a module to list of modules to load."""
965 self.kmodule_list.append((self.lustre_dir, dev_dir, modname))
967 def load_module(self):
968 """Load all the modules in the list in the order they appear."""
969 for src_dir, dev_dir, mod in self.kmodule_list:
970 if mod_loaded(mod) and not config.noexec:
972 log ('loading module:', mod, 'srcdir', src_dir, 'devdir', dev_dir)
973 if PLATFORM == 'LINUX':
976 #For LNET we really need modprobe to load defined LNDs
977 run('/sbin/modprobe lnet')
978 #But if that fails, try insmod anyhow with dev option
979 #accept=all for dev liblustre testing
980 options = 'accept=all'
982 module = find_module(src_dir, dev_dir, mod)
984 panic('module not found:', mod)
985 (rc, out) = run('/sbin/insmod', module, options)
986 if rc and not mod_loaded(mod):
988 print("Bad module options? Check dmesg.")
989 raise CommandError('insmod', out, rc)
991 (rc, out) = run('/sbin/modprobe', mod)
992 if rc and not mod_loaded(mod):
994 print("Bad module options? Check dmesg.")
995 raise CommandError('modprobe', out, rc)
996 elif PLATFORM == 'DARWIN':
997 run('/sbin/kextload', KEXTPATH + mod + '.kext');
999 def cleanup_module(self):
1000 """Unload the modules in the list in reverse order."""
1002 rev = self.kmodule_list[:] # make *copy* of list
1004 for src_dir, dev_dir, mod in rev:
1005 if not mod_loaded(mod) and not config.noexec:
1007 if mod == 'ksocklnd' and not config.noexec:
1008 # Ignore ksocklnd in module list (lnet will remove)
1010 log('unloading module:', mod)
1011 if mod == 'lnet' and not config.noexec:
1012 # remove any self-ref portals created
1013 lctl.unconfigure_network()
1015 debug('dumping debug log to', config.dump)
1017 lctl.dump(config.dump)
1018 log('unloading the network')
1019 lctl.unconfigure_network()
1020 if mod_loaded("ksocklnd"):
1021 if PLATFORM == 'LINUX':
1022 run('/sbin/rmmod ksocklnd')
1023 elif PLATFORM == 'DARWIN':
1024 run('/sbin/kextunload', KEXTPATH+'ksocklnd.kext')
1025 if mod_loaded("kqswlnd"):
1026 run('/sbin/rmmod kqswlnd')
1027 if mod_loaded("kgmlnd"):
1028 run('/sbin/rmmod kgmlnd')
1029 if mod_loaded("kopeniblnd"):
1030 run('/sbin/rmmod kopeniblnd')
1031 if mod_loaded("kiiblnd"):
1032 run('/sbin/rmmod kiiblnd')
1033 if mod_loaded("kviblnd"):
1034 run('/sbin/rmmod kviblnd')
1035 if mod_loaded("kciblnd"):
1036 run('/sbin/rmmod kciblnd')
1037 if mod_loaded("ko2iblnd"):
1038 run('/sbin/rmmod ko2iblnd')
1039 if mod_loaded("kralnd"):
1040 run('/sbin/rmmod kralnd')
1041 if mod_loaded("kptllnd"):
1042 run('/sbin/rmmod kptllnd')
1043 if PLATFORM == 'LINUX':
1044 (rc, out) = run('/sbin/rmmod', mod)
1045 elif PLATFORM == 'DARWIN':
1046 (rc, out) = run('/sbin/kextunload', KEXTPATH+mod+'.kext');
1048 log('! unable to unload module:', mod)
1052 # ============================================================
1053 # Classes to prepare and cleanup the various objects
1056 """ Base class for the rest of the modules. The default cleanup method is
1057 defined here, as well as some utilitiy funcs.
1059 def __init__(self, module_name, db):
1061 self.module_name = module_name
1062 self.name = self.db.getName()
1063 self.uuid = self.db.getUUID()
1066 self.kmod = kmod(config.lustre, config.portals)
1068 def info(self, *args):
1069 msg = string.join(map(str,args))
1070 log (self.module_name + ":", self.name, self.uuid, msg)
1073 """ default cleanup, used for most modules """
1076 lctl.cleanup(self.name, self.uuid, config.force)
1077 except CommandError, e:
1078 log(self.module_name, "cleanup failed: ", self.name)
1082 def add_portals_module(self, dev_dir, modname):
1083 """Append a module to list of modules to load."""
1084 self.kmod.add_portals_module(dev_dir, modname)
1086 def add_lustre_module(self, dev_dir, modname):
1087 """Append a module to list of modules to load."""
1088 self.kmod.add_lustre_module(dev_dir, modname)
1090 def load_module(self):
1091 """Load all the modules in the list in the order they appear."""
1092 self.kmod.load_module()
1094 def cleanup_module(self):
1095 """Unload the modules in the list in reverse order."""
1096 if self.safe_to_clean():
1097 self.kmod.cleanup_module()
1099 def safe_to_clean(self):
1102 def safe_to_clean_modules(self):
1103 return self.safe_to_clean()
1105 class Network(Module):
1106 def __init__(self,db,nid_uuid=0):
1107 Module.__init__(self, 'NETWORK', db)
1108 self.net_type = self.db.get_val('nettype')
1109 self.nid = self.db.get_val('nid', '*')
1110 self.cluster_id = self.db.get_val('clusterid', "0")
1111 self.port = self.db.get_val_int('port', 0)
1112 self.nid_uuid = nid_uuid
1113 self.add_portals_module('libcfs', 'libcfs')
1114 self.add_portals_module('lnet', 'lnet')
1115 # Add the socklnd for developers without modprobe.conf (umls)
1116 self.add_portals_module('klnds/socklnd', 'ksocklnd')
1119 if is_network_prepared():
1121 self.info(self.net_type, self.nid)
1122 if self.net_type == 'tcp':
1124 if self.net_type == 'elan':
1127 def safe_to_clean(self):
1128 if PLATFORM == 'LINUX':
1129 return not is_network_prepared()
1130 elif PLATFORM == 'DARWIN':
1131 # XXX always assume it's safe to clean
1136 self.info(self.net_type, self.nid)
1138 # This is only needed to load the modules; the LDLM device
1139 # is now created automatically.
1141 def __init__(self,db):
1142 Module.__init__(self, 'LDLM', db)
1143 self.add_lustre_module('lvfs', 'lvfs')
1144 self.add_lustre_module('obdclass', 'obdclass')
1145 self.add_lustre_module('ptlrpc', 'ptlrpc')
1146 self.add_lustre_module('ptlrpc/gss', 'ptlrpc_gss')
1155 def __init__(self, db, uuid, fs_name, name_override = None, config_only = None):
1156 Module.__init__(self, 'LOV', db)
1157 if name_override != None:
1158 self.name = "lov_%s" % name_override
1159 self.add_lustre_module('lov', 'lov')
1160 self.mds_uuid = self.db.get_first_ref('mds')
1161 self.stripe_sz = self.db.get_val_int('stripesize', 1048576)
1162 self.stripe_off = self.db.get_val_int('stripeoffset', 0)
1163 self.pattern = self.db.get_val_int('stripepattern', 0)
1165 self.stripe_cnt = self.db.get_val_int('stripecount', 1)
1167 self.desc_uuid = self.uuid
1168 self.uuid = generate_client_uuid(self.name)
1169 self.fs_name = fs_name
1170 # settings below here won't be seen by the MDSDEV code!
1172 self.config_only = 1
1174 self.config_only = None
1175 mds = self.db.lookup(self.mds_uuid)
1176 self.mds_name = mds.getName()
1177 self.devlist = self.db.get_lov_tgts('lov_tgt')
1178 for (obd_uuid, index, gen, active) in self.devlist:
1181 obd = self.db.lookup(obd_uuid)
1182 osc = get_osc(obd, self.uuid, fs_name)
1184 self.osclist.append((osc, index, gen, active))
1186 panic('osc not found:', obd_uuid)
1187 if self.osclist == []:
1188 debug("get_lov_tgts failed, using get_refs");
1190 self.devlist = self.db.get_refs('obd')
1191 for obd_uuid in self.devlist:
1192 obd = self.db.lookup(obd_uuid)
1193 osc = get_osc(obd, self.uuid, fs_name)
1195 self.osclist.append((osc, index, 1, 1))
1197 panic('osc not found:', obd_uuid)
1199 if self.osclist == []:
1200 panic('No OSCs configured for LOV')
1201 debug('dbg LOV __init__:', self.osclist, self.devlist, self.stripe_cnt)
1204 debug('dbg LOV prepare')
1205 if is_prepared(self.name):
1207 debug('dbg LOV prepare:', self.osclist, self.devlist)
1208 self.info(self.mds_uuid, self.stripe_cnt, self.stripe_sz,
1209 self.stripe_off, self.pattern, self.devlist,
1211 lctl.lov_setup(self.name, self.uuid,
1212 self.desc_uuid, self.mds_name, self.stripe_cnt,
1213 self.stripe_sz, self.stripe_off, self.pattern)
1214 if self.osclist == []:
1215 panic('No OSCs configured for LOV?')
1216 for (osc, index, gen, active) in self.osclist:
1217 target_uuid = osc.target_uuid
1219 # Only ignore connect failures with --force, which
1220 # isn't implemented here yet.
1222 osc.prepare(ignore_connect_failure=0)
1223 except CommandError, e:
1224 print "Error preparing OSC %s\n" % osc.uuid
1226 lctl.lov_add_obd(self.name, self.uuid, target_uuid, index, gen)
1229 if is_prepared(self.name):
1230 Module.cleanup(self)
1231 for (osc, index, gen, active) in self.osclist:
1233 if self.config_only:
1234 panic("Can't clean up config_only LOV ", self.name)
1236 def load_module(self):
1237 if self.config_only:
1238 panic("Can't load modules for config_only LOV ", self.name)
1239 for (osc, index, gen, active) in self.osclist:
1242 Module.load_module(self)
1244 def cleanup_module(self):
1245 if self.config_only:
1246 panic("Can't cleanup modules for config_only LOV ", self.name)
1247 Module.cleanup_module(self)
1248 for (osc, index, gen, active) in self.osclist:
1250 osc.cleanup_module()
1253 class MDSDEV(Module):
1254 def __init__(self,db):
1255 Module.__init__(self, 'MDSDEV', db)
1256 self.devpath = self.db.get_val('devpath','')
1257 self.size = self.db.get_val_int('devsize', 0)
1258 self.journal_size = self.db.get_val_int('journalsize', 0)
1260 self.fstype = self.db.get_val('fstype', '')
1261 if sys_get_branch() == '2.4' and self.fstype == 'ldiskfs':
1262 self.fstype = 'ext3'
1263 elif sys_get_branch() == '2.6' and self.fstype == 'ext3':
1264 self.fstype = 'ldiskfs'
1266 self.nspath = self.db.get_val('nspath', '')
1267 self.mkfsoptions = '-i 4096 ' + self.db.get_val('mkfsoptions', '')
1268 self.mountfsoptions = self.db.get_val('mountfsoptions', '')
1270 self.quota = config.quota
1272 self.quota = self.db.get_val('quota', '')
1273 # overwrite the orignal MDSDEV name and uuid with the MDS name and uuid
1274 target_uuid = self.db.get_first_ref('target')
1275 mds = self.db.lookup(target_uuid)
1276 self.name = mds.getName()
1277 self.filesystem_uuids = mds.get_refs('filesystem')
1278 # FIXME: if fstype not set, then determine based on kernel version
1279 self.format = self.db.get_val('autoformat', "no")
1280 if mds.get_val('failover', '1') != '0':
1281 self.failover_mds = 'f'
1283 self.failover_mds = 'n'
1284 active_uuid = get_active_target(mds)
1286 panic("No target device found:", target_uuid)
1287 if active_uuid == self.uuid:
1291 if self.active and config.group and config.group != mds.get_val('group', mds.get_val('name')):
1294 self.inode_size = self.db.get_val_int('inodesize', 0)
1295 debug('original inode_size ', self.inode_size)
1296 if self.inode_size == 0:
1297 # find the LOV for this MDS
1298 lovconfig_uuid = mds.get_first_ref('lovconfig')
1299 if not lovconfig_uuid:
1300 panic("No LOV config found for MDS ", mds.name)
1301 lovconfig = mds.lookup(lovconfig_uuid)
1302 lov_uuid = lovconfig.get_first_ref('lov')
1304 panic("No LOV found for lovconfig ", lovconfig.name)
1305 lov = LOV(self.db.lookup(lov_uuid), lov_uuid, 'FS_name', config_only = 1)
1307 # default stripe count controls default inode_size
1308 if (lov.stripe_cnt > 0):
1309 stripe_count = lov.stripe_cnt
1312 if stripe_count > 77:
1313 self.inode_size = 512
1314 elif stripe_count > 34:
1315 self.inode_size = 2048
1316 elif stripe_count > 13:
1317 self.inode_size = 1024
1318 #elif stripe_count < 3:
1319 # self.inode_size = 256
1321 self.inode_size = 512
1322 debug('stripe_count ', stripe_count,' inode_size ',self.inode_size)
1324 self.target_dev_uuid = self.uuid
1325 self.uuid = target_uuid
1329 self.add_lustre_module('quota', 'lquota')
1330 self.add_lustre_module('mdc', 'mdc')
1331 self.add_lustre_module('osc', 'osc')
1332 self.add_lustre_module('lov', 'lov')
1333 self.add_lustre_module('mds', 'mds')
1334 if self.fstype == 'ldiskfs':
1335 self.add_lustre_module('ldiskfs', 'ldiskfs')
1337 self.add_lustre_module('lvfs', 'fsfilt_%s' % (self.fstype))
1339 def load_module(self):
1341 Module.load_module(self)
1344 if is_prepared(self.name):
1347 debug(self.uuid, "not active")
1350 # run write_conf automatically, if --reformat used
1352 self.info(self.devpath, self.fstype, self.size, self.format)
1353 # never reformat here
1354 blkdev = block_dev(self.devpath, self.size, self.fstype, 0,
1355 self.format, self.journal_size, self.inode_size,
1357 if not is_prepared('MDT'):
1358 lctl.newdev("mdt", 'MDT', 'MDT_UUID', setup ="")
1360 mountfsoptions = def_mount_options(self.fstype, 'mds', blkdev)
1362 if config.mountfsoptions:
1364 mountfsoptions = mountfsoptions + ',' + config.mountfsoptions
1366 mountfsoptions = config.mountfsoptions
1367 if self.mountfsoptions:
1368 mountfsoptions = mountfsoptions + ',' + self.mountfsoptions
1370 if self.mountfsoptions:
1372 mountfsoptions = mountfsoptions + ',' + self.mountfsoptions
1374 mountfsoptions = self.mountfsoptions
1376 print 'MDS mount options: ' + mountfsoptions
1378 lctl.newdev("mds", self.name, self.uuid,
1379 setup ="%s %s %s %s %s" %(blkdev, self.fstype, self.name,
1380 mountfsoptions, self.quota))
1381 self.group_upcall = self.db.get_val('group_upcall','')
1382 sys_set_group_upcall(self.name, self.group_upcall)
1384 except CommandError, e:
1386 panic("MDS failed to start. Check the syslog for details." +
1387 " (May need to run lconf --write-conf)")
1391 def write_conf(self):
1392 if is_prepared(self.name):
1394 self.info(self.devpath, self.fstype, self.format)
1395 blkdev = block_dev(self.devpath, self.size, self.fstype,
1396 config.reformat, self.format, self.journal_size,
1397 self.inode_size, self.mkfsoptions)
1398 lctl.newdev("mds", self.name, self.uuid,
1399 setup ="%s %s" %(blkdev, self.fstype))
1401 # record logs for the MDS lov
1402 for uuid in self.filesystem_uuids:
1403 log("recording clients for filesystem:", uuid)
1404 fs = self.db.lookup(uuid)
1405 obd_uuid = fs.get_first_ref('obd')
1406 client_uuid = generate_client_uuid(self.name)
1407 client = VOSC(self.db.lookup(obd_uuid), client_uuid, self.name,
1410 lctl.clear_log(self.name, self.name)
1411 lctl.record(self.name, self.name)
1413 lctl.mount_option(self.name, client.get_name(), "")
1417 # record logs for each client
1419 config_options = "--ldapurl " + config.ldapurl + " --config " + config.config
1421 config_options = CONFIG_FILE
1423 for node_db in self.db.lookup_class('node'):
1424 client_name = node_db.getName()
1425 for prof_uuid in node_db.get_refs('profile'):
1426 prof_db = node_db.lookup(prof_uuid)
1427 # refactor this into a funtion to test "clientness" of a node.
1428 for ref_class, ref_uuid in prof_db.get_all_refs():
1429 if ref_class in ('mountpoint','echoclient'):
1430 thing = self.db.lookup(ref_uuid);
1431 fs_uuid = thing.get_first_ref('filesystem')
1432 if not fs_uuid in self.filesystem_uuids:
1435 log("Recording log", client_name, "on", self.name)
1436 old_noexec = config.noexec
1438 noexec_opt = ('', '-n')
1439 ret, out = run (sys.argv[0],
1440 noexec_opt[old_noexec == 1],
1441 " -v --record --nomod --old_conf",
1442 "--record_log", client_name,
1443 "--record_device", self.name,
1444 "--node", client_name,
1447 lctl.clear_log(self.name, client_name)
1450 panic("Record client log %s on %s failed" %(
1451 client_name, self.name))
1453 for s in out: log("record> ", string.strip(s))
1454 config.noexec = old_noexec
1456 lctl.cleanup(self.name, self.uuid, config.force, config.failover)
1457 except CommandError, e:
1458 log(self.module_name, "cleanup failed: ", self.name)
1461 Module.cleanup(self)
1462 clean_loop(self.devpath)
1464 #change the mtime of LLOG to match the XML creation time
1465 if toplustreDB.get_mtime():
1466 mtime = toplustreDB.get_mtime()
1467 debug("changing mtime of LOGS to %s" %mtime)
1468 ret, mktemp = runcmd("mktemp /tmp/lustre-cmd.XXXXXXXX")
1470 log(self.module_name, "create mtime LOGS cmdfile failed: ", self.name)
1472 mtimecmdfile = string.split(mktemp[0])[0]
1473 fd = os.open(mtimecmdfile, os.O_RDWR | os.O_CREAT)
1474 os.write(fd, "\n\n\n\n\n%s\n\n" %mtime)
1476 cmd = "debugfs -w -R \"mi /LOGS\" <%s %s" %(mtimecmdfile, self.devpath)
1477 ret, outs = runcmd(cmd)
1478 os.remove(mtimecmdfile)
1480 print "Can not change mtime of LOGS by debugfs."
1482 def mds_remaining(self):
1483 out = lctl.device_list()
1485 if string.split(s)[2] in ('mds',):
1486 if string.split(s)[1] in ('ST',):
1490 def safe_to_clean(self):
1493 def safe_to_clean_modules(self):
1494 return not self.mds_remaining()
1498 debug(self.uuid, "not active")
1501 if is_prepared(self.name):
1503 lctl.cleanup(self.name, self.uuid, config.force,
1505 except CommandError, e:
1506 log(self.module_name, "cleanup failed: ", self.name)
1509 Module.cleanup(self)
1510 if not self.mds_remaining() and is_prepared('MDT'):
1512 lctl.cleanup("MDT", "MDT_UUID", config.force,
1514 except CommandError, e:
1515 print "cleanup failed: ", self.name
1518 clean_loop(self.devpath)
1521 def __init__(self, db):
1522 Module.__init__(self, 'OSD', db)
1523 self.osdtype = self.db.get_val('osdtype')
1524 self.devpath = self.db.get_val('devpath', '')
1525 self.size = self.db.get_val_int('devsize', 0)
1526 self.journal_size = self.db.get_val_int('journalsize', 0)
1528 # now as we store fids in EA on OST we need to make inode bigger
1529 self.inode_size = self.db.get_val_int('inodesize', 0)
1530 if self.inode_size == 0:
1531 self.inode_size = 256
1532 self.mkfsoptions = self.db.get_val('mkfsoptions', '')
1533 # Allocate fewer inodes on large OST devices. Most filesystems
1534 # can be much more aggressive than this, but by default we can't.
1535 if self.size > 1000000:
1536 self.mkfsoptions = '-i 16384 ' + self.mkfsoptions
1537 self.mountfsoptions = self.db.get_val('mountfsoptions', '')
1539 self.quota = config.quota
1541 self.quota = self.db.get_val('quota', '')
1543 self.fstype = self.db.get_val('fstype', '')
1544 if sys_get_branch() == '2.4' and self.fstype == 'ldiskfs':
1545 self.fstype = 'ext3'
1546 elif sys_get_branch() == '2.6' and self.fstype == 'ext3':
1547 self.fstype = 'ldiskfs'
1549 self.nspath = self.db.get_val('nspath', '')
1550 target_uuid = self.db.get_first_ref('target')
1551 ost = self.db.lookup(target_uuid)
1552 self.name = ost.getName()
1553 self.format = self.db.get_val('autoformat', 'yes')
1554 if ost.get_val('failover', '1') != '0':
1555 self.failover_ost = 'f'
1557 self.failover_ost = 'n'
1559 active_uuid = get_active_target(ost)
1561 panic("No target device found:", target_uuid)
1562 if active_uuid == self.uuid:
1566 if self.active and config.group and config.group != ost.get_val('group', ost.get_val('name')):
1569 self.target_dev_uuid = self.uuid
1570 self.uuid = target_uuid
1573 self.add_lustre_module('quota', 'lquota')
1574 self.add_lustre_module('ost', 'ost')
1575 # FIXME: should we default to ext3 here?
1576 if self.fstype == 'ldiskfs':
1577 self.add_lustre_module('ldiskfs', 'ldiskfs')
1579 self.add_lustre_module('lvfs' , 'fsfilt_%s' % (self.fstype))
1580 self.add_lustre_module(self.osdtype, self.osdtype)
1582 def load_module(self):
1584 Module.load_module(self)
1586 # need to check /proc/mounts and /etc/mtab before
1587 # formatting anything.
1588 # FIXME: check if device is already formatted.
1590 if is_prepared(self.name):
1593 debug(self.uuid, "not active")
1595 self.info(self.osdtype, self.devpath, self.size, self.fstype,
1596 self.format, self.journal_size, self.inode_size)
1597 if self.osdtype == 'obdecho':
1600 blkdev = block_dev(self.devpath, self.size, self.fstype,
1601 config.reformat, self.format, self.journal_size,
1602 self.inode_size, self.mkfsoptions)
1604 mountfsoptions = def_mount_options(self.fstype, 'ost', blkdev)
1606 if config.mountfsoptions:
1608 mountfsoptions = mountfsoptions + ',' + config.mountfsoptions
1610 mountfsoptions = config.mountfsoptions
1611 if self.mountfsoptions:
1612 mountfsoptions = mountfsoptions + ',' + self.mountfsoptions
1614 if self.mountfsoptions:
1616 mountfsoptions = mountfsoptions + ',' + self.mountfsoptions
1618 mountfsoptions = self.mountfsoptions
1620 print 'OST mount options: ' + mountfsoptions
1622 lctl.newdev(self.osdtype, self.name, self.uuid,
1623 setup ="%s %s %s %s %s" %(blkdev, self.fstype,
1624 self.failover_ost, mountfsoptions,
1626 if not is_prepared('OSS'):
1627 lctl.newdev("ost", 'OSS', 'OSS_UUID', setup ="")
1629 def osd_remaining(self):
1630 out = lctl.device_list()
1632 if string.split(s)[2] in ('obdfilter', 'obdecho'):
1635 def safe_to_clean(self):
1638 def safe_to_clean_modules(self):
1639 return not self.osd_remaining()
1643 debug(self.uuid, "not active")
1645 if is_prepared(self.name):
1648 lctl.cleanup(self.name, self.uuid, config.force,
1650 except CommandError, e:
1651 log(self.module_name, "cleanup failed: ", self.name)
1654 if not self.osd_remaining() and is_prepared('OSS'):
1656 lctl.cleanup("OSS", "OSS_UUID", config.force,
1658 except CommandError, e:
1659 print "cleanup failed: ", self.name
1662 if not self.osdtype == 'obdecho':
1663 clean_loop(self.devpath)
1665 # Generic client module, used by OSC and MDC
1666 class Client(Module):
1667 def __init__(self, tgtdb, uuid, module, fs_name, self_name=None,
1669 self.target_name = tgtdb.getName()
1670 self.target_uuid = tgtdb.getUUID()
1672 self.backup_targets = []
1674 self.tgt_dev_uuid = get_active_target(tgtdb)
1675 if not self.tgt_dev_uuid:
1676 panic("No target device found for target:", self.target_name)
1678 self.kmod = kmod(config.lustre, config.portals)
1682 self.module = module
1683 self.module_name = string.upper(module)
1685 self.name = '%s_%s_%s_%s' % (self.module_name, socket.gethostname(),
1686 self.target_name, fs_name)
1688 self.name = self_name
1690 self.lookup_server(self.tgt_dev_uuid)
1691 self.lookup_backup_targets()
1692 self.fs_name = fs_name
1695 self.add_lustre_module(module_dir, module)
1697 def lookup_server(self, srv_uuid):
1698 """ Lookup a server's network information """
1699 self._server_nets = get_ost_net(self.db, srv_uuid)
1700 if len(self._server_nets) == 0:
1701 panic("Unable to find a server for:", srv_uuid)
1703 def get_servers(self):
1704 return self._server_nets
1706 def lookup_backup_targets(self):
1707 """ Lookup alternative network information """
1708 prof_list = toplustreDB.get_refs('profile')
1709 for prof_uuid in prof_list:
1710 prof_db = toplustreDB.lookup(prof_uuid)
1712 panic("profile:", prof_uuid, "not found.")
1713 for ref_class, ref_uuid in prof_db.get_all_refs():
1714 if ref_class in ('osd', 'mdsdev'):
1715 devdb = toplustreDB.lookup(ref_uuid)
1716 uuid = devdb.get_first_ref('target')
1717 if self.target_uuid == uuid and self.tgt_dev_uuid != ref_uuid:
1718 debug("add backup target", ref_uuid)
1719 self.backup_targets.append(ref_uuid)
1721 def prepare(self, ignore_connect_failure = 0):
1722 self.info(self.target_uuid)
1723 if is_prepared(self.name):
1726 srv_list = self.get_servers()
1727 debug('dbg CLIENT __prepare__:', self.target_uuid, srv_list)
1728 for srv in srv_list:
1730 if len(srv_list) == 0:
1731 panic("no servers for ", self.target_uuid)
1732 except CommandError, e:
1733 if not ignore_connect_failure:
1738 if self.target_uuid in config.inactive and self.permits_inactive():
1739 debug("%s inactive" % self.target_uuid)
1740 inactive_p = "inactive"
1742 debug("%s active" % self.target_uuid)
1744 lctl.newdev(self.module, self.name, self.uuid,
1745 setup ="%s %s %s" % (self.target_uuid, srv.nid_uuid,
1748 panic("Unable to create OSC for ", self.target_uuid)
1750 for tgt_dev_uuid in self.backup_targets:
1751 this_nets = get_ost_net(toplustreDB, tgt_dev_uuid)
1752 if len(this_nets) == 0:
1753 panic ("Unable to find a backup server for:", tgt_dev_uuid)
1755 for srv in this_nets:
1758 lctl.add_conn(self.name, srv.nid_uuid);
1762 if is_prepared(self.name):
1763 Module.cleanup(self)
1764 srv_list = self.get_servers()
1765 for srv in srv_list:
1766 lctl.disconnect(srv)
1767 for tgt_dev_uuid in self.backup_targets:
1768 this_nets = get_ost_net(toplustreDB, tgt_dev_uuid)
1769 if len(this_nets) == 0:
1770 panic ("Unable to find a backup server for:", tgt_dev_uuid)
1772 for srv in this_nets:
1773 lctl.disconnect(srv)
1776 def __init__(self, db, uuid, fs_name):
1777 Client.__init__(self, db, uuid, 'mdc', fs_name)
1779 def permits_inactive(self):
1783 def __init__(self, db, uuid, fs_name):
1784 Client.__init__(self, db, uuid, 'osc', fs_name)
1786 def permits_inactive(self):
1790 def __init__(self, db):
1791 Module.__init__(self, 'COBD', db)
1792 self.real_uuid = self.db.get_first_ref('realobd')
1793 self.cache_uuid = self.db.get_first_ref('cacheobd')
1794 self.add_lustre_module('cobd' , 'cobd')
1796 # need to check /proc/mounts and /etc/mtab before
1797 # formatting anything.
1798 # FIXME: check if device is already formatted.
1800 if is_prepared(self.name):
1802 self.info(self.real_uuid, self.cache_uuid)
1803 lctl.newdev("cobd", self.name, self.uuid,
1804 setup ="%s %s" %(self.real_uuid, self.cache_uuid))
1807 # virtual interface for OSC and LOV
1809 def __init__(self, db, uuid, fs_name, name_override = None, quota = None):
1810 Module.__init__(self, 'VOSC', db)
1812 self.add_lustre_module('quota', 'lquota')
1813 if db.get_class() == 'lov':
1814 self.osc = LOV(db, uuid, fs_name, name_override)
1816 self.osc = get_osc(db, uuid, fs_name)
1818 return self.osc.uuid
1820 return self.osc.name
1825 def load_module(self):
1826 Module.load_module(self)
1827 self.osc.load_module()
1828 def cleanup_module(self):
1829 self.osc.cleanup_module()
1830 Module.cleanup_module(self)
1833 class ECHO_CLIENT(Module):
1834 def __init__(self,db):
1835 Module.__init__(self, 'ECHO_CLIENT', db)
1836 self.add_lustre_module('obdecho', 'obdecho')
1837 self.obd_uuid = self.db.get_first_ref('obd')
1838 obd = self.db.lookup(self.obd_uuid)
1839 self.uuid = generate_client_uuid(self.name)
1840 self.osc = VOSC(obd, self.uuid, self.name)
1843 if is_prepared(self.name):
1845 self.osc.prepare() # XXX This is so cheating. -p
1846 self.info(self.obd_uuid)
1848 lctl.newdev("echo_client", self.name, self.uuid,
1849 setup = self.osc.get_name())
1852 if is_prepared(self.name):
1853 Module.cleanup(self)
1856 def load_module(self):
1857 self.osc.load_module()
1858 Module.load_module(self)
1860 def cleanup_module(self):
1861 Module.cleanup_module(self)
1862 self.osc.cleanup_module()
1865 def generate_client_uuid(name):
1866 client_uuid = '%05x_%.19s_%05x%05x' % (int(random.random() * 1048576),
1868 int(random.random() * 1048576),
1869 int(random.random() * 1048576))
1870 return client_uuid[:36]
1873 def my_rstrip(s, chars):
1874 """my_rstrip(s, chars) -> strips any instances of the characters
1875 found in chars from the right side of string s"""
1876 # XXX required because python versions pre 2.2.3 don't allow
1877 #string.rstrip() to take alternate char lists
1881 ns = string.rstrip(s, '/')
1882 except TypeError, e:
1883 for i in range(len(s) - 1, 0, -1):
1892 class Mountpoint(Module):
1893 def __init__(self,db):
1894 Module.__init__(self, 'MTPT', db)
1895 self.path = my_rstrip(self.db.get_val('path'), '/')
1896 self.clientoptions = self.db.get_val('clientoptions', '')
1897 self.fs_uuid = self.db.get_first_ref('filesystem')
1898 fs = self.db.lookup(self.fs_uuid)
1899 self.mds_uuid = fs.get_first_ref('mds')
1900 mds_db = self.db.lookup(self.mds_uuid)
1902 quota = config.quota
1904 quota = mds_db.get_val('quota', config.quota)
1905 self.obd_uuid = fs.get_first_ref('obd')
1906 obd = self.db.lookup(self.obd_uuid)
1907 client_uuid = generate_client_uuid(self.name)
1908 self.vosc = VOSC(obd, client_uuid, self.name, quota=quota)
1909 self.mdc = get_mdc(db, client_uuid, self.name, self.mds_uuid)
1911 self.add_lustre_module('mdc', 'mdc')
1912 self.add_lustre_module('llite', 'llite')
1915 if fs_is_mounted(self.path):
1916 log(self.path, "already mounted.")
1920 mdc_name = self.mdc.name
1922 self.info(self.path, self.mds_uuid, self.obd_uuid)
1923 if config.record or config.lctl_dump:
1924 lctl.mount_option(local_node_name, self.vosc.get_name(), mdc_name)
1927 if config.clientoptions:
1928 if self.clientoptions:
1929 self.clientoptions = self.clientoptions + ',' + config.clientoptions
1931 self.clientoptions = config.clientoptions
1932 if self.clientoptions:
1933 self.clientoptions = ',' + self.clientoptions
1934 # Linux kernel will deal with async and not pass it to ll_fill_super,
1935 # so replace it with Lustre async
1936 self.clientoptions = string.replace(self.clientoptions, "async", "lasync")
1938 cmd = "mount -t lustre_lite -o osc=%s,mdc=%s%s %s %s" % \
1939 (self.vosc.get_name(), mdc_name, self.clientoptions, config.config, self.path)
1940 run("mkdir", self.path)
1945 panic("mount failed:", self.path, ":", string.join(val))
1948 self.info(self.path, self.mds_uuid,self.obd_uuid)
1950 if config.record or config.lctl_dump:
1951 lctl.del_mount_option(local_node_name)
1953 if fs_is_mounted(self.path):
1955 (rc, out) = run("umount", "-f", self.path)
1957 (rc, out) = run("umount", self.path)
1959 raise CommandError('umount', out, rc)
1961 if fs_is_mounted(self.path):
1962 panic("fs is still mounted:", self.path)
1967 def load_module(self):
1968 self.vosc.load_module()
1969 Module.load_module(self)
1971 def cleanup_module(self):
1972 Module.cleanup_module(self)
1973 self.vosc.cleanup_module()
1976 # ============================================================
1977 # misc query functions
1979 def get_ost_net(self, osd_uuid):
1983 osd = self.lookup(osd_uuid)
1984 node_uuid = osd.get_first_ref('node')
1985 node = self.lookup(node_uuid)
1987 panic("unable to find node for osd_uuid:", osd_uuid,
1988 " node_ref:", node_uuid)
1989 for net_uuid in node.get_networks():
1990 db = node.lookup(net_uuid)
1991 net = Network(db, node_uuid)
1992 srv_list.append(net)
1996 # the order of iniitailization is based on level.
1997 def getServiceLevel(self):
1998 type = self.get_class()
2000 if type in ('network',):
2002 elif type in ('ldlm',):
2004 elif type in ('osd', 'cobd'):
2006 elif type in ('mdsdev',):
2008 elif type in ('mountpoint', 'echoclient'):
2011 panic("Unknown type: ", type)
2013 if ret < config.minlevel or ret > config.maxlevel:
2018 # return list of services in a profile. list is a list of tuples
2019 # [(level, db_object),]
2020 def getServices(self):
2022 for ref_class, ref_uuid in self.get_all_refs():
2023 servdb = self.lookup(ref_uuid)
2025 level = getServiceLevel(servdb)
2027 list.append((level, servdb))
2029 panic('service not found: ' + ref_uuid)
2035 ############################################################
2037 # FIXME: clean this mess up!
2039 # OSC is no longer in the xml, so we have to fake it.
2040 # this is getting ugly and begging for another refactoring
2041 def get_osc(ost_db, uuid, fs_name):
2042 osc = OSC(ost_db, uuid, fs_name)
2045 def get_mdc(db, uuid, fs_name, mds_uuid):
2046 mds_db = db.lookup(mds_uuid);
2048 panic("no mds:", mds_uuid)
2049 mdc = MDC(mds_db, uuid, fs_name)
2052 def get_active_target(db):
2053 target_uuid = db.getUUID()
2054 target_name = db.getName()
2055 node_name = get_select(target_name)
2057 tgt_dev_uuid = db.get_node_tgt_dev(node_name, target_uuid)
2059 tgt_dev_uuid = db.get_first_ref('active')
2062 def get_server_by_nid_uuid(db, nid_uuid):
2063 for n in db.lookup_class("network"):
2065 if net.nid_uuid == nid_uuid:
2069 ############################################################
2073 type = db.get_class()
2074 debug('Service:', type, db.getName(), db.getUUID())
2079 n = LOV(db, "YOU_SHOULD_NEVER_SEE_THIS_UUID")
2080 elif type == 'network':
2084 elif type == 'cobd':
2086 elif type == 'mdsdev':
2088 elif type == 'mountpoint':
2090 elif type == 'echoclient':
2093 panic("unknown service type:", type)
2097 # Prepare the system to run lustre using a particular profile
2098 # in a the configuration.
2099 # * load & the modules
2100 # * setup networking for the current node
2101 # * make sure partitions are in place and prepared
2102 # * initialize devices with lctl
2103 # Levels is important, and needs to be enforced.
2104 def for_each_profile(db, prof_list, operation):
2105 for prof_uuid in prof_list:
2106 prof_db = db.lookup(prof_uuid)
2108 panic("profile:", prof_uuid, "not found.")
2109 services = getServices(prof_db)
2112 def doWriteconf(services):
2117 if s[1].get_class() == 'mdsdev':
2118 n = newService(s[1])
2122 panic("Cannot find mds device, please run --write_conf on the mds node.")
2125 def doSetup(services):
2129 n = newService(s[1])
2132 def doModules(services):
2136 n = newService(s[1])
2139 def doCleanup(services):
2144 n = newService(s[1])
2145 if n.safe_to_clean():
2148 def doUnloadModules(services):
2153 n = newService(s[1])
2154 if n.safe_to_clean_modules():
2157 def doMakeServiceScript(services):
2161 os.makedirs(config.service_scripts)
2163 if e[0] != errno.EEXIST:
2164 panic("Couldn't create scripts dir " + config.service_scripts + ": " + e[1])
2167 if s[1].get_class() != 'osd' and s[1].get_class() != 'mdsdev':
2170 target_uuid = s[1].get_first_ref('target')
2171 target = toplustreDB.lookup(target_uuid)
2172 target_symlink = config.service_scripts + "/" + target.getName()
2176 os.unlink(target_symlink)
2178 print "Removed " + target_symlink
2180 if e[0] != errno.EISDIR:
2182 os.rmdir(target_symlink)
2184 print "Removed " + target_symlink
2186 if e[0] != errno.ENOENT:
2187 panic("Error removing " + target_symlink + ": " + e[1])
2190 os.symlink("/etc/init.d/lustre", target_symlink)
2192 print "Created service link " + target_symlink + " to /etc/init.d/lustre"
2195 if e[0] == errno.EEXIST:
2196 extra_error = " (use --force option to remove existing files)"
2199 panic("Error creating " + target_symlink + ": " + e[1] + extra_error)
2201 # Check mtime of config logs
2202 def doCheckMtime(lustreDB, hosts):
2204 node_db = lustreDB.lookup_name(h, 'node')
2211 prof_list = node_db.get_refs('profile')
2212 for prof_uuid in prof_list:
2213 prof_db = node_db.lookup(prof_uuid)
2215 services = getServices(prof_db)
2217 if s[1].get_class() == 'mdsdev':
2221 if mdsdb and lustreDB.get_mtime():
2222 debug("Checking XML modification time")
2223 devpath = mdsdb.get_val('devpath','')
2224 xmtime = string.atol(lustreDB.get_mtime())
2225 cmd = "debugfs -c -R 'stat /LOGS' %s 2>&1 | grep mtime" %devpath
2226 ret, kmtimes = runcmd(cmd)
2228 log("Can not get mtime info of MDS LOGS directory")
2230 kmtime = string.atoi(string.split(kmtimes[0])[1], 0)
2232 debug('xmtime ', xmtime, '> kmtime', kmtime)
2234 log("Warning: MDS startup logs are older than config %s."
2235 " Please run --write_conf on stopped MDS to update."
2238 panic("Error: MDS startup logs are older than config %s."
2239 " Please run --write_conf on stopped MDS to update."
2240 " Use '--old_conf' to start anyways." %CONFIG_FILE)
2245 def doHost(lustreDB, hosts):
2246 global local_node_name, tgt_select
2249 node_db = lustreDB.lookup_name(h, 'node')
2252 tgt_select[config.service] = h
2253 config.group = config.service
2256 panic('No host entry found.')
2258 local_node_name = node_db.get_val('name', 0)
2259 lustre_upcall = node_db.get_val('lustreUpcall', '')
2260 portals_upcall = node_db.get_val('portalsUpcall', '')
2261 timeout = node_db.get_val_int('timeout', 0)
2262 ptldebug = node_db.get_val('ptldebug', '')
2263 subsystem = node_db.get_val('subsystem', '')
2265 # Two step process: (1) load modules, (2) setup lustre
2266 # if not cleaning, load modules first.
2267 prof_list = node_db.get_refs('profile')
2269 if config.make_service_scripts:
2270 for_each_profile(node_db, prof_list, doMakeServiceScript)
2273 elif config.write_conf:
2274 for_each_profile(node_db, prof_list, doModules)
2275 for_each_profile(node_db, prof_list, doWriteconf)
2276 for_each_profile(node_db, prof_list, doUnloadModules)
2279 elif config.recover:
2280 if not (config.tgt_uuid and config.client_uuid and config.conn_uuid):
2281 raise Lustre.LconfError( "--recovery requires --tgt_uuid <UUID> " +
2282 "--client_uuid <UUID> --conn_uuid <UUID>")
2283 doRecovery(lustreDB, lctl, config.tgt_uuid, config.client_uuid,
2285 elif config.cleanup:
2286 if not mod_loaded('lnet'):
2289 # ugly hack, only need to run lctl commands for --dump
2290 if config.lctl_dump or config.record:
2291 for_each_profile(node_db, prof_list, doCleanup)
2294 sys_set_ptldebug(ptldebug)
2295 sys_set_subsystem(subsystem)
2296 sys_set_lustre_upcall(lustre_upcall)
2297 sys_set_portals_upcall(portals_upcall)
2299 for_each_profile(node_db, prof_list, doCleanup)
2300 for_each_profile(node_db, prof_list, doUnloadModules)
2304 # ugly hack, only need to run lctl commands for --dump
2305 if config.lctl_dump or config.record:
2306 sys_set_timeout(timeout)
2307 sys_set_lustre_upcall(lustre_upcall)
2308 for_each_profile(node_db, prof_list, doSetup)
2311 if PLATFORM == 'LINUX':
2312 sys_set_netmem_max('/proc/sys/net/core/rmem_max', MAXTCPBUF)
2313 sys_set_netmem_max('/proc/sys/net/core/wmem_max', MAXTCPBUF)
2315 for_each_profile(node_db, prof_list, doModules)
2317 if PLATFORM == 'LINUX':
2318 # XXX need to be fixed for Darwin
2319 sys_set_debug_path()
2320 sys_set_ptldebug(ptldebug)
2321 sys_set_subsystem(subsystem)
2322 script = config.gdb_script
2323 run(lctl.lctl, ' modules >', script)
2325 log ("The GDB module script is in", script)
2326 # pause, so user has time to break and
2329 sys_set_timeout(timeout)
2330 sys_set_lustre_upcall(lustre_upcall)
2331 sys_set_portals_upcall(portals_upcall)
2333 for_each_profile(node_db, prof_list, doSetup)
2336 def add_clumanager_node(node_db, nodes, services):
2338 node_name = node_db.getUUID()
2339 nodes[node_name] = []
2341 for prof_uuid in node_db.get_refs('profile'):
2342 prof_db = toplustreDB.lookup(prof_uuid)
2343 for ref_class, ref_uuid in prof_db.get_all_refs():
2344 if ref_class not in ('osd', 'mdsdev'):
2346 devdb = toplustreDB.lookup(ref_uuid)
2347 tgt_uuid = devdb.get_first_ref('target')
2349 nodes[node_name].append(ref_uuid)
2351 if not services.has_key(tgt_uuid):
2353 print "New service: " + tgt_uuid + " (originally found on " + node_name + ")"
2354 new_services.append(tgt_uuid)
2355 services[tgt_uuid] = []
2356 services[tgt_uuid].append(ref_uuid)
2360 def add_clumanager_services(new_services, nodes, dev_list):
2362 for devdb in dev_list:
2363 tgt_uuid = devdb.get_first_ref('target')
2364 if tgt_uuid in new_services:
2365 node_uuid = devdb.get_first_ref('node')
2367 if not (nodes.has_key(node_uuid) or node_uuid in new_nodes):
2369 print "New node: " + node_uuid + " for service " + tgt_uuid
2370 new_nodes.append(node_uuid)
2374 def doClumanager(lustreDB, hosts):
2380 for dev_uuid in toplustreDB.get_refs('osd') + toplustreDB.get_refs('mdsdev'):
2381 dev_list.append(lustreDB.lookup(dev_uuid))
2385 node_db = lustreDB.lookup_name(h, 'node')
2388 new_services = add_clumanager_node(node_db, nodes, services)
2392 panic('No host entry found.')
2395 if len(new_services) == 0:
2398 new_nodes = add_clumanager_services(new_services, nodes, dev_list)
2399 if len(new_nodes) == 0:
2402 if len(new_nodes) + len(nodes.keys()) > 8:
2403 panic("CluManager only supports 8 nodes per failover \"cluster.\"")
2406 for node_uuid in new_nodes:
2407 node_db = lustreDB.lookup(node_uuid)
2409 panic("No node entry for " + node_uuid + " was found.")
2411 new_services.append(add_clumanager_node(node_db, nodes, services))
2414 for node in nodes.keys():
2415 nodedb = lustreDB.lookup(node)
2416 nodenames.append(nodedb.getName())
2419 print """<?xml version="1.0"?>
2420 <cluconfig version="3.0">
2421 <clumembd broadcast="no" interval="750000" loglevel="5" multicast="yes" multicast_ipaddress="225.0.0.11" thread="yes" tko_count="20"/>
2422 <cluquorumd loglevel="5" pinginterval="2"/>
2423 <clurmtabd loglevel="5" pollinterval="4"/>
2424 <clusvcmgrd loglevel="5"/>
2425 <clulockd loglevel="5"/>
2426 <cluster config_viewnumber="1" name="%s"/>
2427 <sharedstate driver="libsharedraw.so" rawprimary="%s" rawshadow="%s" type="raw"/>
2428 <members> """ % (string.join(nodenames), config.rawprimary, config.rawsecondary)
2432 for node in nodenames:
2433 print " <member id=\"%d\" name=\"%s\" watchdog=\"yes\"/>" % (i, node)
2436 print " </members>\n <failoverdomains>"
2438 servicekeys = services.keys()
2442 for service in servicekeys:
2443 svcdb = lustreDB.lookup(service)
2444 print " <failoverdomain id=\"%d\" name=\"%s\" ordered=\"yes\" restricted=\"yes\">" % (i, svcdb.getName())
2448 active_uuid = get_active_target(svcdb)
2449 for svc_uuid in [active_uuid] + services[service]:
2450 if svc_uuid == active_uuid and j > 0:
2452 svcdb = lustreDB.lookup(svc_uuid)
2454 svc_node_uuid = svcdb.get_first_ref('node')
2455 svc_nodedb = lustreDB.lookup(svc_node_uuid)
2457 print " <failoverdomainnode id=\"%d\" name=\"%s\"/>" % (j, svc_nodedb.getName())
2460 print " </failoverdomain>"
2462 print " </failoverdomains>\n <services>"
2465 for service in servicekeys:
2466 svcdb = lustreDB.lookup(service)
2467 active_uuid = get_active_target(svcdb)
2468 activedb = lustreDB.lookup(active_uuid)
2470 svc_node_uuid = activedb.get_first_ref('node')
2471 svc_nodedb = lustreDB.lookup(svc_node_uuid)
2473 print " <service checkinterval=\"30\" failoverdomain=\"%s\" id=\"%d\" name=\"%s\" userscript=\"%s/%s\">" \
2474 % ( svcdb.getName(), i, svcdb.getName(), config.service_scripts, svcdb.getName())
2475 print " <service_ipaddresses/>\n </service>"
2478 print " </services>\n</cluconfig>"
2480 def doRecovery(lustreDB, lctl, tgt_uuid, client_uuid, nid_uuid):
2481 tgt = lustreDB.lookup(tgt_uuid)
2483 raise Lustre.LconfError("doRecovery: "+ tgt_uuid +" not found.")
2484 new_uuid = get_active_target(tgt)
2486 raise Lustre.LconfError("doRecovery: no active target found for: " +
2488 srv_list = find_local_servers(get_ost_net(lustreDB, new_uuid))
2490 raise Lustre.LconfError("Unable to find a connection to:" + new_uuid)
2492 oldsrv = get_server_by_nid_uuid(lustreDB, nid_uuid)
2495 for srv in srv_list:
2496 if oldsrv.net_type != srv.net_type:
2499 log("Reconnecting", tgt_uuid, "to", srv.nid_uuid)
2501 lctl.recover(client_uuid, srv.nid_uuid)
2504 def setupModulePath(cmd, portals_dir = PORTALS_DIR):
2505 base = os.path.dirname(cmd)
2506 if development_mode():
2507 if not config.lustre:
2508 debug('using objdir module paths')
2509 config.lustre = (os.path.join(base, ".."))
2510 # normalize the portals dir, using command line arg if set
2512 portals_dir = config.portals
2513 dir = os.path.join(config.lustre, portals_dir)
2514 config.portals = dir
2515 debug('config.portals', config.portals)
2516 elif config.lustre and config.portals:
2518 # if --lustre and --portals, normalize portals
2519 # can ignore POTRALS_DIR here, since it is probly useless here
2520 config.portals = os.path.join(config.lustre, config.portals)
2521 debug('config.portals B', config.portals)
2523 def sysctl(path, val):
2524 debug("+ sysctl", path, val)
2528 fp = open(os.path.join('/proc/sys', path), 'w')
2535 def sys_set_debug_path():
2536 sysctl('lnet/debug_path', config.debug_path)
2538 def validate_upcall(upcall):
2540 if upcall in ('DEFAULT','NONE'):
2542 elif os.path.exists(upcall):
2543 if not os.access(upcall, os.X_OK):
2544 print "WARNING upcall script not executable: %s" % upcall
2546 print "WARNING invalid upcall script specified: %s" % upcall
2548 def sys_set_lustre_upcall(upcall):
2549 # the command line overrides the value in the node config
2550 if config.lustre_upcall:
2551 upcall = config.lustre_upcall
2553 upcall = config.upcall
2555 validate_upcall(upcall)
2556 lctl.set_lustre_upcall(upcall)
2558 def sys_set_portals_upcall(upcall):
2559 # the command line overrides the value in the node config
2560 if config.portals_upcall:
2561 upcall = config.portals_upcall
2563 upcall = config.upcall
2565 validate_upcall(upcall)
2566 sysctl('lnet/upcall', upcall)
2568 def sys_set_group_upcall(mds, upcall):
2571 # the command line overrides the value in the MDS config
2572 if config.group_upcall:
2573 upcall = config.group_upcall
2575 validate_upcall(upcall)
2576 debug("setting MDS", mds, "upcall to:", upcall)
2577 path = "/proc/fs/lustre/mds/" + mds + "/group_upcall"
2578 fp = open(path, 'w')
2582 def sys_set_timeout(timeout):
2583 # the command overrides the value in the node config
2584 if config.timeout and config.timeout > 0:
2585 timeout = config.timeout
2586 if timeout != None and timeout > 0:
2587 lctl.set_timeout(timeout)
2589 def sys_tweak_socknal ():
2590 if config.single_socket:
2591 sysctl("socknal/typed", 0)
2593 def sys_optimize_elan ():
2594 procfiles = ["/proc/elan/config/eventint_punt_loops",
2595 "/proc/qsnet/elan3/config/eventint_punt_loops",
2596 "/proc/qsnet/elan4/config/elan4_mainint_punt_loops"]
2598 if os.access(p, os.W_OK):
2599 run ("echo 1 > " + p)
2601 def sys_set_ptldebug(ptldebug):
2603 ptldebug = config.ptldebug
2606 val = eval(ptldebug, ptldebug_names)
2607 val = "0x%x" % (val)
2608 sysctl('lnet/debug', val)
2609 except NameError, e:
2612 def sys_set_subsystem(subsystem):
2613 if config.subsystem:
2614 subsystem = config.subsystem
2617 val = eval(subsystem, subsystem_names)
2618 val = "0x%x" % (val)
2619 sysctl('lnet/subsystem_debug', val)
2620 except NameError, e:
2623 def sys_set_netmem_max(path, max):
2624 debug("setting", path, "to at least", max)
2632 fp = open(path, 'w')
2633 fp.write('%d\n' %(max))
2637 # Add dir to the global PATH, if not already there.
2638 def add_to_path(new_dir):
2639 syspath = string.split(os.environ['PATH'], ':')
2640 if new_dir in syspath:
2642 os.environ['PATH'] = os.environ['PATH'] + ':' + new_dir
2644 def default_debug_path():
2645 path = '/tmp/lustre-log'
2646 if os.path.isdir('/r'):
2651 def default_gdb_script():
2652 script = '/tmp/ogdb'
2653 if os.path.isdir('/r'):
2654 return '/r' + script
2658 DEFAULT_PATH = ('/sbin', '/usr/sbin', '/bin', '/usr/bin')
2659 # ensure basic elements are in the system path
2660 def sanitise_path():
2661 for dir in DEFAULT_PATH:
2664 # global hack for the --select handling
2666 def init_select(args):
2667 # args = [service=nodeA,service2=nodeB service3=nodeC]
2668 # --service <service> is analagous to:
2669 # --group <service> --select <service>=<node>
2670 # this is handled in doHost()
2673 list = string.split(arg, ',')
2675 srv, node = string.split(entry, '=')
2676 tgt_select[srv] = node
2678 def get_select(srv):
2679 if tgt_select.has_key(srv):
2680 return tgt_select[srv]
2684 FLAG = Lustre.Options.FLAG
2685 PARAM = Lustre.Options.PARAM
2686 INTPARAM = Lustre.Options.INTPARAM
2687 PARAMLIST = Lustre.Options.PARAMLIST
2689 ('verbose,v', "Print system commands as they are run"),
2690 ('ldapurl',"LDAP server URL, eg. ldap://localhost", PARAM),
2691 ('config', "Cluster config name used for LDAP query", PARAM),
2692 ('select', "service=nodeA,service2=nodeB ", PARAMLIST),
2693 ('service', "shorthand for --group <service> --select <service>=<node>", PARAM),
2694 ('node', "Load config for <nodename>", PARAM),
2695 ('cleanup,d', "Cleans up config. (Shutdown)"),
2696 ('force,f', "Forced unmounting and/or obd detach during cleanup",
2698 ('single_socket', "socknal option: only use one socket instead of bundle",
2700 ('failover',"""Used to shut down without saving state.
2701 This will allow this node to "give up" a service to a
2702 another node for failover purposes. This will not
2703 be a clean shutdown.""",
2705 ('abort_recovery',"""Used to start a service when you know recovery
2706 will not succeed. This will skip the recovery
2707 timeout period."""),
2708 ('gdb', """Prints message after creating gdb module script
2709 and sleeps for 5 seconds."""),
2710 ('noexec,n', """Prints the commands and steps that will be run for a
2711 config without executing them. This can used to check if a
2712 config file is doing what it should be doing"""),
2713 ('nomod', "Skip load/unload module step."),
2714 ('nosetup', "Skip device setup/cleanup step."),
2715 ('reformat', "Reformat all devices (without question)"),
2716 ('mkfsoptions', "Additional options for the mk*fs command line", PARAM),
2717 ('mountfsoptions', "Additional options for mount fs command line", PARAM),
2718 ('clientoptions', "Additional options for Lustre", PARAM),
2719 ('dump', "Dump the kernel debug log to file before portals is unloaded",
2721 ('write_conf', "Save all the client config information on mds."),
2722 ('old_conf', "Start up service even though config logs appear outdated."),
2723 ('record', "Write config information on mds."),
2724 ('record_log', "Name of config record log.", PARAM),
2725 ('record_device', "MDS device name that will record the config commands",
2727 ('minlevel', "Minimum level of services to configure/cleanup",
2729 ('maxlevel', """Maximum level of services to configure/cleanup
2730 Levels are aproximatly like:
2735 70 - mountpoint, echo_client, osc, mdc, lov""",
2737 ('lustre', """Base directory of lustre sources. This parameter will
2738 cause lconf to load modules from a source tree.""", PARAM),
2739 ('portals', """Portals source directory. If this is a relative path,
2740 then it is assumed to be relative to lustre. """, PARAM),
2741 ('timeout', "Set recovery timeout", INTPARAM),
2742 ('upcall', "Set both portals and lustre upcall script", PARAM),
2743 ('lustre_upcall', "Set lustre upcall script", PARAM),
2744 ('portals_upcall', "Set portals upcall script", PARAM),
2745 ('group_upcall', "Set supplementary group upcall program", PARAM),
2746 ('lctl_dump', "Save lctl ioctls to the dumpfile argument", PARAM),
2747 ('ptldebug', "Set the portals debug level", PARAM),
2748 ('subsystem', "Set the portals debug subsystem", PARAM),
2749 ('gdb_script', "Fullname of gdb debug script", PARAM, default_gdb_script()),
2750 ('debug_path', "Path to save debug dumps", PARAM, default_debug_path()),
2751 ('allow_unprivileged_port', "Allow connections from unprivileged ports"),
2752 ('clumanager', "Generate CluManager config file for this node's cluster"),
2753 ('rawprimary', "For clumanager, device of the primary quorum", PARAM, "/dev/raw/raw1"),
2754 ('rawsecondary', "For clumanager, device of the secondary quorum", PARAM, "/dev/raw/raw2"),
2755 ('service_scripts', "For clumanager, directory containing per-service scripts", PARAM, "/etc/lustre/services"),
2756 ('make_service_scripts', "Create per-service symlinks for use with clumanager"),
2757 # Client recovery options
2758 ('recover', "Recover a device"),
2759 ('group,g', "The group of devices to configure or cleanup", PARAM),
2760 ('tgt_uuid', "The failed target (required for recovery)", PARAM),
2761 ('client_uuid', "The failed client (required for recovery)", PARAM),
2762 ('conn_uuid', "The failed connection (required for recovery)", PARAM),
2764 ('inactive', """The name of an inactive service, to be ignored during
2765 mounting (currently OST-only). Can be repeated.""",
2767 ('user_xattr', """Enable user_xattr support on MDS""", FLAG, 0),
2768 ('acl', """Enable ACL support on MDS""", FLAG, 0),
2769 ('quota', "Enable quota support for client file system", PARAM),
2773 global lctl, config, toplustreDB, CONFIG_FILE
2775 # in the upcall this is set to SIG_IGN
2776 signal.signal(signal.SIGCHLD, signal.SIG_DFL)
2778 cl = Lustre.Options("lconf", "config.xml", lconf_options)
2780 config, args = cl.parse(sys.argv[1:])
2781 except Lustre.OptionError, e:
2785 setupModulePath(sys.argv[0])
2787 host = socket.gethostname()
2789 # the PRNG is normally seeded with time(), which is not so good for starting
2790 # time-synchronized clusters
2791 input = open('/dev/urandom', 'r')
2793 print 'Unable to open /dev/urandom!'
2795 seed = input.read(32)
2801 init_select(config.select)
2804 # allow config to be fetched via HTTP, but only with python2
2805 if sys.version[0] != '1' and args[0].startswith('http://'):
2808 config_file = urllib2.urlopen(args[0])
2809 except (urllib2.URLError, socket.error), err:
2810 if hasattr(err, 'args'):
2812 print "Could not access '%s': %s" %(args[0], err)
2814 elif not os.access(args[0], os.R_OK):
2815 print 'File not found or readable:', args[0]
2819 config_file = open(args[0], 'r')
2821 dom = xml.dom.minidom.parse(config_file)
2823 panic("%s does not appear to be a config file." % (args[0]))
2824 sys.exit(1) # make sure to die here, even in debug mode.
2826 CONFIG_FILE = args[0]
2827 lustreDB = Lustre.LustreDB_XML(dom.documentElement, dom.documentElement)
2828 if not config.config:
2829 config.config = os.path.basename(args[0])# use full path?
2830 if config.config[-4:] == '.xml':
2831 config.config = config.config[:-4]
2832 elif config.ldapurl:
2833 if not config.config:
2834 panic("--ldapurl requires --config name")
2835 dn = "config=%s,fs=lustre" % (config.config)
2836 lustreDB = Lustre.LustreDB_LDAP('', {}, base=dn, url = config.ldapurl)
2837 elif config.ptldebug or config.subsystem:
2838 sys_set_ptldebug(None)
2839 sys_set_subsystem(None)
2842 print 'Missing config file or ldap URL.'
2843 print 'see lconf --help for command summary'
2846 if config.reformat and config.cleanup:
2847 panic("Options \"reformat\" and \"cleanup\" are incompatible. "+
2848 "Please specify only one.")
2850 toplustreDB = lustreDB
2852 ver = lustreDB.get_version()
2854 panic("No version found in config data, please recreate.")
2855 if ver != Lustre.CONFIG_VERSION:
2856 panic("Config version", ver, "does not match lconf version",
2857 Lustre.CONFIG_VERSION)
2861 node_list.append(config.node)
2864 node_list.append(host)
2865 # node_list.append('localhost')
2867 debug("configuring for host: ", node_list)
2870 config.debug_path = config.debug_path + '-' + host
2871 config.gdb_script = config.gdb_script + '-' + host
2873 lctl = LCTLInterface('lctl')
2875 if config.lctl_dump:
2876 lctl.use_save_file(config.lctl_dump)
2878 if not (config.reformat or config.write_conf or config.cleanup):
2879 doCheckMtime(lustreDB, node_list)
2882 if not (config.record_device and config.record_log):
2883 panic("When recording, both --record_log and --record_device must be specified.")
2884 lctl.clear_log(config.record_device, config.record_log)
2885 lctl.record(config.record_device, config.record_log)
2887 if config.clumanager:
2888 doClumanager(lustreDB, node_list)
2890 doHost(lustreDB, node_list)
2895 if __name__ == "__main__":
2898 except Lustre.LconfError, e:
2900 # traceback.print_exc(file=sys.stdout)
2902 except CommandError, e:
2909 if first_cleanup_error:
2910 sys.exit(first_cleanup_error)