X-Git-Url: https://git.whamcloud.com/?a=blobdiff_plain;f=lustre%2Futils%2Flconf;h=ff9c71b174405c09e68f640bc9e203cb56cb06b5;hb=b3d839c2925f4e7f19fd06841b7302bc827956f3;hp=4ce8e98f19a3947270a35c28afd384bf080b860a;hpb=f3518bdbebef06af3d5680e7bfd4acbb8c9b8966;p=fs%2Flustre-release.git diff --git a/lustre/utils/lconf b/lustre/utils/lconf index 4ce8e98..ff9c71b 100755 --- a/lustre/utils/lconf +++ b/lustre/utils/lconf @@ -1,8 +1,8 @@ #!/usr/bin/env python # -# Copyright (C) 2002 Cluster File Systems, Inc. -# Author: Robert Read - +# Copyright (C) 2002-2003 Cluster File Systems, Inc. +# Authors: Robert Read +# Mike Shaver # This file is part of Lustre, http://www.lustre.org. # # Lustre is free software; you can redistribute it and/or @@ -23,141 +23,118 @@ # lconf is the main driver script for starting and stopping # lustre filesystem services. # -# Based in part on the XML obdctl modifications done by Brian Behlendorf +# Based in part on the XML obdctl modifications done by Brian Behlendorf -import sys, getopt -import string, os, stat, popen2, socket, time -import re, exceptions +import sys, getopt, types +import string, os, stat, popen2, socket, time, random, fcntl, select +import re, exceptions, signal, traceback import xml.dom.minidom +if sys.version[0] == '1': + from FCNTL import F_GETFL, F_SETFL +else: + from fcntl import F_GETFL, F_SETFL + +PYMOD_DIR = "/usr/lib/lustre/python" + +def development_mode(): + base = os.path.dirname(sys.argv[0]) + if os.access(base+"/Makefile", os.R_OK): + return 1 + return 0 + +if development_mode(): + sys.path.append('../utils') +else: + sys.path.append(PYMOD_DIR) + +import Lustre + # Global parameters -TCP_ACCEPTOR = '' -MAXTCPBUF = 1048576 -DEFAULT_TCPBUF = 1048576 +MAXTCPBUF = 16777216 +DEFAULT_TCPBUF = 8388608 +DEFAULT_PORT = 988 # # Maximum number of devices to search for. # (the /dev/loop* nodes need to be created beforehand) MAX_LOOP_DEVICES = 256 - - -def usage(): - print """usage: lconf config.xml - -config.xml Lustre configuration in xml format. ---get URL to fetch a config file ---node Load config for --d | --cleanup Cleans up config. (Shutdown) --v | --verbose Print system commands as they are run --h | --help Print this help ---gdb Prints message after creating gdb module script - and sleeps for 5 seconds. --n | --noexec Prints the commands and steps that will be run for a - config without executing them. This can used to check if a - config file is doing what it should be doing. (Implies -v) ---nomod Skip load/unload module step. ---nosetup Skip device setup/cleanup step. ---reformat Reformat all devices (without question) ---dump Dump the kernel debug log before portals is unloaded -""" - TODO = """ ---ldap server LDAP server with lustre config database ---makeldiff Translate xml source to LDIFF -This are perhaps not needed: ---lustre="src dir" Base directory of lustre sources. Used to search - for modules. ---portals=src Portals source -""" - sys.exit() +PORTALS_DIR = 'portals' + +# Needed to call lconf --record +CONFIG_FILE = "" + +# Please keep these in sync with the values in portals/kp30.h +ptldebug_names = { + "trace" : (1 << 0), + "inode" : (1 << 1), + "super" : (1 << 2), + "ext2" : (1 << 3), + "malloc" : (1 << 4), + "cache" : (1 << 5), + "info" : (1 << 6), + "ioctl" : (1 << 7), + "blocks" : (1 << 8), + "net" : (1 << 9), + "warning" : (1 << 10), + "buffs" : (1 << 11), + "other" : (1 << 12), + "dentry" : (1 << 13), + "portals" : (1 << 14), + "page" : (1 << 15), + "dlmtrace" : (1 << 16), + "error" : (1 << 17), + "emerg" : (1 << 18), + "ha" : (1 << 19), + "rpctrace" : (1 << 20), + "vfstrace" : (1 << 21), + "reada" : (1 << 22), + "config" : (1 << 23), + } + +subsystem_names = { + "undefined" : (1 << 0), + "mdc" : (1 << 1), + "mds" : (1 << 2), + "osc" : (1 << 3), + "ost" : (1 << 4), + "class" : (1 << 5), + "log" : (1 << 6), + "llite" : (1 << 7), + "rpc" : (1 << 8), + "mgmt" : (1 << 9), + "portals" : (1 << 10), + "socknal" : (1 << 11), + "qswnal" : (1 << 12), + "pinger" : (1 << 13), + "filter" : (1 << 14), + "ptlbd" : (1 << 15), + "echo" : (1 << 16), + "ldlm" : (1 << 17), + "lov" : (1 << 18), + "gmnal" : (1 << 19), + "ptlrouter" : (1 << 20), + "cobd" : (1 << 21), + "ibnal" : (1 << 22), + "cmobd" : (1 << 23), + } + + +first_cleanup_error = 0 +def cleanup_error(rc): + global first_cleanup_error + if not first_cleanup_error: + first_cleanup_error = rc # ============================================================ -# Config parameters, encapsulated in a class -class Config: - def __init__(self): - # flags - self._noexec = 0 - self._verbose = 0 - self._reformat = 0 - self._cleanup = 0 - self._gdb = 0 - self._nomod = 0 - self._nosetup = 0 - # parameters - self._modules = None - self._node = None - self._url = None - self._gdb_script = '/tmp/ogdb' - self._debug_path = '/tmp/lustre-log' - self._dump_file = None - self._src_dir = None - - def verbose(self, flag = None): - if flag: self._verbose = flag - return self._verbose - - def noexec(self, flag = None): - if flag: self._noexec = flag - return self._noexec - - def reformat(self, flag = None): - if flag: self._reformat = flag - return self._reformat - - def cleanup(self, flag = None): - if flag: self._cleanup = flag - return self._cleanup - - def gdb(self, flag = None): - if flag: self._gdb = flag - return self._gdb - - def nomod(self, flag = None): - if flag: self._nomod = flag - return self._nomod - - def nosetup(self, flag = None): - if flag: self._nosetup = flag - return self._nosetup - - def node(self, val = None): - if val: self._node = val - return self._node - - def url(self, val = None): - if val: self._url = val - return self._url - - def gdb_script(self): - if os.path.isdir('/r'): - return '/r' + self._gdb_script - else: - return self._gdb_script - - def debug_path(self): - if os.path.isdir('/r'): - return '/r' + self._debug_path - else: - return self._debug_path - - def src_dir(self, val = None): - if val: self._src_dir = val - return self._src_dir - - def dump_file(self, val = None): - if val: self._dump_file = val - return self._dump_file - -config = Config() - -# ============================================================ # debugging and error funcs def fixme(msg = "this feature"): - raise LconfError, msg + ' not implmemented yet.' + raise Lustre.LconfError, msg + ' not implemented yet.' def panic(*args): msg = string.join(map(str,args)) - if not config.noexec(): - raise LconfError(msg) + if not config.noexec: + raise Lustre.LconfError(msg) else: print "! " + msg @@ -170,10 +147,23 @@ def logall(msgs): print string.strip(s) def debug(*args): - if config.verbose(): + if config.verbose: msg = string.join(map(str,args)) print msg +# ack, python's builtin int() does not support '0x123' syntax. +# eval can do it, although what a hack! +def my_int(s): + try: + if s[0:2] == '0x': + return eval(s, {}, {}) + else: + return int(s) + except SyntaxError, e: + raise ValueError("not a number") + except NameError, e: + raise ValueError("not a number") + # ============================================================ # locally defined exceptions class CommandError (exceptions.Exception): @@ -199,9 +189,111 @@ class CommandError (exceptions.Exception): else: print self.cmd_err -class LconfError (exceptions.Exception): - def __init__(self, args): - self.args = args + +# ============================================================ +# handle daemons, like the acceptor +class DaemonHandler: + """ Manage starting and stopping a daemon. Assumes daemon manages + it's own pid file. """ + + def __init__(self, cmd): + self.command = cmd + self.path ="" + + def start(self): + if self.running(): + log(self.command, "already running.") + if not self.path: + self.path = find_prog(self.command) + if not self.path: + panic(self.command, "not found.") + ret, out = runcmd(self.path +' '+ self.command_line()) + if ret: + raise CommandError(self.path, out, ret) + + def stop(self): + if self.running(): + pid = self.read_pidfile() + try: + log ("killing process", pid) + os.kill(pid, 15) + #time.sleep(1) # let daemon die + except OSError, e: + log("unable to kill", self.command, e) + if self.running(): + log("unable to kill", self.command) + + def running(self): + pid = self.read_pidfile() + if pid: + try: + os.kill(pid, 0) + except OSError: + self.clean_pidfile() + else: + return 1 + return 0 + + def read_pidfile(self): + try: + fp = open(self.pidfile(), 'r') + pid = int(fp.read()) + fp.close() + return pid + except IOError: + return 0 + + def clean_pidfile(self): + """ Remove a stale pidfile """ + log("removing stale pidfile:", self.pidfile()) + try: + os.unlink(self.pidfile()) + except OSError, e: + log(self.pidfile(), e) + +class AcceptorHandler(DaemonHandler): + def __init__(self, port, net_type, send_mem, recv_mem, irq_aff): + DaemonHandler.__init__(self, "acceptor") + self.port = port + self.flags = '' + self.send_mem = send_mem + self.recv_mem = recv_mem + + if irq_aff: + self.flags = self.flags + ' -i' + + def pidfile(self): + return "/var/run/%s-%d.pid" % (self.command, self.port) + + def command_line(self): + return string.join(map(str,('-s', self.send_mem, '-r', self.recv_mem, self.flags, self.port))) + +acceptors = {} + +# start the acceptors +def run_acceptors(): + if config.lctl_dump or config.record: + return + for port in acceptors.keys(): + daemon = acceptors[port] + if not daemon.running(): + daemon.start() + +def run_one_acceptor(port): + if config.lctl_dump or config.record: + return + if acceptors.has_key(port): + daemon = acceptors[port] + if not daemon.running(): + daemon.start() + else: + panic("run_one_acceptor: No acceptor defined for port:", port) + +def stop_acceptor(port): + if acceptors.has_key(port): + daemon = acceptors[port] + if daemon.running(): + daemon.stop() # ============================================================ @@ -216,13 +308,32 @@ class LCTLInterface: Initialize close by finding the lctl binary. """ self.lctl = find_prog(cmd) + self.save_file = '' + self.record_device = '' if not self.lctl: - if config.noexec(): + if config.noexec: debug('! lctl not found') self.lctl = 'lctl' else: raise CommandError('lctl', "unable to find lctl binary.") - + + def use_save_file(self, file): + self.save_file = file + + def record(self, dev_name, logname): + log("Recording log", logname, "on", dev_name) + self.record_device = dev_name + self.record_log = logname + + def end_record(self): + log("End recording log", self.record_log, "on", self.record_device) + self.record_device = None + self.record_log = None + + def set_nonblock(self, fd): + fl = fcntl.fcntl(fd, F_GETFL) + fcntl.fcntl(fd, F_SETFL, fl | os.O_NDELAY) + def run(self, cmds): """ run lctl @@ -232,143 +343,322 @@ class LCTLInterface: should modify command line to accept multiple commands, or create complex command line options """ - debug("+", self.lctl, cmds) - if config.noexec(): return (0, []) - p = popen2.Popen3(self.lctl, 1) - p.tochild.write(cmds + "\n") - p.tochild.close() - out = p.fromchild.readlines() - err = p.childerr.readlines() - ret = p.wait() - if ret or len(err): - raise CommandError(self.lctl, err, ret) - return ret, out + cmd_line = self.lctl + if self.save_file: + cmds = '\n dump ' + self.save_file + '\n' + cmds + elif self.record_device: + cmds = """ + device $%s + record %s + %s""" % (self.record_device, self.record_log, cmds) + + debug("+", cmd_line, cmds) + if config.noexec: return (0, []) + + child = popen2.Popen3(cmd_line, 1) # Capture stdout and stderr from command + child.tochild.write(cmds + "\n") + child.tochild.close() +# print "LCTL:", cmds + + # From "Python Cookbook" from O'Reilly + outfile = child.fromchild + outfd = outfile.fileno() + self.set_nonblock(outfd) + errfile = child.childerr + errfd = errfile.fileno() + self.set_nonblock(errfd) + + outdata = errdata = '' + outeof = erreof = 0 + while 1: + ready = select.select([outfd,errfd],[],[]) # Wait for input + if outfd in ready[0]: + outchunk = outfile.read() + if outchunk == '': outeof = 1 + outdata = outdata + outchunk + if errfd in ready[0]: + errchunk = errfile.read() + if errchunk == '': erreof = 1 + errdata = errdata + errchunk + if outeof and erreof: break + # end of "borrowed" code + + ret = child.wait() + if os.WIFEXITED(ret): + rc = os.WEXITSTATUS(ret) + else: + rc = 0 + if rc or len(errdata): + raise CommandError(self.lctl, errdata, rc) + return rc, outdata + + def runcmd(self, *args): + """ + run lctl using the command line + """ + cmd = string.join(map(str,args)) + debug("+", self.lctl, cmd) + rc, out = run(self.lctl, cmd) + if rc: + raise CommandError(self.lctl, out, rc) + return rc, out + + + def clear_log(self, dev, log): + """ clear an existing log """ + cmds = """ + device $%s + probe + clear_log %s + quit """ % (dev, log) + self.run(cmds) - def network(self, net, nid): - """ initialized network and add "self" """ - # Idea: "mynid" could be used for all network types to add "self," and then - # this special case would be gone and the "self" hack would be hidden. - if net == 'tcp': - cmds = """ + """ set mynid """ + cmds = """ network %s mynid %s - add_uuid self %s - quit""" % (net, nid, nid) - else: - cmds = """ - network %s - add_uuid self %s - quit""" % (net, nid) - + quit """ % (net, nid) self.run(cmds) # create a new connection - def connect(self, net, nid, port, servuuid, send_mem, recv_mem): - if net == 'tcp': + def add_uuid(self, net_type, uuid, nid): + cmds = "\n add_uuid %s %s %s" %(uuid, nid, net_type) + self.run(cmds) + + def add_autoconn(self, net_type, send_mem, recv_mem, nid, hostaddr, + port, flags): + if net_type in ('tcp',) and not config.lctl_dump: cmds = """ network %s - add_uuid %s %s send_mem %d recv_mem %d - connect %s %d - quit""" % (net, servuuid, nid, send_mem, recv_mem, nid, port, ) - else: - cmds = """ - network %s - add_uuid %s %s - connect %s %d - quit""" % (net, servuuid, nid, nid, port, ) - + add_autoconn %s %s %d %s + quit""" % (net_type, + send_mem, + recv_mem, + nid, hostaddr, port, flags ) + self.run(cmds) + + def connect(self, srv): + self.add_uuid(srv.net_type, srv.nid_uuid, srv.nid) + if srv.net_type in ('tcp',) and not config.lctl_dump: + flags = 's' + if srv.irq_affinity: + flags = flags + 'i' + self.add_autoconn(srv.net_type, srv.send_mem, srv.recv_mem, + srv.nid, srv.hostaddr, srv.port, flags) + + # Recover a device + def recover(self, dev_name, new_conn): + cmds = """ + device $%s + recover %s""" %(dev_name, new_conn) self.run(cmds) - + # add a route to a range def add_route(self, net, gw, lo, hi): cmds = """ network %s add_route %s %s %s - quit """ % (net, gw, lo, hi) - self.run(cmds) + quit """ % (net, + gw, lo, hi) + try: + self.run(cmds) + except CommandError, e: + log ("ignore: ") + e.dump() - - # add a route to a range def del_route(self, net, gw, lo, hi): cmds = """ + ignore_errors network %s - del_route %s - quit """ % (net, lo) + del_route %s %s %s + quit """ % (net, gw, lo, hi) self.run(cmds) # add a route to a host def add_route_host(self, net, uuid, gw, tgt): + self.add_uuid(net, uuid, tgt) cmds = """ network %s - add_uuid %s %s add_route %s %s - quit """ % (net, uuid, tgt, gw, tgt) - self.run(cmds) + quit """ % (net, + gw, tgt) + try: + self.run(cmds) + except CommandError, e: + log ("ignore: ") + e.dump() # add a route to a range def del_route_host(self, net, uuid, gw, tgt): + self.del_uuid(uuid) cmds = """ + ignore_errors network %s - del_uuid %s - del_route %s - quit """ % (net, uuid, tgt) + del_route %s %s + quit """ % (net, gw, tgt) self.run(cmds) + + def del_autoconn(self, net_type, nid, hostaddr): + if net_type in ('tcp',) and not config.lctl_dump: + cmds = """ + ignore_errors + network %s + del_autoconn %s %s s + quit""" % (net_type, + nid, hostaddr) + self.run(cmds) + # disconnect one connection - def disconnect(self, net, nid, port, servuuid): + def disconnect(self, srv): + self.del_uuid(srv.nid_uuid) + if srv.net_type in ('tcp',) and not config.lctl_dump: + self.del_autoconn(srv.net_type, srv.nid, srv.hostaddr) + + def del_uuid(self, uuid): cmds = """ - network %s - disconnect %s + ignore_errors del_uuid %s - quit""" % (net, nid, servuuid) + quit""" % (uuid,) self.run(cmds) # disconnect all def disconnectAll(self, net): cmds = """ + ignore_errors network %s - del_uuid self disconnect quit""" % (net) self.run(cmds) - # create a new device with lctl - def newdev(self, attach, setup = ""): + def attach(self, type, name, uuid): cmds = """ - newdev - attach %s + attach %s %s %s + quit""" % (type, name, uuid) + self.run(cmds) + + def setup(self, name, setup = ""): + cmds = """ + cfg_device %s setup %s - quit""" % (attach, setup) + quit""" % (name, setup) self.run(cmds) + + # create a new device with lctl + def newdev(self, type, name, uuid, setup = ""): + self.attach(type, name, uuid); + try: + self.setup(name, setup) + except CommandError, e: + self.cleanup(name, uuid, 0) + raise e + + # cleanup a device - def cleanup(self, name, uuid): + def cleanup(self, name, uuid, force, failover = 0): + if failover: force = 1 cmds = """ - device $%s - cleanup + ignore_errors + cfg_device $%s + cleanup %s %s detach - quit""" % (name) + quit""" % (name, ('', 'force')[force], + ('', 'failover')[failover]) self.run(cmds) # create an lov - def lovconfig(self, uuid, mdsuuid, stripe_cnt, stripe_sz, stripe_off, pattern, devlist): + def lov_setup(self, name, uuid, desc_uuid, stripe_cnt, + stripe_sz, stripe_off, pattern): + cmds = """ + attach lov %s %s + lov_setup %s %d %d %d %s + quit""" % (name, uuid, desc_uuid, stripe_cnt, stripe_sz, stripe_off, pattern) + self.run(cmds) + + # add an OBD to a LOV + def lov_add_obd(self, name, uuid, obd_uuid, index, gen): + cmds = """ + lov_modify_tgts add %s %s %s %s + quit""" % (name, obd_uuid, index, gen) + self.run(cmds) + + # create an lmv + def lmv_setup(self, name, uuid, desc_uuid, devlist): + cmds = """ + attach lmv %s %s + lmv_setup %s %s + quit""" % (name, uuid, desc_uuid, devlist) + self.run(cmds) + + # delete an OBD from a LOV + def lov_del_obd(self, name, uuid, obd_uuid, index, gen): + cmds = """ + lov_modify_tgts del %s %s %s %s + quit""" % (name, obd_uuid, index, gen) + self.run(cmds) + + # deactivate an OBD + def deactivate(self, name): cmds = """ device $%s - probe - lovconfig %s %d %d %d %s %s - quit""" % (mdsuuid, uuid, stripe_cnt, stripe_sz, stripe_off, pattern, devlist) + deactivate + quit""" % (name) self.run(cmds) - # cleanup a device + # dump the log file def dump(self, dump_file): cmds = """ debug_kernel %s 1 quit""" % (dump_file) self.run(cmds) + # get list of devices + def device_list(self): + devices = '/proc/fs/lustre/devices' + ret = [] + if os.access(devices, os.R_OK): + try: + fp = open(devices, 'r') + ret = fp.readlines() + fp.close() + except IOError, e: + log(e) + return ret + + # get lustre version + def lustre_version(self): + rc, out = self.runcmd('version') + return out + + # dump mount options + def mount_option(self, profile, osc, mdc): + cmds = """ + mount_option %s %s %s + quit""" % (profile, osc, mdc) + self.run(cmds) + + # delete mount options + def del_mount_option(self, profile): + cmds = """ + del_mount_option %s + quit""" % (profile,) + self.run(cmds) + + def set_timeout(self, timeout): + cmds = """ + set_timeout %s + quit""" % (timeout,) + self.run(cmds) + + def set_lustre_upcall(self, upcall): + cmds = """ + set_lustre_upcall %s + quit""" % (upcall,) + self.run(cmds) # ============================================================ # Various system-level functions # (ideally moved to their own module) @@ -376,10 +666,9 @@ class LCTLInterface: # Run a command and return the output and status. # stderr is sent to /dev/null, could use popen3 to # save it if necessary -def run(*args): - cmd = string.join(map(str,args)) +def runcmd(cmd): debug ("+", cmd) - if config.noexec(): return (0, []) + if config.noexec: return (0, []) f = os.popen(cmd + ' 2>&1') out = f.readlines() ret = f.close() @@ -389,11 +678,15 @@ def run(*args): ret = 0 return (ret, out) +def run(*args): + cmd = string.join(map(str,args)) + return runcmd(cmd) + # Run a command in the background. def run_daemon(*args): cmd = string.join(map(str,args)) debug ("+", cmd) - if config.noexec(): return 0 + if config.noexec: return 0 f = os.popen(cmd + ' 2>&1') ret = f.close() if ret: @@ -408,7 +701,8 @@ def find_prog(cmd): syspath = string.split(os.environ['PATH'], ':') cmdpath = os.path.dirname(sys.argv[0]) syspath.insert(0, cmdpath); - syspath.insert(0, os.path.join(cmdpath, '../../portals/linux/utils/')) + if config.portals: + syspath.insert(0, os.path.join(config.portals, 'utils/')) for d in syspath: prog = os.path.join(d,cmd) if os.access(prog, os.X_OK): @@ -428,13 +722,14 @@ def do_find_file(base, mod): return module def find_module(src_dir, dev_dir, modname): - mod = '%s.o' % (modname) - module = src_dir +'/'+ dev_dir +'/'+ mod - try: - if os.access(module, os.R_OK): - return module - except OSError: - pass + modbase = src_dir +'/'+ dev_dir +'/'+ modname + for modext in '.ko', '.o': + module = modbase + modext + try: + if os.access(module, os.R_OK): + return module + except OSError: + pass return None # is the path a block device? @@ -445,23 +740,55 @@ def is_block(path): except OSError: return 0 return stat.S_ISBLK(s[stat.ST_MODE]) - + # build fs according to type # fixme: dangerous -def mkfs(fstype, dev): - if(fstype in ('ext3', 'extN')): - mkfs = 'mkfs.ext2 -j -b 4096' - else: - print 'unsupported fs type: ', fstype - if not is_block(dev): - force = '-F' +def mkfs(dev, devsize, fstype, jsize, isize, mkfsoptions, isblock=1): + block_cnt = '' + jopt = '' + iopt = '' + if devsize: + if devsize < 8000: + panic("size of filesystem on '%s' must be larger than 8MB, but is set to %s"% + (dev, devsize)) + # devsize is in 1k, and fs block count is in 4k + block_cnt = devsize/4 + + if fstype in ('ext3', 'extN', 'ldiskfs'): + # ext3 journal size is in megabytes + if jsize == 0: + if devsize == 0: + if not is_block(dev): + ret, out = runcmd("ls -l %s" %dev) + devsize = int(string.split(out[0])[4]) / 1024 + else: + ret, out = runcmd("sfdisk -s %s" %dev) + devsize = int(out[0]) + if devsize > 1024 * 1024: + jsize = ((devsize / 102400) * 4) + if jsize > 400: + jsize = 400 + if jsize: jopt = "-J size=%d" %(jsize,) + if isize: iopt = "-I %d" %(isize,) + mkfs = 'mkfs.ext2 -j -b 4096 ' + if not isblock or config.force: + mkfs = mkfs + ' -F ' + elif fstype == 'reiserfs': + # reiserfs journal size is in blocks + if jsize: jopt = "--journal_size %d" %(jsize,) + mkfs = 'mkreiserfs -ff' else: - force = '' - (ret, out) = run (mkfs, force, dev) + panic('unsupported fs type: ', fstype) + + if config.mkfsoptions != None: + mkfs = mkfs + ' ' + config.mkfsoptions + if mkfsoptions != None: + mkfs = mkfs + ' ' + mkfsoptions + (ret, out) = run (mkfs, jopt, iopt, dev, block_cnt) if ret: - panic("Unable to build fs:", dev) - # enable hash tree indexing on fs - if fstype == 'extN': + panic("Unable to build fs:", dev, string.join(out)) + # enable hash tree indexing on fsswe + if fstype in ('ext3', 'extN', 'ldiskfs'): htree = 'echo "feature FEATURE_C5" | debugfs -w' (ret, out) = run (htree, dev) if ret: @@ -477,14 +804,14 @@ def loop_base(): panic ("can't access loop devices") return loop -# find loop device assigned to thefile -def find_loop(file): +# find loop device assigned to the file +def find_assigned_loop(file): loop = loop_base() for n in xrange(0, MAX_LOOP_DEVICES): dev = loop + str(n) if os.access(dev, os.R_OK): (stat, out) = run('losetup', dev) - if (out and stat == 0): + if out and stat == 0: m = re.search(r'\((.*)\)', out[0]) if m and file == m.group(1): return dev @@ -493,21 +820,42 @@ def find_loop(file): return '' # create file if necessary and assign the first free loop device -def init_loop(file, size, fstype): - dev = find_loop(file) +def init_loop(file, size, fstype, journal_size, inode_size, + mkfsoptions, reformat, autoformat, backfstype, backfile): + if fstype == 'smfs': + realfile = backfile + realfstype = backfstype + if is_block(backfile): + if reformat or (need_format(realfstype, backfile) and autoformat == 'yes'): + mkfs(realfile, size, realfstype, journal_size, inode_size, mkfsoptions, isblock=0) + return realfile + else: + realfile = file + realfstype = fstype + + dev = find_assigned_loop(realfile) if dev: - print 'WARNING file:', file, 'already mapped to', dev + print 'WARNING file:', realfile, 'already mapped to', dev return dev - if config.reformat() or not os.access(file, os.R_OK | os.W_OK): - run("dd if=/dev/zero bs=1k count=0 seek=%d of=%s" %(size, file)) + + if reformat or not os.access(realfile, os.R_OK | os.W_OK): + if size < 8000: + panic("size of loopback file '%s' must be larger than 8MB, but is set to %s" % (realfile, size)) + (ret, out) = run("dd if=/dev/zero bs=1k count=0 seek=%d of=%s" %(size, realfile)) + if ret: + panic("Unable to create backing store:", realfile) + + mkfs(realfile, size, realfstype, journal_size, inode_size, + mkfsoptions, isblock=0) + loop = loop_base() # find next free loop for n in xrange(0, MAX_LOOP_DEVICES): dev = loop + str(n) if os.access(dev, os.R_OK): (stat, out) = run('losetup', dev) - if (stat): - run('losetup', dev, file) + if stat: + run('losetup', dev, realfile) return dev else: print "out of loop devices" @@ -517,7 +865,7 @@ def init_loop(file, size, fstype): # undo loop assignment def clean_loop(file): - dev = find_loop(file) + dev = find_assigned_loop(file) if dev: ret, out = run('losetup -d', dev) if ret: @@ -526,22 +874,26 @@ def clean_loop(file): # determine if dev is formatted as a filesystem def need_format(fstype, dev): - # FIXME don't know how to implement this + # FIXME don't know how to implement this return 0 # initialize a block device if needed -def block_dev(dev, size, fstype, format): - if config.noexec(): return dev - if not is_block(dev): - dev = init_loop(dev, size, fstype) - if config.reformat() or (need_format(fstype, dev) and format == 'yes'): - mkfs(fstype, dev) - +def block_dev(dev, size, fstype, reformat, autoformat, journal_size, + inode_size, mkfsoptions, backfstype, backdev): + if config.noexec: + return dev + + if fstype == 'smfs' or not is_block(dev): + dev = init_loop(dev, size, fstype, journal_size, inode_size, + mkfsoptions, reformat, autoformat, backfstype, backdev) + elif reformat or (need_format(fstype, dev) and autoformat == 'yes'): + mkfs(dev, size, fstype, journal_size, inode_size, mkfsoptions, + isblock=0) # else: # panic("device:", dev, # "not prepared, and autoformat is not set.\n", # "Rerun with --reformat option to format ALL filesystems") - + return dev def if2addr(iface): @@ -553,10 +905,37 @@ def if2addr(iface): ip = string.split(addr, ':')[1] return ip -def get_local_address(net_type, wildcard): +def def_mount_options(fstype, target): + """returns deafult mount options for passed fstype and target (mds, ost)""" + if fstype == 'ext3' or fstype == 'ldiskfs': + mountfsoptions = "errors=remount-ro" + if target == 'ost' and sys_get_branch() == '2.4': + mountfsoptions = "%s,asyncdel" % (mountfsoptions) + return mountfsoptions + return "" + +def sys_get_elan_position_file(): + procfiles = ["/proc/elan/device0/position", + "/proc/qsnet/elan4/device0/position", + "/proc/qsnet/elan3/device0/position"] + for p in procfiles: + if os.access(p, os.R_OK): + return p + return "" + +def sys_get_local_nid(net_type, wildcard, cluster_id): + """Return the local nid.""" + local = "" + if sys_get_elan_position_file(): + local = sys_get_local_address('elan', '*', cluster_id) + else: + local = sys_get_local_address(net_type, wildcard, cluster_id) + return local + +def sys_get_local_address(net_type, wildcard, cluster_id): """Return the local address for the network type.""" local = "" - if net_type == 'tcp': + if net_type in ('tcp',): if ':' in wildcard: iface, star = string.split(wildcard, ':') local = if2addr(iface) @@ -566,77 +945,50 @@ def get_local_address(net_type, wildcard): host = socket.gethostname() local = socket.gethostbyname(host) elif net_type == 'elan': - # awk '/NodeId/ { print $2 }' '/proc/elan/device0/position' + # awk '/NodeId/ { print $2 }' 'sys_get_elan_position_file()' + f = sys_get_elan_position_file() + if not f: + panic ("unable to determine local Elan ID") try: - fp = open('/proc/elan/device0/position', 'r') + fp = open(f, 'r') lines = fp.readlines() fp.close() for l in lines: a = string.split(l) if a[0] == 'NodeId': - local = a[1] + elan_id = a[1] break + try: + nid = my_int(cluster_id) + my_int(elan_id) + local = "%d" % (nid) + except ValueError, e: + local = elan_id except IOError, e: log(e) elif net_type == 'gm': fixme("automatic local address for GM") + return local - - -# ============================================================ -# Classes to prepare and cleanup the various objects -# -class Module: - """ Base class for the rest of the modules. The default cleanup method is - defined here, as well as some utilitiy funcs. - """ - def __init__(self, module_name, dom_node): - self.dom_node = dom_node - self.module_name = module_name - self.name = get_attr(dom_node, 'name') - self.uuid = get_attr(dom_node, 'uuid') - self.kmodule_list = [] - self._server = None - self._connected = 0 +def sys_get_branch(): + """Returns kernel release""" + try: + fp = open('/proc/sys/kernel/osrelease') + lines = fp.readlines() + fp.close() - def info(self, *args): - msg = string.join(map(str,args)) - print self.module_name + ":", self.name, self.uuid, msg - - - def lookup_server(self, srv_uuid): - """ Lookup a server's network information """ - net = get_ost_net(self.dom_node.parentNode, srv_uuid) - if not net: - panic ("Unable to find a server for:", srv_uuid) - self._server = Network(net) - - def get_server(self): - return self._server - - def cleanup(self): - """ default cleanup, used for most modules """ - self.info() - srv = self.get_server() - if srv and local_net(srv): - try: - lctl.disconnect(srv.net_type, srv.nid, srv.port, srv.uuid) - except CommandError, e: - log(self.module_name, "disconnect failed: ", self.name) - e.dump() - try: - lctl.cleanup(self.name, self.uuid) - except CommandError, e: - log(self.module_name, "cleanup failed: ", self.name) - e.dump() + for l in lines: + version = string.split(l) + a = string.split(version[0], '.') + return a[0] + '.' + a[1] + except IOError, e: + log(e) + return "" - def add_module(self, dev_dir, modname): - """Append a module to list of modules to load.""" - self.kmodule_list.append((dev_dir, modname)) - def mod_loaded(self, modname): - """Check if a module is already loaded. Look in /proc/modules for it.""" +def mod_loaded(modname): + """Check if a module is already loaded. Look in /proc/modules for it.""" + try: fp = open('/proc/modules') lines = fp.readlines() fp.close() @@ -644,16 +996,71 @@ class Module: ret = filter(lambda word, mod=modname: word == mod, map(lambda line: string.split(line)[0], lines)) return ret + except Exception, e: + return 0 + +# XXX: instead of device_list, ask for $name and see what we get +def is_prepared(name): + """Return true if a device exists for the name""" + if config.lctl_dump: + return 0 + if (config.noexec or config.record) and config.cleanup: + return 1 + try: + # expect this format: + # 1 UP ldlm ldlm ldlm_UUID 2 + out = lctl.device_list() + for s in out: + if name == string.split(s)[3]: + return 1 + except CommandError, e: + e.dump() + return 0 + +def is_network_prepared(): + """If the any device exists, then assume that all networking + has been configured""" + out = lctl.device_list() + return len(out) > 0 + +def fs_is_mounted(path): + """Return true if path is a mounted lustre filesystem""" + try: + fp = open('/proc/mounts') + lines = fp.readlines() + fp.close() + for l in lines: + a = string.split(l) + if a[1] == path and a[2] == 'lustre_lite': + return 1 + except IOError, e: + log(e) + return 0 + + +class kmod: + """Manage kernel modules""" + def __init__(self, lustre_dir, portals_dir): + self.lustre_dir = lustre_dir + self.portals_dir = portals_dir + self.kmodule_list = [] + + def add_portals_module(self, dev_dir, modname): + """Append a module to list of modules to load.""" + self.kmodule_list.append((self.portals_dir, dev_dir, modname)) + + def add_lustre_module(self, dev_dir, modname): + """Append a module to list of modules to load.""" + self.kmodule_list.append((self.lustre_dir, dev_dir, modname)) def load_module(self): """Load all the modules in the list in the order they appear.""" - for dev_dir, mod in self.kmodule_list: - # (rc, out) = run ('/sbin/lsmod | grep -s', mod) - if self.mod_loaded(mod) and not config.noexec(): + for src_dir, dev_dir, mod in self.kmodule_list: + if mod_loaded(mod) and not config.noexec: continue - log ('loading module:', mod) - if config.src_dir(): - module = find_module(config.src_dir(),dev_dir, mod) + log ('loading module:', mod, 'srcdir', src_dir, 'devdir', dev_dir) + if src_dir: + module = find_module(src_dir, dev_dir, mod) if not module: panic('module not found:', mod) (rc, out) = run('/sbin/insmod', module) @@ -663,297 +1070,1286 @@ class Module: (rc, out) = run('/sbin/modprobe', mod) if rc: raise CommandError('modprobe', out, rc) - + def cleanup_module(self): """Unload the modules in the list in reverse order.""" rev = self.kmodule_list rev.reverse() - for dev_dir, mod in rev: - if not self.mod_loaded(mod): + for src_dir, dev_dir, mod in rev: + if not mod_loaded(mod) and not config.noexec: continue # debug hack - if mod == 'portals' and config.dump_file(): - lctl.dump(config.dump_file()) + if mod == 'portals' and config.dump: + lctl.dump(config.dump) log('unloading module:', mod) - if config.noexec(): - continue (rc, out) = run('/sbin/rmmod', mod) if rc: log('! unable to unload module:', mod) logall(out) - -class Network(Module): - def __init__(self,dom_node): - Module.__init__(self, 'NETWORK', dom_node) - self.net_type = get_attr(dom_node,'type') - self.nid = get_text(dom_node, 'server', '*') - self.port = get_text_int(dom_node, 'port', 0) - self.send_mem = get_text_int(dom_node, 'send_mem', DEFAULT_TCPBUF) - self.recv_mem = get_text_int(dom_node, 'recv_mem', DEFAULT_TCPBUF) - if '*' in self.nid: - self.nid = get_local_address(self.net_type, self.nid) - if not self.nid: - panic("unable to set nid for", self.net_type, self.nid) - debug("nid:", self.nid) +# ============================================================ +# Classes to prepare and cleanup the various objects +# +class Module: + """ Base class for the rest of the modules. The default cleanup method is + defined here, as well as some utilitiy funcs. + """ + def __init__(self, module_name, db): + self.db = db + self.module_name = module_name + self.name = self.db.getName() + self.uuid = self.db.getUUID() + self._server = None + self._connected = 0 + self.kmod = kmod(config.lustre, config.portals) - self.add_module('portals/linux/oslib/', 'portals') - if node_needs_router(): - self.add_module('portals/linux/router', 'kptlrouter') - if self.net_type == 'tcp': - self.add_module('portals/linux/socknal', 'ksocknal') - if self.net_type == 'elan': - self.add_module('portals/linux/rqswnal', 'kqswnal') - if self.net_type == 'gm': - self.add_module('portals/linux/gmnal', 'kgmnal') - self.add_module('lustre/obdclass', 'obdclass') - self.add_module('lustre/ptlrpc', 'ptlrpc') + def info(self, *args): + msg = string.join(map(str,args)) + print self.module_name + ":", self.name, self.uuid, msg + + def cleanup(self): + """ default cleanup, used for most modules """ + self.info() + try: + lctl.cleanup(self.name, self.uuid, config.force) + except CommandError, e: + log(self.module_name, "cleanup failed: ", self.name) + e.dump() + cleanup_error(e.rc) + + def add_portals_module(self, dev_dir, modname): + """Append a module to list of modules to load.""" + self.kmod.add_portals_module(dev_dir, modname) + + def add_lustre_module(self, dev_dir, modname): + """Append a module to list of modules to load.""" + self.kmod.add_lustre_module(dev_dir, modname) + + def load_module(self): + """Load all the modules in the list in the order they appear.""" + self.kmod.load_module() + + def cleanup_module(self): + """Unload the modules in the list in reverse order.""" + if self.safe_to_clean(): + self.kmod.cleanup_module() + + def safe_to_clean(self): + return 1 + + def safe_to_clean_modules(self): + return self.safe_to_clean() + +class Network(Module): + def __init__(self,db): + Module.__init__(self, 'NETWORK', db) + self.net_type = self.db.get_val('nettype') + self.nid = self.db.get_val('nid', '*') + self.cluster_id = self.db.get_val('clusterid', "0") + self.port = self.db.get_val_int('port', 0) + self.send_mem = self.db.get_val_int('sendmem', DEFAULT_TCPBUF) + self.recv_mem = self.db.get_val_int('recvmem', DEFAULT_TCPBUF) + self.irq_affinity = self.db.get_val_int('irqaffinity', 0) + + if '*' in self.nid: + self.nid = sys_get_local_nid(self.net_type, self.nid, self.cluster_id) + if not self.nid: + panic("unable to set nid for", self.net_type, self.nid, cluster_id) + self.generic_nid = 1 + debug("nid:", self.nid) + else: + self.generic_nid = 0 + + self.nid_uuid = self.nid_to_uuid(self.nid) + + self.hostaddr = self.db.get_val('hostaddr', self.nid) + if '*' in self.hostaddr: + self.hostaddr = sys_get_local_address(self.net_type, self.hostaddr, self.cluster_id) + if not self.hostaddr: + panic("unable to set hostaddr for", self.net_type, self.hostaddr, self.cluster_id) + debug("hostaddr:", self.hostaddr) + + self.add_portals_module("libcfs", 'libcfs') + self.add_portals_module("portals", 'portals') + if node_needs_router(): + self.add_portals_module("router", 'kptlrouter') + if self.net_type == 'tcp': + self.add_portals_module("knals/socknal", 'ksocknal') + if self.net_type == 'elan': + self.add_portals_module("knals/qswnal", 'kqswnal') + if self.net_type == 'gm': + self.add_portals_module("knals/gmnal", 'kgmnal') + + def nid_to_uuid(self, nid): + return "NID_%s_UUID" %(nid,) def prepare(self): + if not config.record and is_network_prepared(): + return self.info(self.net_type, self.nid, self.port) + if not (config.record and self.generic_nid): + lctl.network(self.net_type, self.nid) if self.net_type == 'tcp': - ret, out = run(TCP_ACCEPTOR, '-s', self.send_mem, '-r', self.recv_mem, self.port) - if ret: - raise CommandError(TCP_ACCEPTOR, out, ret) - ret = self.dom_node.getElementsByTagName('route_tbl') - for a in ret: - for r in a.getElementsByTagName('route'): - net_type = get_attr(r, 'type') - gw = get_attr(r, 'gw') - lo = get_attr(r, 'lo') - hi = get_attr(r,'hi', '') - lctl.add_route(net_type, gw, lo, hi) - if self.net_type == 'tcp' and hi == '': - srv = nid2server(self.dom_node.parentNode.parentNode, lo) - if not srv: - panic("no server for nid", lo) - else: - lctl.connect(srv.net_type, srv.nid, srv.port, srv.uuid, srv.send_mem, srv.recv_mem) - - - lctl.network(self.net_type, self.nid) - lctl.newdev(attach = "ptlrpc RPCDEV") + sys_tweak_socknal() + if self.net_type == 'elan': + sys_optimize_elan() + if self.port and node_is_router(): + run_one_acceptor(self.port) + self.connect_peer_gateways() + + def connect_peer_gateways(self): + for router in self.db.lookup_class('node'): + if router.get_val_int('router', 0): + for netuuid in router.get_networks(): + net = self.db.lookup(netuuid) + gw = Network(net) + if (gw.cluster_id == self.cluster_id and + gw.net_type == self.net_type): + if gw.nid != self.nid: + lctl.connect(gw) + + def disconnect_peer_gateways(self): + for router in self.db.lookup_class('node'): + if router.get_val_int('router', 0): + for netuuid in router.get_networks(): + net = self.db.lookup(netuuid) + gw = Network(net) + if (gw.cluster_id == self.cluster_id and + gw.net_type == self.net_type): + if gw.nid != self.nid: + try: + lctl.disconnect(gw) + except CommandError, e: + print "disconnect failed: ", self.name + e.dump() + cleanup_error(e.rc) + + def safe_to_clean(self): + return not is_network_prepared() def cleanup(self): self.info(self.net_type, self.nid, self.port) - ret = self.dom_node.getElementsByTagName('route_tbl') - for a in ret: - for r in a.getElementsByTagName('route'): - lo = get_attr(r, 'lo') - hi = get_attr(r,'hi', '') - if self.net_type == 'tcp' and hi == '': - srv = nid2server(self.dom_node.parentNode.parentNode, lo) - if not srv: - panic("no server for nid", lo) - else: - try: - lctl.disconnect(srv.net_type, srv.nid, srv.port, srv.uuid) - except CommandError, e: - print "disconnect failed: ", self.name - e.dump() + if self.port: + stop_acceptor(self.port) + if node_is_router(): + self.disconnect_peer_gateways() + + def correct_level(self, level, op=None): + return level + +class RouteTable(Module): + def __init__(self,db): + Module.__init__(self, 'ROUTES', db) + + def server_for_route(self, net_type, gw, gw_cluster_id, tgt_cluster_id, + lo, hi): + # only setup connections for tcp NALs + srvdb = None + if not net_type in ('tcp',): + return None + + # connect to target if route is to single node and this node is the gw + if lo == hi and local_interface(net_type, gw_cluster_id, gw): + if not local_cluster(net_type, tgt_cluster_id): + panic("target", lo, " not on the local cluster") + srvdb = self.db.nid2server(lo, net_type, gw_cluster_id) + # connect to gateway if this node is not the gw + elif (local_cluster(net_type, gw_cluster_id) + and not local_interface(net_type, gw_cluster_id, gw)): + srvdb = self.db.nid2server(gw, net_type, gw_cluster_id) + else: + return None + + if not srvdb: + panic("no server for nid", lo) + return None + + return Network(srvdb) + + def prepare(self): + if not config.record and is_network_prepared(): + return + self.info() + for net_type, gw, gw_cluster_id, tgt_cluster_id, lo, hi in self.db.get_route_tbl(): + lctl.add_route(net_type, gw, lo, hi) + srv = self.server_for_route(net_type, gw, gw_cluster_id, tgt_cluster_id, lo, hi) + if srv: + lctl.connect(srv) + + def safe_to_clean(self): + return not is_network_prepared() + + def cleanup(self): + if is_network_prepared(): + # the network is still being used, don't clean it up + return + for net_type, gw, gw_cluster_id, tgt_cluster_id, lo, hi in self.db.get_route_tbl(): + srv = self.server_for_route(net_type, gw, gw_cluster_id, tgt_cluster_id, lo, hi) + if srv: try: - lctl.del_route(self.net_type, self.nid, lo, hi) + lctl.disconnect(srv) except CommandError, e: - print "del_route failed: ", self.name + print "disconnect failed: ", self.name e.dump() - - try: - lctl.cleanup("RPCDEV", "") - except CommandError, e: - print "cleanup failed: ", self.name - e.dump() - try: - lctl.disconnectAll(self.net_type) - except CommandError, e: - print "disconnectAll failed: ", self.name - e.dump() - if self.net_type == 'tcp': - # yikes, this ugly! need to save pid in /var/something - run("killall acceptor") + cleanup_error(e.rc) + + try: + lctl.del_route(net_type, gw, lo, hi) + except CommandError, e: + print "del_route failed: ", self.name + e.dump() + cleanup_error(e.rc) + +class Management(Module): + def __init__(self, db): + Module.__init__(self, 'MGMT', db) + self.add_lustre_module('lvfs', 'lvfs') + self.add_lustre_module('obdclass', 'obdclass') + self.add_lustre_module('ptlrpc', 'ptlrpc') + self.add_lustre_module('mgmt', 'mgmt_svc') -class LDLM(Module): - def __init__(self,dom_node): - Module.__init__(self, 'LDLM', dom_node) - self.add_module('lustre/ldlm', 'ldlm') def prepare(self): + if not config.record and is_prepared(self.name): + return self.info() - lctl.newdev(attach="ldlm %s %s" % (self.name, self.uuid), - setup ="") + lctl.newdev("mgmt", self.name, self.uuid) -class LOV(Module): - def __init__(self,dom_node): - Module.__init__(self, 'LOV', dom_node) - self.mds_uuid = get_first_ref(dom_node, 'mds') - mds= lookup(dom_node.parentNode, self.mds_uuid) - self.mds_name = getName(mds) - devs = dom_node.getElementsByTagName('devices') - if len(devs) > 0: - dev_node = devs[0] - self.stripe_sz = get_attr_int(dev_node, 'stripesize', 65536) - self.stripe_off = get_attr_int(dev_node, 'stripeoffset', 0) - self.pattern = get_attr_int(dev_node, 'pattern', 0) - self.devlist = get_all_refs(dev_node, 'osc') - self.stripe_cnt = get_attr_int(dev_node, 'stripecount', len(self.devlist)) - self.add_module('lustre/mdc', 'mdc') - self.add_module('lustre/lov', 'lov') + def safe_to_clean(self): + return 1 + + def cleanup(self): + if is_prepared(self.name): + Module.cleanup(self) + + def correct_level(self, level, op=None): + return level + +# This is only needed to load the modules; the LDLM device +# is now created automatically. +class LDLM(Module): + def __init__(self,db): + Module.__init__(self, 'LDLM', db) + self.add_lustre_module('lvfs', 'lvfs') + self.add_lustre_module('obdclass', 'obdclass') + self.add_lustre_module('ptlrpc', 'ptlrpc') def prepare(self): - for osc_uuid in self.devlist: - osc = lookup(self.dom_node.parentNode, osc_uuid) + return + + def cleanup(self): + return + + def correct_level(self, level, op=None): + return level + + +class LOV(Module): + def __init__(self, db, uuid, fs_name, name_override = None, config_only = None): + Module.__init__(self, 'LOV', db) + if name_override != None: + self.name = "lov_%s" % name_override + self.add_lustre_module('lov', 'lov') + self.mds_uuid = self.db.get_first_ref('mds') + self.stripe_sz = self.db.get_val_int('stripesize', 1048576) + self.stripe_off = self.db.get_val_int('stripeoffset', 0) + self.pattern = self.db.get_val_int('stripepattern', 0) + self.devlist = self.db.get_lov_tgts('lov_tgt') + self.stripe_cnt = self.db.get_val_int('stripecount', len(self.devlist)) + self.osclist = [] + self.desc_uuid = self.uuid + self.uuid = generate_client_uuid(self.name) + self.fs_name = fs_name + if config_only: + self.config_only = 1 + return + self.config_only = None + mds = self.db.lookup(self.mds_uuid) + self.mds_name = mds.getName() + for (obd_uuid, index, gen, active) in self.devlist: + if obd_uuid == '': + continue + obd = self.db.lookup(obd_uuid) + osc = get_osc(obd, self.uuid, fs_name) if osc: - n = OSC(osc) - n.prepare() + self.osclist.append((osc, index, gen, active)) else: - panic('osc not found:', osc_uuid) - mdc_uuid = prepare_mdc(self.dom_node.parentNode, self.mds_uuid) + panic('osc not found:', obd_uuid) + def get_uuid(self): + return self.uuid + def get_name(self): + return self.name + def prepare(self): + if not config.record and is_prepared(self.name): + return self.info(self.mds_uuid, self.stripe_cnt, self.stripe_sz, - self.stripe_off, self.pattern, self.devlist, self.mds_name) - lctl.newdev(attach="lov %s %s" % (self.name, self.uuid), - setup ="%s" % (mdc_uuid)) + self.stripe_off, self.pattern, self.devlist, + self.mds_name) + lctl.lov_setup(self.name, self.uuid, self.desc_uuid, self.stripe_cnt, + self.stripe_sz, self.stripe_off, self.pattern) + for (osc, index, gen, active) in self.osclist: + target_uuid = osc.target_uuid + try: + # Only ignore connect failures with --force, which + # isn't implemented here yet. + osc.active = active + osc.prepare(ignore_connect_failure=0) + except CommandError, e: + print "Error preparing OSC %s\n" % osc.uuid + raise e + lctl.lov_add_obd(self.name, self.uuid, target_uuid, index, gen) def cleanup(self): - for osc_uuid in self.devlist: - osc = lookup(self.dom_node.parentNode, osc_uuid) - if osc: - n = OSC(osc) - n.cleanup() - else: - panic('osc not found:', osc_uuid) - Module.cleanup(self) - cleanup_mdc(self.dom_node.parentNode, self.mds_uuid) + for (osc, index, gen, active) in self.osclist: + target_uuid = osc.target_uuid + osc.cleanup() + if is_prepared(self.name): + Module.cleanup(self) + if self.config_only: + panic("Can't clean up config_only LOV ", self.name) + def load_module(self): - for osc_uuid in self.devlist: - osc = lookup(self.dom_node.parentNode, osc_uuid) - if osc: - n = OSC(osc) - n.load_module() - break + if self.config_only: + panic("Can't load modules for config_only LOV ", self.name) + for (osc, index, gen, active) in self.osclist: + osc.load_module() + break + Module.load_module(self) + + def cleanup_module(self): + if self.config_only: + panic("Can't cleanup modules for config_only LOV ", self.name) + Module.cleanup_module(self) + for (osc, index, gen, active) in self.osclist: + if active: + osc.cleanup_module() + break + + def correct_level(self, level, op=None): + return level + +class LMV(Module): + def __init__(self, db, uuid, fs_name, name_override = None): + Module.__init__(self, 'LMV', db) + if name_override != None: + self.name = "lmv_%s" % name_override + self.add_lustre_module('lmv', 'lmv') + self.devlist = self.db.get_refs('mds') + self.mdclist = [] + self.desc_uuid = self.uuid + self.uuid = uuid + self.fs_name = fs_name + for mds_uuid in self.devlist: + mds = self.db.lookup(mds_uuid) + if not mds: + panic("MDS not found!") + mdc = MDC(mds, self.uuid, fs_name) + if mdc: + self.mdclist.append(mdc) else: - panic('osc not found:', osc_uuid) + panic('mdc not found:', mds_uuid) + + def prepare(self): + if is_prepared(self.name): + return + for mdc in self.mdclist: + try: + # Only ignore connect failures with --force, which + # isn't implemented here yet. + mdc.prepare(ignore_connect_failure=0) + except CommandError, e: + print "Error preparing LMV %s\n" % mdc.uuid + raise e + lctl.lmv_setup(self.name, self.uuid, self.desc_uuid, + string.join(self.devlist)) + + def cleanup(self): + for mdc in self.mdclist: + mdc.cleanup() + if is_prepared(self.name): + Module.cleanup(self) + + def load_module(self): + for mdc in self.mdclist: + mdc.load_module() + break Module.load_module(self) + def cleanup_module(self): Module.cleanup_module(self) - for osc_uuid in self.devlist: - osc = lookup(self.dom_node.parentNode, osc_uuid) - if osc: - n = OSC(osc) - n.cleanup_module() - break + for mdc in self.mdclist: + mdc.cleanup_module() + break + + def correct_level(self, level, op=None): + return level + +class MDSDEV(Module): + def __init__(self,db): + Module.__init__(self, 'MDSDEV', db) + self.devpath = self.db.get_val('devpath','') + self.backdevpath = self.db.get_val('backdevpath','') + self.size = self.db.get_val_int('devsize', 0) + self.journal_size = self.db.get_val_int('journalsize', 0) + self.fstype = self.db.get_val('fstype', '') + self.backfstype = self.db.get_val('backfstype', '') + self.nspath = self.db.get_val('nspath', '') + self.mkfsoptions = self.db.get_val('mkfsoptions', '') + self.mountfsoptions = self.db.get_val('mountfsoptions', '') + self.cachetype = self.db.get_val('cachetype', '') + # overwrite the orignal MDSDEV name and uuid with the MDS name and uuid + target_uuid = self.db.get_first_ref('target') + mds = self.db.lookup(target_uuid) + self.name = mds.getName() + self.filesystem_uuids = mds.get_refs('filesystem') + self.lmv_uuid = '' + self.lmv = '' + self.master_mds = "" + if not self.filesystem_uuids: + self.lmv_uuid = self.db.get_first_ref('lmv') + if not self.lmv_uuid: + panic("ALERT: can't find lvm uuid") + if self.lmv_uuid: + self.lmv = self.db.lookup(self.lmv_uuid) + if self.lmv: + self.filesystem_uuids = self.lmv.get_refs('filesystem') + self.master_mds = self.lmv_uuid + # FIXME: if fstype not set, then determine based on kernel version + self.format = self.db.get_val('autoformat', "no") + if mds.get_val('failover', 0): + self.failover_mds = 'f' + else: + self.failover_mds = 'n' + active_uuid = get_active_target(mds) + if not active_uuid: + panic("No target device found:", target_uuid) + if active_uuid == self.uuid: + self.active = 1 + else: + self.active = 0 + if self.active and config.group and config.group != mds.get_val('group'): + self.active = 0 + + self.inode_size = self.db.get_val_int('inodesize', 0) + if self.inode_size == 0: + # find the LOV for this MDS + lovconfig_uuid = mds.get_first_ref('lovconfig') + if not lovconfig_uuid: + if not self.lmv_uuid: + panic("No LOV found for lovconfig ", lovconfig.name) + + if not self.lmv: + panic("No LMV initialized and not lovconfig_uuid found") + + lovconfig_uuid = self.lmv.get_first_ref('lovconfig') + lovconfig = self.lmv.lookup(lovconfig_uuid) + lov_uuid = lovconfig.get_first_ref('lov') + if not lov_uuid: + panic("No LOV found for lovconfig ", lovconfig.name) + else: + lovconfig = mds.lookup(lovconfig_uuid) + lov_uuid = lovconfig.get_first_ref('lov') + if not lov_uuid: + panic("No LOV found for lovconfig ", lovconfig.name) + + if self.lmv: + lovconfig_uuid = self.lmv.get_first_ref('lovconfig') + lovconfig = self.lmv.lookup(lovconfig_uuid) + lov_uuid = lovconfig.get_first_ref('lov') + + lov = LOV(self.db.lookup(lov_uuid), lov_uuid, 'FS_name', config_only = 1) + + # default stripe count controls default inode_size + stripe_count = lov.stripe_cnt + if stripe_count > 77: + self.inode_size = 4096 + elif stripe_count > 35: + self.inode_size = 2048 + elif stripe_count > 13: + self.inode_size = 1024 + elif stripe_count > 3: + self.inode_size = 512 else: - panic('osc not found:', osc_uuid) - -class LOVConfig(Module): - def __init__(self,dom_node): - Module.__init__(self, 'LOVConfig', dom_node) - self.lov_uuid = get_first_ref(dom_node, 'lov') - l = lookup(dom_node.parentNode, self.lov_uuid) - self.lov = LOV(l) + self.inode_size = 256 + + self.target_dev_uuid = self.uuid + self.uuid = target_uuid + # setup LMV + if self.master_mds: + client_uuid = generate_client_uuid(self.name) + client_uuid = self.name + "_lmv_" + "UUID" + self.master = LMV(self.db.lookup(self.lmv_uuid), client_uuid, self.name, self.name) + self.master_mds = self.master.name + + # modules + self.add_lustre_module('mdc', 'mdc') + self.add_lustre_module('osc', 'osc') + self.add_lustre_module('lov', 'lov') + self.add_lustre_module('lmv', 'lmv') + self.add_lustre_module('ost', 'ost') + self.add_lustre_module('mds', 'mds') + + if self.fstype == 'smfs': + self.add_lustre_module('smfs', 'smfs') + if self.fstype == 'ldiskfs': + self.add_lustre_module('ldiskfs', 'ldiskfs') + + if self.fstype: + self.add_lustre_module('lvfs', 'fsfilt_%s' % (self.fstype)) + + # if fstype is smfs, then we should also take care about backing + # store fs. + if self.fstype == 'smfs': + self.add_lustre_module('lvfs', 'fsfilt_%s' % (self.backfstype)) + + for options in string.split(self.mountfsoptions, ','): + if options == 'snap': + if not self.fstype == 'smfs': + panic("mountoptions with snap, but fstype is not smfs\n") + self.add_lustre_module('lvfs', 'fsfilt_snap_%s' % (self.fstype)) + self.add_lustre_module('lvfs', 'fsfilt_snap_%s' % (self.backfstype)) + def load_module(self): + if self.active: + Module.load_module(self) + def prepare(self): - lov = self.lov - self.info(lov.mds_uuid, lov.stripe_cnt, lov.stripe_sz, lov.stripe_off, - lov.pattern, lov.devlist, lov.mds_name) - lctl.lovconfig(lov.uuid, lov.mds_name, lov.stripe_cnt, - lov.stripe_sz, lov.stripe_off, lov.pattern, - string.join(lov.devlist)) + if not config.record and is_prepared(self.name): + return + if not self.active: + debug(self.uuid, "not active") + return + if config.reformat: + # run write_conf automatically, if --reformat used + self.write_conf() + self.info(self.devpath, self.fstype, self.size, self.format) + run_acceptors() + # prepare LMV + if self.master_mds: + self.master.prepare() + # never reformat here + blkdev = block_dev(self.devpath, self.size, self.fstype, 0, + self.format, self.journal_size, self.inode_size, + self.mkfsoptions, self.backfstype, self.backdevpath) + + if not is_prepared('MDT'): + lctl.newdev("mdt", 'MDT', 'MDT_UUID', setup ="") + try: + mountfsoptions = def_mount_options(self.fstype, 'mds') + + if config.mountfsoptions: + if mountfsoptions: + mountfsoptions = mountfsoptions + ',' + config.mountfsoptions + else: + mountfsoptions = config.mountfsoptions + if self.mountfsoptions: + mountfsoptions = mountfsoptions + ',' + self.mountfsoptions + else: + if self.mountfsoptions: + if mountfsoptions: + mountfsoptions = mountfsoptions + ',' + self.mountfsoptions + else: + mountfsoptions = self.mountfsoptions + + if self.fstype == 'smfs': + realdev = self.fstype + + if mountfsoptions: + mountfsoptions = "%s,type=%s,dev=%s" % (mountfsoptions, + self.backfstype, + blkdev) + else: + mountfsoptions = "type=%s,dev=%s" % (self.backfstype, + blkdev) + else: + realdev = blkdev + + print 'MDS mount options: ' + mountfsoptions + + if not self.master_mds: + self.master_mds = 'dumb' + if not self.cachetype: + self.cachetype = 'dumb' + lctl.newdev("mds", self.name, self.uuid, + setup ="%s %s %s %s %s %s" %(realdev, self.fstype, + self.name, mountfsoptions, + self.master_mds, self.cachetype)) + except CommandError, e: + if e.rc == 2: + panic("MDS is missing the config log. Need to run " + + "lconf --write_conf.") + else: + raise e + + def write_conf(self): + do_cleanup = 0 + if not is_prepared(self.name): + self.info(self.devpath, self.fstype, self.format) + + blkdev = block_dev(self.devpath, self.size, self.fstype, + config.reformat, self.format, self.journal_size, + self.inode_size, self.mkfsoptions, + self.backfstype, self.backdevpath) + + # Even for writing logs we mount mds with supplied mount options + # because it will not mount smfs (if used) otherwise. + + mountfsoptions = def_mount_options(self.fstype, 'mds') + + if config.mountfsoptions: + if mountfsoptions: + mountfsoptions = mountfsoptions + ',' + config.mountfsoptions + else: + mountfsoptions = config.mountfsoptions + if self.mountfsoptions: + mountfsoptions = mountfsoptions + ',' + self.mountfsoptions + else: + if self.mountfsoptions: + if mountfsoptions: + mountfsoptions = mountfsoptions + ',' + self.mountfsoptions + else: + mountfsoptions = self.mountfsoptions - def cleanup(self): - #nothing to do here - pass - - -class MDS(Module): - def __init__(self,dom_node): - Module.__init__(self, 'MDS', dom_node) - self.devname, self.size = get_device(dom_node) - self.fstype = get_text(dom_node, 'fstype') - self.format = get_text(dom_node, 'autoformat', "no") - if self.fstype == 'extN': - self.add_module('lustre/extN', 'extN') - self.add_module('lustre/mds', 'mds') - self.add_module('lustre/mds', 'mds_%s' % (self.fstype)) + if self.fstype == 'smfs': + realdev = self.fstype + + if mountfsoptions: + mountfsoptions = "%s,type=%s,dev=%s" % (mountfsoptions, + self.backfstype, + blkdev) + else: + mountfsoptions = "type=%s,dev=%s" % (self.backfstype, + blkdev) + else: + realdev = blkdev + + print 'MDS mount options: ' + mountfsoptions + + # As mount options are passed by 4th param to config tool, we need + # to pass something in 3rd param. But we do not want this 3rd param + # be counted as a profile name for reading log on MDS setup, thus, + # we pass there some predefined sign like 'dumb', which will be + # checked in MDS code and skipped. Probably there is more nice way + # like pass empty string and check it in config tool and pass null + # as 4th param. + lctl.newdev("mds", self.name, self.uuid, + setup ="%s %s %s %s" %(realdev, self.fstype, + 'dumb', mountfsoptions)) + do_cleanup = 1 + + # record logs for the MDS lov + for uuid in self.filesystem_uuids: + log("recording clients for filesystem:", uuid) + fs = self.db.lookup(uuid) + + # this is ugly, should be organized nice later. + target_uuid = self.db.get_first_ref('target') + mds = self.db.lookup(target_uuid) + lovconfig_uuid = mds.get_first_ref('lovconfig') + if lovconfig_uuid: + lovconfig = mds.lookup(lovconfig_uuid) + obd_uuid = lovconfig.get_first_ref('lov') + else: + obd_uuid = fs.get_first_ref('obd') + + client_uuid = generate_client_uuid(self.name) + client = VOSC(self.db.lookup(obd_uuid), client_uuid, self.name, + self.name) + config.record = 1 + lctl.clear_log(self.name, self.name) + lctl.record(self.name, self.name) + client.prepare() + lctl.mount_option(self.name, client.get_name(), "") + lctl.end_record() + process_updates(self.db, self.name, self.name, client) + + config.cleanup = 1 + lctl.clear_log(self.name, self.name + '-clean') + lctl.record(self.name, self.name + '-clean') + client.cleanup() + lctl.del_mount_option(self.name) + lctl.end_record() + process_updates(self.db, self.name, self.name + '-clean', client) + config.cleanup = 0 + config.record = 0 + + # record logs for each client + if config.noexec: + noexec_opt = '-n' + else: + noexec_opt = '' + if config.ldapurl: + config_options = "--ldapurl " + config.ldapurl + " --config " + config.config + else: + config_options = CONFIG_FILE + + for node_db in self.db.lookup_class('node'): + client_name = node_db.getName() + for prof_uuid in node_db.get_refs('profile'): + prof_db = node_db.lookup(prof_uuid) + # refactor this into a funtion to test "clientness" + # of a node. + for ref_class, ref_uuid in prof_db.get_all_refs(): + if ref_class in ('mountpoint','echoclient'): + debug("recording", client_name) + old_noexec = config.noexec + config.noexec = 0 + ret, out = run (sys.argv[0], noexec_opt, + " -v --record --nomod", + "--record_log", client_name, + "--record_device", self.name, + "--node", client_name, + config_options) + if config.verbose: + for s in out: log("record> ", string.strip(s)) + ret, out = run (sys.argv[0], noexec_opt, + "--cleanup -v --record --nomod", + "--record_log", client_name + "-clean", + "--record_device", self.name, + "--node", client_name, + config_options) + if config.verbose: + for s in out: log("record> ", string.strip(s)) + config.noexec = old_noexec + if do_cleanup: + try: + lctl.cleanup(self.name, self.uuid, 0, 0) + except CommandError, e: + log(self.module_name, "cleanup failed: ", self.name) + e.dump() + cleanup_error(e.rc) + Module.cleanup(self) + + if self.fstype == 'smfs': + clean_loop(self.backdevpath) + else: + clean_loop(self.devpath) + + def msd_remaining(self): + out = lctl.device_list() + for s in out: + if string.split(s)[2] in ('mds',): + return 1 + + def safe_to_clean(self): + return self.active + + def safe_to_clean_modules(self): + return not self.msd_remaining() + + def cleanup(self): + if not self.active: + debug(self.uuid, "not active") + return + self.info() + if is_prepared(self.name): + try: + lctl.cleanup(self.name, self.uuid, config.force, + config.failover) + except CommandError, e: + log(self.module_name, "cleanup failed: ", self.name) + e.dump() + cleanup_error(e.rc) + Module.cleanup(self) + # cleanup LMV + if self.master_mds: + self.master.cleanup() + if not self.msd_remaining() and is_prepared('MDT'): + try: + lctl.cleanup("MDT", "MDT_UUID", config.force, + config.failover) + except CommandError, e: + print "cleanup failed: ", self.name + e.dump() + cleanup_error(e.rc) + + if self.fstype == 'smfs': + clean_loop(self.backdevpath) + else: + clean_loop(self.devpath) + + def correct_level(self, level, op=None): + #if self.master_mds: + # level = level + 2 + return level + +class OSD(Module): + def __init__(self, db): + Module.__init__(self, 'OSD', db) + self.osdtype = self.db.get_val('osdtype') + self.devpath = self.db.get_val('devpath', '') + self.backdevpath = self.db.get_val('backdevpath', '') + self.size = self.db.get_val_int('devsize', 0) + self.journal_size = self.db.get_val_int('journalsize', 0) + self.inode_size = self.db.get_val_int('inodesize', 0) + self.mkfsoptions = self.db.get_val('mkfsoptions', '') + self.mountfsoptions = self.db.get_val('mountfsoptions', '') + self.fstype = self.db.get_val('fstype', '') + self.backfstype = self.db.get_val('backfstype', '') + self.nspath = self.db.get_val('nspath', '') + target_uuid = self.db.get_first_ref('target') + ost = self.db.lookup(target_uuid) + self.name = ost.getName() + self.format = self.db.get_val('autoformat', 'yes') + if ost.get_val('failover', 0): + self.failover_ost = 'f' + else: + self.failover_ost = 'n' + + active_uuid = get_active_target(ost) + if not active_uuid: + panic("No target device found:", target_uuid) + if active_uuid == self.uuid: + self.active = 1 + else: + self.active = 0 + if self.active and config.group and config.group != ost.get_val('group'): + self.active = 0 + + self.target_dev_uuid = self.uuid + self.uuid = target_uuid + # modules + self.add_lustre_module('ost', 'ost') + if self.fstype == 'smfs': + self.add_lustre_module('smfs', 'smfs') + # FIXME: should we default to ext3 here? + if self.fstype == 'ldiskfs': + self.add_lustre_module('ldiskfs', 'ldiskfs') + if self.fstype: + self.add_lustre_module('lvfs' , 'fsfilt_%s' % (self.fstype)) + if self.fstype == 'smfs': + self.add_lustre_module('lvfs' , 'fsfilt_%s' % (self.backfstype)) + + for options in self.mountfsoptions: + if options == 'snap': + if not self.fstype == 'smfs': + panic("mountoptions with snap, but fstype is not smfs\n") + self.add_lustre_module('lvfs', 'fsfilt_snap_%s' % (self.fstype)) + self.add_lustre_module('lvfs', 'fsfilt_snap_%s' % (self.backfstype)) + + self.add_lustre_module(self.osdtype, self.osdtype) + + def load_module(self): + if self.active: + Module.load_module(self) + + # need to check /proc/mounts and /etc/mtab before + # formatting anything. + # FIXME: check if device is already formatted. def prepare(self): - self.info(self.devname, self.fstype, self.format) - blkdev = block_dev(self.devname, self.size, self.fstype, self.format) - lctl.newdev(attach="mds %s %s" % (self.name, self.uuid), - setup ="%s %s" %(blkdev, self.fstype)) + if is_prepared(self.name): + return + if not self.active: + debug(self.uuid, "not active") + return + self.info(self.osdtype, self.devpath, self.size, self.fstype, + self.format, self.journal_size, self.inode_size) + run_acceptors() + if self.osdtype == 'obdecho': + blkdev = '' + else: + blkdev = block_dev(self.devpath, self.size, self.fstype, + config.reformat, self.format, self.journal_size, + self.inode_size, self.mkfsoptions, self.backfstype, + self.backdevpath) + + mountfsoptions = def_mount_options(self.fstype, 'ost') + + if config.mountfsoptions: + if mountfsoptions: + mountfsoptions = mountfsoptions + ',' + config.mountfsoptions + else: + mountfsoptions = config.mountfsoptions + if self.mountfsoptions: + mountfsoptions = mountfsoptions + ',' + self.mountfsoptions + else: + if self.mountfsoptions: + if mountfsoptions: + mountfsoptions = mountfsoptions + ',' + self.mountfsoptions + else: + mountfsoptions = self.mountfsoptions + + if self.fstype == 'smfs': + realdev = self.fstype + + if mountfsoptions: + mountfsoptions = "%s,type=%s,dev=%s" % (mountfsoptions, + self.backfstype, + blkdev) + else: + mountfsoptions = "type=%s,dev=%s" % (self.backfstype, + blkdev) + else: + realdev = blkdev + + print 'OSD mount options: ' + mountfsoptions + + lctl.newdev(self.osdtype, self.name, self.uuid, + setup ="%s %s %s %s" %(realdev, self.fstype, + self.failover_ost, + mountfsoptions)) + if not is_prepared('OSS'): + lctl.newdev("ost", 'OSS', 'OSS_UUID', setup ="") + + def osd_remaining(self): + out = lctl.device_list() + for s in out: + if string.split(s)[2] in ('obdfilter', 'obdecho'): + return 1 + + def safe_to_clean(self): + return self.active + + def safe_to_clean_modules(self): + return not self.osd_remaining() + def cleanup(self): - Module.cleanup(self) - clean_loop(self.devname) - -# Very unusual case, as there is no MDC element in the XML anymore -# Builds itself from an MDS node -class MDC(Module): - def __init__(self,dom_node): - self.mds = MDS(dom_node) - self.dom_node = dom_node - self.module_name = 'MDC' - self.kmodule_list = [] + if not self.active: + debug(self.uuid, "not active") + return + if is_prepared(self.name): + self.info() + try: + lctl.cleanup(self.name, self.uuid, config.force, + config.failover) + except CommandError, e: + log(self.module_name, "cleanup failed: ", self.name) + e.dump() + cleanup_error(e.rc) + if not self.osd_remaining() and is_prepared('OSS'): + try: + lctl.cleanup("OSS", "OSS_UUID", config.force, + config.failover) + except CommandError, e: + print "cleanup failed: ", self.name + e.dump() + cleanup_error(e.rc) + if not self.osdtype == 'obdecho': + if self.fstype == 'smfs': + clean_loop(self.backdevpath) + else: + clean_loop(self.devpath) + + def correct_level(self, level, op=None): + return level + +def mgmt_uuid_for_fs(mtpt_name): + if not mtpt_name: + return '' + mtpt_db = toplustreDB.lookup_name(mtpt_name) + fs_uuid = mtpt_db.get_first_ref('filesystem') + fs = toplustreDB.lookup(fs_uuid) + if not fs: + return '' + return fs.get_first_ref('mgmt') + +# Generic client module, used by OSC and MDC +class Client(Module): + def __init__(self, tgtdb, uuid, module, fs_name, self_name=None, + module_dir=None): + self.target_name = tgtdb.getName() + self.target_uuid = tgtdb.getUUID() + self.db = tgtdb + self.active = 1 + + self.tgt_dev_uuid = get_active_target(tgtdb) + if not self.tgt_dev_uuid: + panic("No target device found for target(1):", self.target_name) + + self.kmod = kmod(config.lustre, config.portals) self._server = None self._connected = 0 - host = socket.gethostname() - self.name = 'MDC_'+host - self.uuid = self.name+'_UUID' + self.module = module + self.module_name = string.upper(module) + if not self_name: + self.name = '%s_%s_%s_%s' % (self.module_name, socket.gethostname(), + self.target_name, fs_name) + else: + self.name = self_name + self.uuid = uuid + self.lookup_server(self.tgt_dev_uuid) + mgmt_uuid = mgmt_uuid_for_fs(fs_name) + if mgmt_uuid: + self.mgmt_name = mgmtcli_name_for_uuid(mgmt_uuid) + else: + self.mgmt_name = '' + self.fs_name = fs_name + if not module_dir: + module_dir = module + self.add_lustre_module(module_dir, module) + + def lookup_server(self, srv_uuid): + """ Lookup a server's network information """ + self._server_nets = get_ost_net(self.db, srv_uuid) + if len(self._server_nets) == 0: + panic ("Unable to find a server for:", srv_uuid) + def get_name(self): + return self.name + def get_servers(self): + return self._server_nets + + def prepare(self, ignore_connect_failure = 0): + self.info(self.target_uuid) + if not config.record and is_prepared(self.name): + self.cleanup() + try: + srv = choose_local_server(self.get_servers()) + if srv: + lctl.connect(srv) + else: + routes = find_route(self.get_servers()) + if len(routes) == 0: + panic ("no route to", self.target_uuid) + for (srv, r) in routes: + lctl.add_route_host(r[0], srv.nid_uuid, r[1], r[3]) + except CommandError, e: + if not ignore_connect_failure: + raise e + if srv: + if self.permits_inactive() and (self.target_uuid in config.inactive or self.active == 0): + debug("%s inactive" % self.target_uuid) + inactive_p = "inactive" + else: + debug("%s active" % self.target_uuid) + inactive_p = "" + lctl.newdev(self.module, self.name, self.uuid, + setup ="%s %s %s %s" % (self.target_uuid, srv.nid_uuid, + inactive_p, self.mgmt_name)) + + def cleanup(self): + if is_prepared(self.name): + Module.cleanup(self) + try: + srv = choose_local_server(self.get_servers()) + if srv: + lctl.disconnect(srv) + else: + for (srv, r) in find_route(self.get_servers()): + lctl.del_route_host(r[0], srv.nid_uuid, r[1], r[3]) + except CommandError, e: + log(self.module_name, "cleanup failed: ", self.name) + e.dump() + cleanup_error(e.rc) - self.lookup_server(self.mds.uuid) - self.add_module('lustre/mdc', 'mdc') + def correct_level(self, level, op=None): + return level + def deactivate(self): + try: + lctl.deactivate(self.name) + except CommandError, e: + log(self.module_name, "deactivate failed: ", self.name) + e.dump() + cleanup_error(e.rc) + +class MDC(Client): + def __init__(self, db, uuid, fs_name): + Client.__init__(self, db, uuid, 'mdc', fs_name) + + def permits_inactive(self): + return 0 + +class OSC(Client): + def __init__(self, db, uuid, fs_name): + Client.__init__(self, db, uuid, 'osc', fs_name) + + def permits_inactive(self): + return 1 + +def mgmtcli_name_for_uuid(uuid): + return 'MGMTCLI_%s' % uuid + +class ManagementClient(Client): + def __init__(self, db, uuid): + Client.__init__(self, db, uuid, 'mgmt_cli', '', + self_name = mgmtcli_name_for_uuid(db.getUUID()), + module_dir = 'mgmt') +class VLOV(Module): + def __init__(self, db, uuid, fs_name, name_override = None, config_only = None): + Module.__init__(self, 'VLOV', db) + if name_override != None: + self.name = "lov_%s" % name_override + self.add_lustre_module('lov', 'lov') + self.stripe_sz = 65536 + self.stripe_off = 0 + self.pattern = 0 + self.stripe_cnt = 1 + self.desc_uuid = self.uuid + self.uuid = generate_client_uuid(self.name) + self.fs_name = fs_name + self.osc = get_osc(db, self.uuid, fs_name) + if not self.osc: + panic('osc not found:', self.uuid) + if config_only: + self.config_only = 1 + return + self.config_only = None + def get_uuid(self): + return self.uuid + def get_name(self): + return self.name def prepare(self): - self.info(self.mds.uuid) - srv = self.get_server() - lctl.connect(srv.net_type, srv.nid, srv.port, srv.uuid, srv.send_mem, srv.recv_mem) - lctl.newdev(attach="mdc %s %s" % (self.name, self.uuid), - setup ="%s %s" %(self.mds.uuid, srv.uuid)) - -class OBD(Module): - def __init__(self, dom_node): - Module.__init__(self, 'OBD', dom_node) - self.obdtype = get_attr(dom_node, 'type') - self.devname, self.size = get_device(dom_node) - self.fstype = get_text(dom_node, 'fstype') - self.format = get_text(dom_node, 'autoformat', 'yes') - if self.fstype == 'extN': - self.add_module('lustre/extN', 'extN') - self.add_module('lustre/' + self.obdtype, self.obdtype) + if not config.record and is_prepared(self.name): + return + lctl.lov_setup(self.name, self.uuid, self.desc_uuid, self.stripe_cnt, + self.stripe_sz, self.stripe_off, self.pattern) + target_uuid = self.osc.target_uuid + try: + self.osc.active = 1 + self.osc.prepare(ignore_connect_failure=0) + except CommandError, e: + print "Error preparing OSC %s\n" % osc.uuid + raise e + lctl.lov_add_obd(self.name, self.uuid, target_uuid, 0, 1) + + def cleanup(self): + target_uuid = self.osc.target_uuid + self.osc.cleanup() + if is_prepared(self.name): + Module.cleanup(self) + if self.config_only: + panic("Can't clean up config_only LOV ", self.name) + + def load_module(self): + if self.config_only: + panic("Can't load modules for config_only LOV ", self.name) + self.osc.load_module() + Module.load_module(self) + + def cleanup_module(self): + if self.config_only: + panic("Can't cleanup modules for config_only LOV ", self.name) + Module.cleanup_module(self) + self.osc.cleanup_module() + def correct_level(self, level, op=None): + return level + +class CMOBD(Module): + def __init__(self,db): + Module.__init__(self, 'CMOBD', db) + self.name = self.db.getName(); + self.uuid = generate_client_uuid(self.name) + self.master_uuid = self.db.get_first_ref('masterobd') + self.cache_uuid = self.db.get_first_ref('cacheobd') + self.add_lustre_module('cmobd', 'cmobd') + master_obd = self.db.lookup(self.master_uuid) + if not master_obd: + panic('master obd not found:', self.master_uuid) + cache_obd = self.db.lookup(self.cache_uuid) + if not cache_obd: + panic('cache obd not found:', self.cache_uuid) + + if master_obd.get_class() == 'ost': + self.client_uuid = generate_client_uuid(self.name) + self.master= VLOV(master_obd, self.client_uuid, self.name, + "%s_master" % (self.name)) + self.master_uuid = self.master.get_uuid() + else: + self.master = get_mdc(db, self.name, self.master_uuid) # need to check /proc/mounts and /etc/mtab before # formatting anything. # FIXME: check if device is already formatted. def prepare(self): - self.info(self.obdtype, self.devname, self.size, self.fstype, self.format) - if self.obdtype == 'obdecho': - blkdev = '' - else: - blkdev = block_dev(self.devname, self.size, self.fstype, self.format) - lctl.newdev(attach="%s %s %s" % (self.obdtype, self.name, self.uuid), - setup ="%s %s" %(blkdev, self.fstype)) + self.master.prepare() + if not config.record and is_prepared(self.name): + return + self.info(self.master_uuid, self.cache_uuid) + lctl.newdev("cmobd", self.name, self.uuid, + setup ="%s %s" %(self.master_uuid, + self.cache_uuid)) + def cleanup(self): - Module.cleanup(self) - if not self.obdtype == 'obdecho': - clean_loop(self.devname) + if is_prepared(self.name): + Module.cleanup(self) + self.master.cleanup() -class OST(Module): - def __init__(self,dom_node): - Module.__init__(self, 'OST', dom_node) - self.obd_uuid = get_first_ref(dom_node, 'obd') - self.add_module('lustre/ost', 'ost') + def load_module(self): + self.master.load_module() + Module.load_module(self) + def cleanup_module(self): + Module.cleanup_module(self) + self.master.cleanup_module() + + def correct_level(self, level, op=None): + return level + +class COBD(Module): + def __init__(self, db, uuid, name, type, name_override = None): + Module.__init__(self, 'COBD', db) + self.name = self.db.getName(); + self.uuid = generate_client_uuid(self.name) + self.real_uuid = self.db.get_first_ref('realobd') + self.cache_uuid = self.db.get_first_ref('cacheobd') + self.add_lustre_module('cobd', 'cobd') + real_obd = self.db.lookup(self.real_uuid) + if not real_obd: + panic('real obd not found:', self.real_uuid) + cache_obd = self.db.lookup(self.cache_uuid) + if not cache_obd: + panic('cache obd not found:', self.cache_uuid) + if type == 'obd': + self.real = LOV(real_obd, self.real_uuid, name, + "%s_real" % (self.name)); + self.cache = LOV(cache_obd, self.cache_uuid, name, + "%s_cache" % (self.name)); + else: + self.real = get_mdc(db, name, self.real_uuid) + self.cache = get_mdc(db, name, self.cache_uuid) + # need to check /proc/mounts and /etc/mtab before + # formatting anything. + # FIXME: check if device is already formatted. + def get_uuid(self): + return self.uuid + def get_name(self): + return self.name + def get_real_name(self): + return self.real.name + def get_cache_name(self): + return self.cache.name def prepare(self): - self.info(self.obd_uuid) - lctl.newdev(attach="ost %s %s" % (self.name, self.uuid), - setup ="%s" % (self.obd_uuid)) + self.real.prepare() + self.cache.prepare() + if not config.record and is_prepared(self.name): + return + self.info(self.real_uuid, self.cache_uuid) + lctl.newdev("cobd", self.name, self.uuid, + setup ="%s %s" %(self.real.name, + self.cache.name)) + def cleanup(self): + if is_prepared(self.name): + Module.cleanup(self) + self.real.cleanup() + self.cache.cleanup() + + def load_module(self): + self.real.load_module() + Module.load_module(self) + + def cleanup_module(self): + Module.cleanup_module(self) + self.real.cleanup_module() # virtual interface for OSC and LOV class VOSC(Module): - def __init__(self,dom_node): - Module.__init__(self, 'VOSC', dom_node) - if dom_node.nodeName == 'lov': - self.osc = LOV(dom_node) + def __init__(self, db, client_uuid, name, name_override = None): + Module.__init__(self, 'VOSC', db) + if db.get_class() == 'lov': + self.osc = LOV(db, client_uuid, name, name_override) + self.type = 'lov' + elif db.get_class() == 'cobd': + self.osc = COBD(db, client_uuid, name, 'obd') + self.type = 'cobd' else: - self.osc = OSC(dom_node) + self.osc = OSC(db, client_uuid, name) + self.type = 'osc' + def get_uuid(self): + return self.osc.get_uuid() + def get_name(self): + return self.osc.get_name() def prepare(self): self.osc.prepare() def cleanup(self): @@ -962,607 +2358,1105 @@ class VOSC(Module): self.osc.load_module() def cleanup_module(self): self.osc.cleanup_module() - - -class OSC(Module): - def __init__(self,dom_node): - Module.__init__(self, 'OSC', dom_node) - self.obd_uuid = get_first_ref(dom_node, 'obd') - self.ost_uuid = get_first_ref(dom_node, 'ost') - self.lookup_server(self.ost_uuid) - self.add_module('lustre/osc', 'osc') - - def prepare(self): - self.info(self.obd_uuid, self.ost_uuid) - srv = self.get_server() - if local_net(srv): - lctl.connect(srv.net_type, srv.nid, srv.port, srv.uuid, srv.send_mem, srv.recv_mem) + def correct_level(self, level, op=None): + return self.osc.correct_level(level, op) + +# virtual interface for MDC and LMV +class VMDC(Module): + def __init__(self, db, client_uuid, name, name_override = None): + Module.__init__(self, 'VMDC', db) + if db.get_class() == 'lmv': + self.mdc = LMV(db, client_uuid, name) + elif db.get_class() == 'cobd': + self.mdc = COBD(db, client_uuid, name, 'mds') else: - r = find_route(srv) - if r: - lctl.add_route_host(r[0], srv.uuid, r[1], r[2]) - else: - panic ("no route to", srv.nid) - - lctl.newdev(attach="osc %s %s" % (self.name, self.uuid), - setup ="%s %s" %(self.obd_uuid, srv.uuid)) - + self.mdc = MDC(db, client_uuid, name) + def get_uuid(self): + return self.mdc.uuid + def get_name(self): + return self.mdc.name + def prepare(self): + self.mdc.prepare() def cleanup(self): - srv = self.get_server() - if local_net(srv): - Module.cleanup(self) - else: - self.info(self.obd_uuid, self.ost_uuid) - r = find_route(srv) - if r: - lctl.del_route_host(r[0], srv.uuid, r[1], r[2]) - Module.cleanup(self) - - -class Mountpoint(Module): - def __init__(self,dom_node): - Module.__init__(self, 'MTPT', dom_node) - self.path = get_text(dom_node, 'path') - self.mds_uuid = get_first_ref(dom_node, 'mds') - self.lov_uuid = get_first_ref(dom_node, 'osc') - self.add_module('lustre/mdc', 'mdc') - self.add_module('lustre/llite', 'llite') - l = lookup(self.dom_node.parentNode, self.lov_uuid) - self.osc = VOSC(l) + self.mdc.cleanup() + def load_module(self): + self.mdc.load_module() + def cleanup_module(self): + self.mdc.cleanup_module() + def correct_level(self, level, op=None): + return self.mdc.correct_level(level, op) + +class ECHO_CLIENT(Module): + def __init__(self,db): + Module.__init__(self, 'ECHO_CLIENT', db) + self.add_lustre_module('obdecho', 'obdecho') + self.obd_uuid = self.db.get_first_ref('obd') + obd = self.db.lookup(self.obd_uuid) + self.uuid = generate_client_uuid(self.name) + self.osc = VOSC(obd, self.uuid, self.name) def prepare(self): - self.osc.prepare() - mdc_uuid = prepare_mdc(self.dom_node.parentNode, self.mds_uuid) + if not config.record and is_prepared(self.name): + return + run_acceptors() + self.osc.prepare() # XXX This is so cheating. -p + self.info(self.obd_uuid) - self.info(self.path, self.mds_uuid,self.lov_uuid) - cmd = "mount -t lustre_lite -o osc=%s,mdc=%s none %s" % \ - (self.lov_uuid, mdc_uuid, self.path) - run("mkdir", self.path) - ret, val = run(cmd) - if ret: - panic("mount failed:", self.path) + lctl.newdev("echo_client", self.name, self.uuid, + setup = self.osc.get_name()) def cleanup(self): - self.info(self.path, self.mds_uuid,self.lov_uuid) - (rc, out) = run("umount", self.path) - if rc: - log("umount failed, cleanup will most likely not work.") - l = lookup(self.dom_node.parentNode, self.lov_uuid) + if is_prepared(self.name): + Module.cleanup(self) self.osc.cleanup() - cleanup_mdc(self.dom_node.parentNode, self.mds_uuid) def load_module(self): self.osc.load_module() Module.load_module(self) + def cleanup_module(self): Module.cleanup_module(self) self.osc.cleanup_module() + def correct_level(self, level, op=None): + return level -# ============================================================ -# XML processing and query -# TODO: Change query funcs to use XPath, which is muc cleaner - -def get_device(obd): - list = obd.getElementsByTagName('device') - if len(list) > 0: - dev = list[0] - dev.normalize(); - size = get_attr_int(dev, 'size', 0) - return dev.firstChild.data, size - return '', 0 - -# Get the text content from the first matching child -# If there is no content (or it is all whitespace), return -# the default -def get_text(dom_node, tag, default=""): - list = dom_node.getElementsByTagName(tag) - if len(list) > 0: - dom_node = list[0] - dom_node.normalize() - if dom_node.firstChild: - txt = string.strip(dom_node.firstChild.data) - if txt: - return txt - return default - -def get_text_int(dom_node, tag, default=0): - list = dom_node.getElementsByTagName(tag) - n = default - if len(list) > 0: - dom_node = list[0] - dom_node.normalize() - if dom_node.firstChild: - txt = string.strip(dom_node.firstChild.data) - if txt: - try: - n = int(txt) - except ValueError: - panic("text value is not integer:", txt) - return n +def generate_client_uuid(name): + client_uuid = '%05x_%.19s_%05x%05x' % (int(random.random() * 1048576), + name, + int(random.random() * 1048576), + int(random.random() * 1048576)) + return client_uuid[:36] -def get_attr(dom_node, attr, default=""): - v = dom_node.getAttribute(attr) - if v: - return v - return default +class Mountpoint(Module): + def __init__(self,db): + Module.__init__(self, 'MTPT', db) + self.path = self.db.get_val('path') + self.clientoptions = self.db.get_val('clientoptions', '') + self.fs_uuid = self.db.get_first_ref('filesystem') + fs = self.db.lookup(self.fs_uuid) + self.mds_uuid = fs.get_first_ref('lmv') + if not self.mds_uuid: + self.mds_uuid = fs.get_first_ref('mds') + self.obd_uuid = fs.get_first_ref('obd') + self.mgmt_uuid = fs.get_first_ref('mgmt') + client_uuid = generate_client_uuid(self.name) + + ost = self.db.lookup(self.obd_uuid) + if not ost: + panic("no ost: ", self.obd_uuid) + + mds = self.db.lookup(self.mds_uuid) + if not mds: + panic("no mds: ", self.mds_uuid) + + self.add_lustre_module('mdc', 'mdc') + self.add_lustre_module('lmv', 'lmv') + self.add_lustre_module('llite', 'llite') + + self.vosc = VOSC(ost, client_uuid, self.name) + self.vmdc = VMDC(mds, client_uuid, self.name) + + if self.mgmt_uuid: + self.mgmtcli = ManagementClient(db.lookup(self.mgmt_uuid), + client_uuid) + else: + self.mgmtcli = None -def get_attr_int(dom_node, attr, default=0): - n = default - v = dom_node.getAttribute(attr) - if v: - try: - n = int(v) - except ValueError: - panic("attr value is not integer", v) - return n + def prepare(self): + if not config.record and fs_is_mounted(self.path): + log(self.path, "already mounted.") + return + run_acceptors() + if self.mgmtcli: + self.mgmtcli.prepare() + self.vosc.prepare() + self.vmdc.prepare() + vmdc_name = self.vmdc.get_name() + + self.info(self.path, self.mds_uuid, self.obd_uuid) + if config.record or config.lctl_dump: + lctl.mount_option(local_node_name, self.vosc.get_name(), vmdc_name) + return -def get_first_ref(dom_node, tag): - """ Get the first uuidref of the type TAG. Used one only - one is expected. Returns the uuid.""" - uuid = None - refname = '%s_ref' % tag - list = dom_node.getElementsByTagName(refname) - if len(list) > 0: - uuid = getRef(list[0]) - return uuid - -def get_all_refs(dom_node, tag): - """ Get all the refs of type TAG. Returns list of uuids. """ - uuids = [] - refname = '%s_ref' % tag - list = dom_node.getElementsByTagName(refname) - if len(list) > 0: - for i in list: - uuids.append(getRef(i)) - return uuids - -def get_ost_net(dom_node, uuid): - ost = lookup(dom_node, uuid) - uuid = get_first_ref(ost, 'network') - if not uuid: - return None - return lookup(dom_node, uuid) - -def nid2server(dom_node, nid): - netlist = dom_node.getElementsByTagName('network') - for net_node in netlist: - if get_text(net_node, 'server') == nid: - return Network(net_node) - return None - -def lookup(dom_node, uuid): - for n in dom_node.childNodes: - if n.nodeType == n.ELEMENT_NODE: - if getUUID(n) == uuid: - return n + if config.clientoptions: + if self.clientoptions: + self.clientoptions = self.clientoptions + ',' + \ + config.clientoptions else: - n = lookup(n, uuid) - if n: return n - return None - -# Get name attribute of dom_node -def getName(dom_node): - return dom_node.getAttribute('name') + self.clientoptions = config.clientoptions + if self.clientoptions: + self.clientoptions = ',' + self.clientoptions + # Linux kernel will deal with async and not pass it to ll_fill_super, + # so replace it with Lustre async + self.clientoptions = string.replace(self.clientoptions, "async", + "lasync") + + cmd = "mount -t lustre_lite -o osc=%s,mdc=%s%s %s %s" % \ + (self.vosc.get_name(), vmdc_name, self.clientoptions, + config.config, self.path) + run("mkdir", self.path) + ret, val = run(cmd) + if ret: + self.vmdc.cleanup() + self.vosc.cleanup() + panic("mount failed:", self.path, ":", string.join(val)) -def getRef(dom_node): - return dom_node.getAttribute('uuidref') + def cleanup(self): + self.info(self.path, self.mds_uuid,self.obd_uuid) -# Get name attribute of dom_node -def getUUID(dom_node): - return dom_node.getAttribute('uuid') + if config.record or config.lctl_dump: + lctl.del_mount_option(local_node_name) + else: + if fs_is_mounted(self.path): + if config.force: + (rc, out) = run("umount", "-f", self.path) + else: + (rc, out) = run("umount", self.path) + if rc: + raise CommandError('umount', out, rc) -# the tag name is the service type -# fixme: this should do some checks to make sure the dom_node is a service -def getServiceType(dom_node): - return dom_node.nodeName + if fs_is_mounted(self.path): + panic("fs is still mounted:", self.path) -# -# determine what "level" a particular node is at. -# the order of iniitailization is based on level. -def getServiceLevel(dom_node): - type = getServiceType(dom_node) + self.vmdc.cleanup() + self.vosc.cleanup() + if self.mgmtcli: + self.mgmtcli.cleanup() + + def load_module(self): + if self.mgmtcli: + self.mgmtcli.load_module() + self.vosc.load_module() + Module.load_module(self) + + def cleanup_module(self): + Module.cleanup_module(self) + self.vosc.cleanup_module() + if self.mgmtcli: + self.mgmtcli.cleanup_module() + + def correct_level(self, level, op=None): + return level + +# ============================================================ +# misc query functions + +def get_ost_net(self, osd_uuid): + srv_list = [] + if not osd_uuid: + return srv_list + osd = self.lookup(osd_uuid) + node_uuid = osd.get_first_ref('node') + node = self.lookup(node_uuid) + if not node: + panic("unable to find node for osd_uuid:", osd_uuid, + " node_ref:", node_uuid_) + for net_uuid in node.get_networks(): + db = node.lookup(net_uuid) + srv_list.append(Network(db)) + return srv_list + + +# the order of iniitailization is based on level. +def getServiceLevel(self): + type = self.get_class() + ret=0; if type in ('network',): - return 10 - elif type in ('device', 'ldlm'): - return 20 - elif type in ('obd', 'mdd'): - return 30 - elif type in ('mds','ost'): - return 40 - elif type in ('mdc','osc'): - return 50 - elif type in ('lov', 'lovconfig'): - return 60 - elif type in ('mountpoint',): - return 70 - return 0 + ret = 5 + elif type in ('routetbl',): + ret = 6 + elif type in ('ldlm',): + ret = 20 + elif type in ('mgmt',): + ret = 25 + elif type in ('osd', 'cobd'): + ret = 30 + elif type in ('mdsdev',): + ret = 40 + elif type in ('lmv',): + ret = 45 + elif type in ('cmobd',): + ret = 50 + elif type in ('mountpoint', 'echoclient'): + ret = 70 + else: + panic("Unknown type: ", type) + + if ret < config.minlevel or ret > config.maxlevel: + ret = 0 + return ret # # return list of services in a profile. list is a list of tuples -# [(level, dom_node),] -def getServices(lustreNode, profileNode): +# [(level, db_object),] +def getServices(self): list = [] - for n in profileNode.childNodes: - if n.nodeType == n.ELEMENT_NODE: - servNode = lookup(lustreNode, getRef(n)) - if not servNode: - print n - panic('service not found: ' + getRef(n)) - level = getServiceLevel(servNode) - list.append((level, servNode)) + for ref_class, ref_uuid in self.get_all_refs(): + servdb = self.lookup(ref_uuid) + if servdb: + level = getServiceLevel(servdb) + if level > 0: + list.append((level, servdb)) + else: + panic('service not found: ' + ref_uuid) + list.sort() return list -def getByName(lustreNode, name, tag): - ndList = lustreNode.getElementsByTagName(tag) - for nd in ndList: - if getName(nd) == name: - return nd - return None - ############################################################ -# MDC UUID hack - +# MDC UUID hack - # FIXME: clean this mess up! # -mdc_uuid = None -def prepare_mdc(dom_node, mds_uuid): - global mdc_uuid - mds_node = lookup(dom_node, mds_uuid); - if not mds_node: - panic("no mds:", mds_uuid) - if mdc_uuid: - return mdc_uuid - mdc = MDC(mds_node) - mdc.prepare() - mdc_uuid = mdc.uuid - return mdc_uuid - -mdc_cleaned = None -def cleanup_mdc(dom_node, mds_uuid): - global mdc_cleaned - mds_node = lookup(dom_node, mds_uuid); - if not mds_node: - panic("no mds:", mds_uuid) - if not mdc_cleaned: - mdc = MDC(mds_node) - mdc.cleanup() - mdc_uuid = None - mdc_cleaned = 'yes' - +# OSC is no longer in the xml, so we have to fake it. +# this is getting ugly and begging for another refactoring +def get_osc(ost_db, uuid, fs_name): + osc = OSC(ost_db, uuid, fs_name) + return osc + +def get_mdc(db, fs_name, mds_uuid): + mds_db = db.lookup(mds_uuid); + if not mds_db: + error("no mds:", mds_uuid) + mdc = MDC(mds_db, mds_uuid, fs_name) + return mdc ############################################################ # routing ("rooting") -# -routes = [] -local_node = [] -router_flag = 0 - -def init_node(dom_node): - global local_node, router_flag - netlist = dom_node.getElementsByTagName('network') - for dom_net in netlist: - type = get_attr(dom_net, 'type') - gw = get_text(dom_net, 'server') - local_node.append((type, gw)) - +# list of (nettype, cluster_id, nid) +local_clusters = [] + +def find_local_clusters(node_db): + global local_clusters + for netuuid in node_db.get_networks(): + net = node_db.lookup(netuuid) + srv = Network(net) + debug("add_local", netuuid) + local_clusters.append((srv.net_type, srv.cluster_id, srv.nid)) + if srv.port > 0: + if acceptors.has_key(srv.port): + panic("duplicate port:", srv.port) + acceptors[srv.port] = AcceptorHandler(srv.port, srv.net_type, + srv.send_mem, srv.recv_mem, + srv.irq_affinity) + +# This node is a gateway. +is_router = 0 +def node_is_router(): + return is_router + +# If there are any routers found in the config, then this will be true +# and all nodes will load kptlrouter. +needs_router = 0 def node_needs_router(): - return router_flag - -def get_routes(type, gw, dom_net): - """ Return the routes as a list of tuples of the form: - [(type, gw, lo, hi),]""" - res = [] - tbl = dom_net.getElementsByTagName('route_tbl') - for t in tbl: - routes = t.getElementsByTagName('route') - for r in routes: - lo = get_attr(r, 'lo') - hi = get_attr(r, 'hi', '') - res.append((type, gw, lo, hi)) - return res - + return needs_router or is_router + +# list of (nettype, gw, tgt_cluster_id, lo, hi) +# Currently, these local routes are only added to kptlrouter route +# table if they are needed to connect to a specific server. This +# should be changed so all available routes are loaded, and the +# ptlrouter can make all the decisions. +local_routes = [] -def init_route_config(lustre): - """ Scan the lustre config looking for routers. Build list of +def find_local_routes(lustre): + """ Scan the lustre config looking for routers . Build list of routes. """ - global routes, router_flag - routes = [] - list = lustre.getElementsByTagName('node') - for node in list: - if get_attr(node, 'router'): - router_flag = 1 - for (local_type, local_nid) in local_node: + global local_routes, needs_router + local_routes = [] + list = lustre.lookup_class('node') + for router in list: + if router.get_val_int('router', 0): + needs_router = 1 + for (local_type, local_cluster_id, local_nid) in local_clusters: gw = None - netlist = node.getElementsByTagName('network') - for dom_net in netlist: - if local_type == get_attr(dom_net, 'type'): - gw = get_text(dom_net, 'server') + for netuuid in router.get_networks(): + db = router.lookup(netuuid) + if (local_type == db.get_val('nettype') and + local_cluster_id == db.get_val('clusterid')): + gw = db.get_val('nid') break - if not gw: - continue - for dom_net in netlist: - if local_type != get_attr(dom_net, 'type'): - for route in get_routes(local_type, gw, dom_net): - routes.append(route) - + if gw: + debug("find_local_routes: gw is", gw) + for route in router.get_local_routes(local_type, gw): + local_routes.append(route) + debug("find_local_routes:", local_routes) + -def local_net(net): - global local_node - for iface in local_node: - if net.net_type == iface[0]: +def choose_local_server(srv_list): + for srv in srv_list: + if local_cluster(srv.net_type, srv.cluster_id): + return srv + +def local_cluster(net_type, cluster_id): + for cluster in local_clusters: + if net_type == cluster[0] and cluster_id == cluster[1]: return 1 return 0 -def find_route(net): - global local_node, routes - frm_type = local_node[0][0] - to_type = net.net_type - to = net.nid - debug ('looking for route to', to_type,to) - for r in routes: - if r[2] == to: - return r - return None - - - +def local_interface(net_type, cluster_id, nid): + for cluster in local_clusters: + if (net_type == cluster[0] and cluster_id == cluster[1] + and nid == cluster[2]): + return 1 + return 0 + +def find_route(srv_list): + result = [] + frm_type = local_clusters[0][0] + for srv in srv_list: + debug("find_route: srv:", srv.nid, "type: ", srv.net_type) + to_type = srv.net_type + to = srv.nid + cluster_id = srv.cluster_id + debug ('looking for route to', to_type, to) + for r in local_routes: + debug("find_route: ", r) + if (r[3] <= to and to <= r[4]) and cluster_id == r[2]: + result.append((srv, r)) + return result + +def get_active_target(db): + target_uuid = db.getUUID() + target_name = db.getName() + node_name = get_select(target_name) + if node_name: + tgt_dev_uuid = db.get_node_tgt_dev(node_name, target_uuid) + else: + tgt_dev_uuid = db.get_first_ref('active') + return tgt_dev_uuid + +def get_server_by_nid_uuid(db, nid_uuid): + for n in db.lookup_class("network"): + net = Network(n) + if net.nid_uuid == nid_uuid: + return net + ############################################################ # lconf level logic # Start a service. -def startService(dom_node, module_flag): - type = getServiceType(dom_node) - debug('Service:', type, getName(dom_node), getUUID(dom_node)) - # there must be a more dynamic way of doing this... +def newService(db): + type = db.get_class() + debug('Service:', type, db.getName(), db.getUUID()) n = None if type == 'ldlm': - n = LDLM(dom_node) + n = LDLM(db) elif type == 'lov': - n = LOV(dom_node) - elif type == 'lovconfig': - n = LOVConfig(dom_node) + n = LOV(db, "YOU_SHOULD_NEVER_SEE_THIS_UUID") elif type == 'network': - n = Network(dom_node) - elif type == 'obd': - n = OBD(dom_node) - elif type == 'ost': - n = OST(dom_node) - elif type == 'mds': - n = MDS(dom_node) - elif type == 'osc': - n = VOSC(dom_node) - elif type == 'mdc': - n = MDC(dom_node) + n = Network(db) + elif type == 'routetbl': + n = RouteTable(db) + elif type == 'osd': + n = OSD(db) + elif type == 'cobd': + n = COBD(db, "YOU_SHOULD_NEVER_SEE_THIS_UUID") + elif type == 'cmobd': + n = CMOBD(db) + elif type == 'mdsdev': + n = MDSDEV(db) elif type == 'mountpoint': - n = Mountpoint(dom_node) + n = Mountpoint(db) + elif type == 'echoclient': + n = ECHO_CLIENT(db) + elif type == 'mgmt': + n = Management(db) + elif type == 'lmv': + n = LMV(db) else: panic ("unknown service type:", type) - - if module_flag: - if config.nomod(): - return - if config.cleanup(): - n.cleanup_module() - else: - n.load_module() - else: - if config.nosetup(): - return - if config.cleanup(): - n.cleanup() - else: - n.prepare() + return n # # Prepare the system to run lustre using a particular profile -# in a the configuration. +# in a the configuration. # * load & the modules # * setup networking for the current node # * make sure partitions are in place and prepared # * initialize devices with lctl # Levels is important, and needs to be enforced. -def startProfile(lustreNode, profileNode, module_flag): - if not profileNode: - panic("profile:", profile, "not found.") - services = getServices(lustreNode, profileNode) - if config.cleanup(): - services.reverse() +def for_each_profile(db, prof_list, operation): + for prof_uuid in prof_list: + prof_db = db.lookup(prof_uuid) + if not prof_db: + panic("profile:", profile, "not found.") + services = getServices(prof_db) + operation(services) + +def magic_get_osc(db, rec, lov): + if lov: + lov_uuid = lov.get_uuid() + lov_name = lov.osc.fs_name + else: + lov_uuid = rec.getAttribute('lov_uuidref') + # FIXME: better way to find the mountpoint? + filesystems = db.root_node.getElementsByTagName('filesystem') + fsuuid = None + for fs in filesystems: + ref = fs.getElementsByTagName('obd_ref') + if ref[0].getAttribute('uuidref') == lov_uuid: + fsuuid = fs.getAttribute('uuid') + break + + if not fsuuid: + panic("malformed xml: lov uuid '" + lov_uuid + "' referenced in 'add' record is not used by any filesystems.") + + mtpts = db.root_node.getElementsByTagName('mountpoint') + lov_name = None + for fs in mtpts: + ref = fs.getElementsByTagName('filesystem_ref') + if ref[0].getAttribute('uuidref') == fsuuid: + lov_name = fs.getAttribute('name') + break + + if not lov_name: + panic("malformed xml: 'add' record references lov uuid '" + lov_uuid + "', which references filesystem uuid '" + fsuuid + "', which does not reference a mountpoint.") + + print "lov_uuid: " + lov_uuid + "; lov_name: " + lov_name + + ost_uuid = rec.getAttribute('ost_uuidref') + obd = db.lookup(ost_uuid) + + if not obd: + panic("malformed xml: 'add' record references ost uuid '" + ost_uuid + "' which cannot be found.") + + osc = get_osc(obd, lov_uuid, lov_name) + if not osc: + panic('osc not found:', obd_uuid) + return osc + +# write logs for update records. sadly, logs of all types -- and updates in +# particular -- are something of an afterthought. lconf needs rewritten with +# these as core concepts. so this is a pretty big hack. +def process_update_record(db, update, lov): + for rec in update.childNodes: + if rec.nodeType != rec.ELEMENT_NODE: + continue + + log("found "+rec.nodeName+" record in update version " + + str(update.getAttribute('version'))) + + lov_uuid = rec.getAttribute('lov_uuidref') + ost_uuid = rec.getAttribute('ost_uuidref') + index = rec.getAttribute('index') + gen = rec.getAttribute('generation') + + if not lov_uuid or not ost_uuid or not index or not gen: + panic("malformed xml: 'update' record requires lov_uuid, ost_uuid, index, and generation.") + + if not lov: + tmplov = db.lookup(lov_uuid) + if not tmplov: + panic("malformed xml: 'delete' record contains lov UUID '" + lov_uuid + "', which cannot be located.") + lov_name = tmplov.getName() + else: + lov_name = lov.osc.name + + # ------------------------------------------------------------- add + if rec.nodeName == 'add': + if config.cleanup: + lctl.lov_del_obd(lov_name, lov_uuid, ost_uuid, index, gen) + continue + + osc = magic_get_osc(db, rec, lov) + + try: + # Only ignore connect failures with --force, which + # isn't implemented here yet. + osc.prepare(ignore_connect_failure=0) + except CommandError, e: + print "Error preparing OSC %s\n" % osc.uuid + raise e + + lctl.lov_add_obd(lov_name, lov_uuid, ost_uuid, index, gen) + + # ------------------------------------------------------ deactivate + elif rec.nodeName == 'deactivate': + if config.cleanup: + continue + + osc = magic_get_osc(db, rec, lov) + + try: + osc.deactivate() + except CommandError, e: + print "Error deactivating OSC %s\n" % osc.uuid + raise e + + # ---------------------------------------------------------- delete + elif rec.nodeName == 'delete': + if config.cleanup: + continue + + osc = magic_get_osc(db, rec, lov) + + try: + config.cleanup = 1 + osc.cleanup() + config.cleanup = 0 + except CommandError, e: + print "Error cleaning up OSC %s\n" % osc.uuid + raise e + + lctl.lov_del_obd(lov_name, lov_uuid, ost_uuid, index, gen) + +def process_updates(db, log_device, log_name, lov = None): + updates = db.root_node.getElementsByTagName('update') + for u in updates: + if not u.childNodes: + log("ignoring empty update record (version " + + str(u.getAttribute('version')) + ")") + continue + + version = u.getAttribute('version') + real_name = "%s-%s" % (log_name, version) + lctl.clear_log(log_device, real_name) + lctl.record(log_device, real_name) + + process_update_record(db, u, lov) + + lctl.end_record() + +def doWriteconf(services): + #if config.nosetup: + # return for s in services: - startService(s[1], module_flag) + if s[1].get_class() == 'mdsdev': + n = newService(s[1]) + n.write_conf() +def doSetup(services): + if config.nosetup: + return + slist = [] + for s in services: + n = newService(s[1]) + n.level = s[0] + slist.append((n.level, n)) + nlist = [] + for n in slist: + nl = n[1].correct_level(n[0]) + nlist.append((nl, n[1])) + nlist.sort() + for n in nlist: + n[1].prepare() + +def doModules(services): + if config.nomod: + return + for s in services: + n = newService(s[1]) + n.load_module() + +def doCleanup(services): + if config.nosetup: + return + slist = [] + for s in services: + n = newService(s[1]) + n.level = s[0] + slist.append((n.level, n)) + nlist = [] + for n in slist: + nl = n[1].correct_level(n[0]) + nlist.append((nl, n[1])) + nlist.sort() + nlist.reverse() + for n in nlist: + if n[1].safe_to_clean(): + n[1].cleanup() + +def doUnloadModules(services): + if config.nomod: + return + services.reverse() + for s in services: + n = newService(s[1]) + if n.safe_to_clean_modules(): + n.cleanup_module() # -# Load profile for -def doHost(lustreNode, hosts): - global routes - dom_node = None +# Load profile for +def doHost(lustreDB, hosts): + global is_router, local_node_name + node_db = None for h in hosts: - dom_node = getByName(lustreNode, h, 'node') - if dom_node: + node_db = lustreDB.lookup_name(h, 'node') + if node_db: break + if not node_db: + panic('No host entry found.') - if not dom_node: - print 'No host entry found.' - return + local_node_name = node_db.get_val('name', 0) + is_router = node_db.get_val_int('router', 0) + lustre_upcall = node_db.get_val('lustreUpcall', '') + portals_upcall = node_db.get_val('portalsUpcall', '') + timeout = node_db.get_val_int('timeout', 0) + ptldebug = node_db.get_val('ptldebug', '') + subsystem = node_db.get_val('subsystem', '') - if not get_attr(dom_node, 'router'): - init_node(dom_node) - init_route_config(lustreNode) - else: - global router_flag - router_flag = 1 + find_local_clusters(node_db) + if not is_router: + find_local_routes(lustreDB) # Two step process: (1) load modules, (2) setup lustre # if not cleaning, load modules first. - module_flag = not config.cleanup() - reflist = dom_node.getElementsByTagName('profile') - for profile in reflist: - startProfile(lustreNode, profile, module_flag) + prof_list = node_db.get_refs('profile') + + if config.write_conf: + lustreDB.close() + for_each_profile(node_db, prof_list, doModules) + sys_make_devices() + for_each_profile(node_db, prof_list, doWriteconf) + for_each_profile(node_db, prof_list, doUnloadModules) + + elif config.recover: + if not (config.tgt_uuid and config.client_uuid and config.conn_uuid): + raise Lustre.LconfError( "--recovery requires --tgt_uuid " + + "--client_uuid --conn_uuid ") + doRecovery(lustreDB, lctl, config.tgt_uuid, config.client_uuid, + config.conn_uuid) + elif config.cleanup: + if config.force: + # the command line can override this value + timeout = 5 + # ugly hack, only need to run lctl commands for --dump + if config.lctl_dump or config.record: + for_each_profile(node_db, prof_list, doCleanup) + return + + sys_set_timeout(timeout) + sys_set_ptldebug(ptldebug) + sys_set_subsystem(subsystem) + sys_set_lustre_upcall(lustre_upcall) + sys_set_portals_upcall(portals_upcall) + + for_each_profile(node_db, prof_list, doCleanup) + for_each_profile(node_db, prof_list, doUnloadModules) + lustreDB.close() + + else: + # ugly hack, only need to run lctl commands for --dump + if config.lctl_dump or config.record: + sys_set_timeout(timeout) + sys_set_lustre_upcall(lustre_upcall) + for_each_profile(node_db, prof_list, doSetup) + return + + sys_make_devices() + sys_set_netmem_max('/proc/sys/net/core/rmem_max', MAXTCPBUF) + sys_set_netmem_max('/proc/sys/net/core/wmem_max', MAXTCPBUF) + + for_each_profile(node_db, prof_list, doModules) - if not config.cleanup(): sys_set_debug_path() - script = config.gdb_script() + sys_set_ptldebug(ptldebug) + sys_set_subsystem(subsystem) + script = config.gdb_script run(lctl.lctl, ' modules >', script) - if config.gdb(): - # dump /tmp/ogdb and sleep/pause here + if config.gdb: log ("The GDB module script is in", script) + # pause, so user has time to break and + # load the script time.sleep(5) - - module_flag = not module_flag - for profile in reflist: - startProfile(lustreNode, profile, module_flag) - -############################################################ -# Command line processing -# -def parse_cmdline(argv): - short_opts = "hdnv" - long_opts = ["ldap", "reformat", "lustre=", "verbose", "gdb", - "portals=", "makeldiff", "cleanup", "noexec", - "help", "node=", "get=", "nomod", "nosetup", - "dump="] - opts = [] - args = [] + sys_set_timeout(timeout) + sys_set_lustre_upcall(lustre_upcall) + sys_set_portals_upcall(portals_upcall) + + for_each_profile(node_db, prof_list, doSetup) + lustreDB.close() + +def doRecovery(lustreDB, lctl, tgt_uuid, client_uuid, nid_uuid): + tgt = lustreDB.lookup(tgt_uuid) + if not tgt: + raise Lustre.LconfError("doRecovery: "+ tgt_uuid +" not found.") + new_uuid = get_active_target(tgt) + if not new_uuid: + raise Lustre.LconfError("doRecovery: no active target found for: " + + tgt_uuid) + net = choose_local_server(get_ost_net(lustreDB, new_uuid)) + if not net: + raise Lustre.LconfError("Unable to find a connection to:" + new_uuid) + + log("Reconnecting", tgt_uuid, " to ", net.nid_uuid); try: - opts, args = getopt.getopt(argv, short_opts, long_opts) - except getopt.error: - print "invalid opt" - usage() - - for o, a in opts: - if o in ("-h", "--help"): - usage() - if o in ("-d","--cleanup"): - config.cleanup(1) - if o in ("-v", "--verbose"): - config.verbose(1) - if o in ("-n", "--noexec"): - config.noexec(1) - config.verbose(1) - if o == "--portals": - config.portals = a - if o == "--lustre": - config.lustre = a - if o == "--reformat": - config.reformat(1) - if o == "--node": - config.node(a) - if o == "--get": - config.url(a) - if o == "--gdb": - config.gdb(1) - if o == "--nomod": - config.nomod(1) - if o == "--nosetup": - config.nosetup(1) - if o == "--dump": - config.dump_file(a) - return args - -def fetch(url): - import urllib - data = "" + oldnet = get_server_by_nid_uuid(lustreDB, nid_uuid) + lustreDB.close() + if oldnet: + lctl.disconnect(oldnet) + except CommandError, e: + log("recover: disconnect", nid_uuid, "failed: ") + e.dump() + try: - s = urllib.urlopen(url) - data = s.read() - except: - usage() - return data + lctl.connect(net) + except CommandError, e: + log("recover: connect failed") + e.dump() -def setupModulePath(cmd): - base = os.path.dirname(cmd) - if os.access(base+"/Makefile", os.R_OK): - config.src_dir(base + "/../../") + lctl.recover(client_uuid, net.nid_uuid) -def sys_set_debug_path(): - debug("debug path: ", config.debug_path()) - if config.noexec(): + +def setupModulePath(cmd, portals_dir = PORTALS_DIR): + base = os.path.dirname(cmd) + if development_mode(): + if not config.lustre: + debug('using objdir module paths') + config.lustre = (os.path.join(base, "..")) + # normalize the portals dir, using command line arg if set + if config.portals: + portals_dir = config.portals + dir = os.path.join(config.lustre, portals_dir) + config.portals = dir + debug('config.portals', config.portals) + elif config.lustre and config.portals: + # production mode + # if --lustre and --portals, normalize portals + # can ignore POTRALS_DIR here, since it is probly useless here + config.portals = os.path.join(config.lustre, config.portals) + debug('config.portals B', config.portals) + +def sysctl(path, val): + debug("+ sysctl", path, val) + if config.noexec: return try: - fp = open('/proc/sys/portals/debug_path', 'w') - fp.write(config.debug_path()) + fp = open(os.path.join('/proc/sys', path), 'w') + fp.write(str(val)) fp.close() except IOError, e: - print e - -#/proc/sys/net/core/rmem_max -#/proc/sys/net/core/wmem_max + panic(str(e)) + + +def sys_set_debug_path(): + sysctl('portals/debug_path', config.debug_path) + +def sys_set_lustre_upcall(upcall): + # the command overrides the value in the node config + if config.lustre_upcall: + upcall = config.lustre_upcall + elif config.upcall: + upcall = config.upcall + if upcall: + lctl.set_lustre_upcall(upcall) + +def sys_set_portals_upcall(upcall): + # the command overrides the value in the node config + if config.portals_upcall: + upcall = config.portals_upcall + elif config.upcall: + upcall = config.upcall + if upcall: + sysctl('portals/upcall', upcall) + +def sys_set_timeout(timeout): + # the command overrides the value in the node config + if config.timeout and config.timeout > 0: + timeout = config.timeout + if timeout != None and timeout > 0: + lctl.set_timeout(timeout) + +def sys_tweak_socknal (): + if config.single_socket: + sysctl("socknal/typed", 0) + +def sys_optimize_elan (): + procfiles = ["/proc/elan/config/eventint_punt_loops", + "/proc/qsnet/elan3/config/eventint_punt_loops", + "/proc/qsnet/elan4/config/elan4_mainint_punt_loops"] + for p in procfiles: + if os.access(p, os.R_OK): + run ("echo 1 > " + p) + +def sys_set_ptldebug(ptldebug): + if config.ptldebug: + ptldebug = config.ptldebug + if ptldebug: + try: + val = eval(ptldebug, ptldebug_names) + val = "0x%x" % (val) + sysctl('portals/debug', val) + except NameError, e: + panic(str(e)) + +def sys_set_subsystem(subsystem): + if config.subsystem: + subsystem = config.subsystem + if subsystem: + try: + val = eval(subsystem, subsystem_names) + val = "0x%x" % (val) + sysctl('portals/subsystem_debug', val) + except NameError, e: + panic(str(e)) + def sys_set_netmem_max(path, max): debug("setting", path, "to at least", max) - if config.noexec(): + if config.noexec: return fp = open(path) str = fp.readline() - fp.close + fp.close() cur = int(str) if max > cur: fp = open(path, 'w') fp.write('%d\n' %(max)) fp.close() - - + + def sys_make_devices(): if not os.access('/dev/portals', os.R_OK): run('mknod /dev/portals c 10 240') if not os.access('/dev/obd', os.R_OK): run('mknod /dev/obd c 10 241') -# Initialize or shutdown lustre according to a configuration file -# * prepare the system for lustre -# * configure devices with lctl -# Shutdown does steps in reverse -# + +# Add dir to the global PATH, if not already there. +def add_to_path(new_dir): + syspath = string.split(os.environ['PATH'], ':') + if new_dir in syspath: + return + os.environ['PATH'] = os.environ['PATH'] + ':' + new_dir + +def default_debug_path(): + path = '/tmp/lustre-log' + if os.path.isdir('/r'): + return '/r' + path + else: + return path + +def default_gdb_script(): + script = '/tmp/ogdb' + if os.path.isdir('/r'): + return '/r' + script + else: + return script + + +DEFAULT_PATH = ('/sbin', '/usr/sbin', '/bin', '/usr/bin') +# ensure basic elements are in the system path +def sanitise_path(): + for dir in DEFAULT_PATH: + add_to_path(dir) + +# global hack for the --select handling +tgt_select = {} +def init_select(args): + # args = [service=nodeA,service2=nodeB service3=nodeC] + global tgt_select + for arg in args: + list = string.split(arg, ',') + for entry in list: + srv, node = string.split(entry, '=') + tgt_select[srv] = node + +def get_select(srv): + if tgt_select.has_key(srv): + return tgt_select[srv] + return None + + +FLAG = Lustre.Options.FLAG +PARAM = Lustre.Options.PARAM +INTPARAM = Lustre.Options.INTPARAM +PARAMLIST = Lustre.Options.PARAMLIST +lconf_options = [ + ('verbose,v', "Print system commands as they are run"), + ('ldapurl',"LDAP server URL, eg. ldap://localhost", PARAM), + ('config', "Cluster config name used for LDAP query", PARAM), + ('select', "service=nodeA,service2=nodeB ", PARAMLIST), + ('node', "Load config for ", PARAM), + ('cleanup,d', "Cleans up config. (Shutdown)"), + ('force,f', "Forced unmounting and/or obd detach during cleanup", + FLAG, 0), + ('single_socket', "socknal option: only use one socket instead of bundle", + FLAG, 0), + ('failover',"""Used to shut down without saving state. + This will allow this node to "give up" a service to a + another node for failover purposes. This will not + be a clean shutdown.""", + FLAG, 0), + ('gdb', """Prints message after creating gdb module script + and sleeps for 5 seconds."""), + ('noexec,n', """Prints the commands and steps that will be run for a + config without executing them. This can used to check if a + config file is doing what it should be doing"""), + ('nomod', "Skip load/unload module step."), + ('nosetup', "Skip device setup/cleanup step."), + ('reformat', "Reformat all devices (without question)"), + ('mkfsoptions', "Additional options for the mk*fs command line", PARAM), + ('mountfsoptions', "Additional options for mount fs command line", PARAM), + ('clientoptions', "Additional options for Lustre", PARAM), + ('dump', "Dump the kernel debug log to file before portals is unloaded", + PARAM), + ('write_conf', "Save all the client config information on mds."), + ('record', "Write config information on mds."), + ('record_log', "Name of config record log.", PARAM), + ('record_device', "MDS device name that will record the config commands", + PARAM), + ('minlevel', "Minimum level of services to configure/cleanup", + INTPARAM, 0), + ('maxlevel', """Maximum level of services to configure/cleanup + Levels are aproximatly like: + 10 - netwrk + 20 - device, ldlm + 30 - osd, mdd + 40 - mds, ost + 70 - mountpoint, echo_client, osc, mdc, lov""", + INTPARAM, 100), + ('lustre', """Base directory of lustre sources. This parameter will + cause lconf to load modules from a source tree.""", PARAM), + ('portals', """Portals source directory. If this is a relative path, + then it is assumed to be relative to lustre. """, PARAM), + ('timeout', "Set recovery timeout", INTPARAM), + ('upcall', "Set both portals and lustre upcall script", PARAM), + ('lustre_upcall', "Set lustre upcall script", PARAM), + ('portals_upcall', "Set portals upcall script", PARAM), + ('lctl_dump', "Save lctl ioctls to the dumpfile argument", PARAM), + ('ptldebug', "Set the portals debug level", PARAM), + ('subsystem', "Set the portals debug subsystem", PARAM), + ('gdb_script', "Fullname of gdb debug script", PARAM, default_gdb_script()), + ('debug_path', "Path to save debug dumps", PARAM, default_debug_path()), +# Client recovery options + ('recover', "Recover a device"), + ('group', "The group of devices to configure or cleanup", PARAM), + ('tgt_uuid', "The failed target (required for recovery)", PARAM), + ('client_uuid', "The failed client (required for recovery)", PARAM), + ('conn_uuid', "The failed connection (required for recovery)", PARAM), + + ('inactive', """The name of an inactive service, to be ignored during + mounting (currently OST-only). Can be repeated.""", + PARAMLIST), + ] + def main(): - global TCP_ACCEPTOR, lctl, MAXTCPBUF + global lctl, config, toplustreDB, CONFIG_FILE + + # in the upcall this is set to SIG_IGN + signal.signal(signal.SIGCHLD, signal.SIG_DFL) + + cl = Lustre.Options("lconf", "config.xml", lconf_options) + try: + config, args = cl.parse(sys.argv[1:]) + except Lustre.OptionError, e: + print e + sys.exit(1) + + setupModulePath(sys.argv[0]) + host = socket.gethostname() - args = parse_cmdline(sys.argv[1:]) + # the PRNG is normally seeded with time(), which is not so good for starting + # time-synchronized clusters + input = open('/dev/urandom', 'r') + if not input: + print 'Unable to open /dev/urandom!' + sys.exit(1) + seed = input.read(32) + input.close() + random.seed(seed) + + sanitise_path() + + init_select(config.select) + if len(args) > 0: - if not os.access(args[0], os.R_OK | os.W_OK): - print 'File not found:', args[0] + # allow config to be fetched via HTTP, but only with python2 + if sys.version[0] != '1' and args[0].startswith('http://'): + import urllib2 + try: + config_file = urllib2.urlopen(args[0]) + except (urllib2.URLError, socket.error), err: + if hasattr(err, 'args'): + err = err.args[1] + print "Could not access '%s': %s" %(args[0], err) + sys.exit(1) + elif not os.access(args[0], os.R_OK): + print 'File not found or readable:', args[0] sys.exit(1) - dom = xml.dom.minidom.parse(args[0]) - elif config.url(): - xmldata = fetch(config.url()) - dom = xml.dom.minidom.parseString(xmldata) + else: + # regular file + config_file = open(args[0], 'r') + try: + dom = xml.dom.minidom.parse(config_file) + except Exception: + panic("%s does not appear to be a config file." % (args[0])) + sys.exit(1) # make sure to die here, even in debug mode. + config_file.close() + CONFIG_FILE = args[0] + lustreDB = Lustre.LustreDB_XML(dom.documentElement, dom.documentElement) + if not config.config: + config.config = os.path.basename(args[0])# use full path? + if config.config[-4:] == '.xml': + config.config = config.config[:-4] + elif config.ldapurl: + if not config.config: + panic("--ldapurl requires --config name") + dn = "config=%s,fs=lustre" % (config.config) + lustreDB = Lustre.LustreDB_LDAP('', {}, base=dn, url = config.ldapurl) + elif config.ptldebug or config.subsystem: + sys_set_ptldebug(None) + sys_set_subsystem(None) + sys.exit(0) else: - usage() + print 'Missing config file or ldap URL.' + print 'see lconf --help for command summary' + sys.exit(1) + + toplustreDB = lustreDB + + ver = lustreDB.get_version() + if not ver: + panic("No version found in config data, please recreate.") + if ver != Lustre.CONFIG_VERSION: + panic("Config version", ver, "does not match lconf version", + Lustre.CONFIG_VERSION) node_list = [] - if config.node(): - node_list.append(config.node()) + if config.node: + node_list.append(config.node) else: if len(host) > 0: node_list.append(host) node_list.append('localhost') + debug("configuring for host: ", node_list) if len(host) > 0: - config._debug_path = config._debug_path + '-' + host - config._gdb_script = config._gdb_script + '-' + host - - TCP_ACCEPTOR = find_prog('acceptor') - if not TCP_ACCEPTOR: - if config.noexec(): - TCP_ACCEPTOR = 'acceptor' - debug('! acceptor not found') - else: - panic('acceptor not found') + config.debug_path = config.debug_path + '-' + host + config.gdb_script = config.gdb_script + '-' + host lctl = LCTLInterface('lctl') - setupModulePath(sys.argv[0]) - sys_make_devices() - sys_set_netmem_max('/proc/sys/net/core/rmem_max', MAXTCPBUF) - sys_set_netmem_max('/proc/sys/net/core/wmem_max', MAXTCPBUF) - doHost(dom.documentElement, node_list) + if config.lctl_dump: + lctl.use_save_file(config.lctl_dump) + + if config.record: + if not (config.record_device and config.record_log): + panic("When recording, both --record_log and --record_device must be specified.") + lctl.clear_log(config.record_device, config.record_log) + lctl.record(config.record_device, config.record_log) + + doHost(lustreDB, node_list) + + if not config.record: + return + + lctl.end_record() + + process_updates(lustreDB, config.record_device, config.record_log) if __name__ == "__main__": try: main() - except LconfError, e: + except Lustre.LconfError, e: print e +# traceback.print_exc(file=sys.stdout) + sys.exit(1) except CommandError, e: e.dump() + sys.exit(e.rc) + + if first_cleanup_error: + sys.exit(first_cleanup_error)