#!/usr/bin/env python
#
-# Copyright (C) 2002 Cluster File Systems, Inc.
-# Author: Robert Read <rread@clusterfs.com>
-
+# Copyright (C) 2002-2003 Cluster File Systems, Inc.
+# Authors: Robert Read <rread@clusterfs.com>
+# Mike Shaver <shaver@clusterfs.com>
# This file is part of Lustre, http://www.lustre.org.
#
# Lustre is free software; you can redistribute it and/or
# lconf is the main driver script for starting and stopping
# lustre filesystem services.
#
-# Based in part on the XML obdctl modifications done by Brian Behlendorf
+# Based in part on the XML obdctl modifications done by Brian Behlendorf
-import sys, getopt
-import string, os, stat, popen2, socket, time
-import re, exceptions
+import sys, getopt, types
+import string, os, stat, popen2, socket, time, random, fcntl, select
+import re, exceptions, signal, traceback
import xml.dom.minidom
+if sys.version[0] == '1':
+ from FCNTL import F_GETFL, F_SETFL
+else:
+ from fcntl import F_GETFL, F_SETFL
+
+PYMOD_DIR = "/usr/lib/lustre/python"
+
+def development_mode():
+ base = os.path.dirname(sys.argv[0])
+ if os.access(base+"/Makefile", os.R_OK):
+ return 1
+ return 0
+
+if development_mode():
+ sys.path.append('../utils')
+else:
+ sys.path.append(PYMOD_DIR)
+
+import Lustre
+
# Global parameters
-TCP_ACCEPTOR = ''
-MAXTCPBUF = 1048576
+MAXTCPBUF = 16777216
+DEFAULT_TCPBUF = 8388608
+DEFAULT_PORT = 988
#
# Maximum number of devices to search for.
# (the /dev/loop* nodes need to be created beforehand)
MAX_LOOP_DEVICES = 256
-
-
-def usage():
- print """usage: lconf config.xml
-
-config.xml Lustre configuration in xml format.
---get <url> URL to fetch a config file
---node <nodename> Load config for <nodename>
--d | --cleanup Cleans up config. (Shutdown)
--v | --verbose Print system commands as they are run
--h | --help Print this help
---gdb Prints message after creating gdb module script
- and sleeps for 5 seconds.
--n | --noexec Prints the commands and steps that will be run for a
- config without executing them. This can used to check if a
- config file is doing what it should be doing. (Implies -v)
---nomod Skip load/unload module step.
---nosetup Skip device setup/cleanup step.
-"""
- TODO = """
---ldap server LDAP server with lustre config database
---makeldiff Translate xml source to LDIFF
---reformat Reformat all devices (will confirm)
-This are perhaps not needed:
---lustre="src dir" Base directory of lustre sources. Used to search
- for modules.
---portals=src Portals source
-"""
- sys.exit()
+PORTALS_DIR = 'portals'
+
+# Needed to call lconf --record
+CONFIG_FILE = ""
+
+# Please keep these in sync with the values in portals/kp30.h
+ptldebug_names = {
+ "trace" : (1 << 0),
+ "inode" : (1 << 1),
+ "super" : (1 << 2),
+ "ext2" : (1 << 3),
+ "malloc" : (1 << 4),
+ "cache" : (1 << 5),
+ "info" : (1 << 6),
+ "ioctl" : (1 << 7),
+ "blocks" : (1 << 8),
+ "net" : (1 << 9),
+ "warning" : (1 << 10),
+ "buffs" : (1 << 11),
+ "other" : (1 << 12),
+ "dentry" : (1 << 13),
+ "portals" : (1 << 14),
+ "page" : (1 << 15),
+ "dlmtrace" : (1 << 16),
+ "error" : (1 << 17),
+ "emerg" : (1 << 18),
+ "ha" : (1 << 19),
+ "rpctrace" : (1 << 20),
+ "vfstrace" : (1 << 21),
+ "reada" : (1 << 22),
+ "config" : (1 << 23),
+ }
+
+subsystem_names = {
+ "undefined" : (1 << 0),
+ "mdc" : (1 << 1),
+ "mds" : (1 << 2),
+ "osc" : (1 << 3),
+ "ost" : (1 << 4),
+ "class" : (1 << 5),
+ "log" : (1 << 6),
+ "llite" : (1 << 7),
+ "rpc" : (1 << 8),
+ "mgmt" : (1 << 9),
+ "portals" : (1 << 10),
+ "socknal" : (1 << 11),
+ "qswnal" : (1 << 12),
+ "pinger" : (1 << 13),
+ "filter" : (1 << 14),
+ "ptlbd" : (1 << 15),
+ "echo" : (1 << 16),
+ "ldlm" : (1 << 17),
+ "lov" : (1 << 18),
+ "gmnal" : (1 << 19),
+ "ptlrouter" : (1 << 20),
+ "cobd" : (1 << 21),
+ "ibnal" : (1 << 22),
+ "cmobd" : (1 << 23),
+ }
+
+
+first_cleanup_error = 0
+def cleanup_error(rc):
+ global first_cleanup_error
+ if not first_cleanup_error:
+ first_cleanup_error = rc
# ============================================================
-# Config parameters, encapsulated in a class
-class Config:
- def __init__(self):
- # flags
- self._noexec = 0
- self._verbose = 0
- self._reformat = 0
- self._cleanup = 0
- self._gdb = 0
- self._nomod = 0
- self._nosetup = 0
- # parameters
- self._modules = None
- self._node = None
- self._url = None
- self._gdb_script = '/tmp/ogdb'
- self._debug_path = '/tmp/lustre-log'
- self._src_dir = None
-
- def verbose(self, flag = None):
- if flag: self._verbose = flag
- return self._verbose
-
- def noexec(self, flag = None):
- if flag: self._noexec = flag
- return self._noexec
-
- def reformat(self, flag = None):
- if flag: self._reformat = flag
- return self._reformat
-
- def cleanup(self, flag = None):
- if flag: self._cleanup = flag
- return self._cleanup
-
- def gdb(self, flag = None):
- if flag: self._gdb = flag
- return self._gdb
-
- def nomod(self, flag = None):
- if flag: self._nomod = flag
- return self._nomod
-
- def nosetup(self, flag = None):
- if flag: self._nosetup = flag
- return self._nosetup
-
- def node(self, val = None):
- if val: self._node = val
- return self._node
-
- def url(self, val = None):
- if val: self._url = val
- return self._url
-
- def gdb_script(self):
- if os.path.isdir('/r'):
- return '/r' + self._gdb_script
- else:
- return self._gdb_script
-
- def debug_path(self):
- if os.path.isdir('/r'):
- return '/r' + self._debug_path
- else:
- return self._debug_path
-
- def src_dir(self, val = None):
- if val: self._url = val
- return self._url
-
-config = Config()
-
-# ============================================================
# debugging and error funcs
def fixme(msg = "this feature"):
- raise LconfError, msg + ' not implmemented yet.'
+ raise Lustre.LconfError, msg + ' not implemented yet.'
def panic(*args):
msg = string.join(map(str,args))
- if not config.noexec():
- raise LconfError(msg)
+ if not config.noexec:
+ raise Lustre.LconfError(msg)
else:
print "! " + msg
print string.strip(s)
def debug(*args):
- if config.verbose():
+ if config.verbose:
msg = string.join(map(str,args))
print msg
+# ack, python's builtin int() does not support '0x123' syntax.
+# eval can do it, although what a hack!
+def my_int(s):
+ try:
+ if s[0:2] == '0x':
+ return eval(s, {}, {})
+ else:
+ return int(s)
+ except SyntaxError, e:
+ raise ValueError("not a number")
+ except NameError, e:
+ raise ValueError("not a number")
+
# ============================================================
# locally defined exceptions
class CommandError (exceptions.Exception):
else:
print self.cmd_err
-class LconfError (exceptions.Exception):
- def __init__(self, args):
- self.args = args
+
+# ============================================================
+# handle daemons, like the acceptor
+class DaemonHandler:
+ """ Manage starting and stopping a daemon. Assumes daemon manages
+ it's own pid file. """
+
+ def __init__(self, cmd):
+ self.command = cmd
+ self.path =""
+
+ def start(self):
+ if self.running():
+ log(self.command, "already running.")
+ if not self.path:
+ self.path = find_prog(self.command)
+ if not self.path:
+ panic(self.command, "not found.")
+ ret, out = runcmd(self.path +' '+ self.command_line())
+ if ret:
+ raise CommandError(self.path, out, ret)
+
+ def stop(self):
+ if self.running():
+ pid = self.read_pidfile()
+ try:
+ log ("killing process", pid)
+ os.kill(pid, 15)
+ #time.sleep(1) # let daemon die
+ except OSError, e:
+ log("unable to kill", self.command, e)
+ if self.running():
+ log("unable to kill", self.command)
+
+ def running(self):
+ pid = self.read_pidfile()
+ if pid:
+ try:
+ os.kill(pid, 0)
+ except OSError:
+ self.clean_pidfile()
+ else:
+ return 1
+ return 0
+
+ def read_pidfile(self):
+ try:
+ fp = open(self.pidfile(), 'r')
+ pid = int(fp.read())
+ fp.close()
+ return pid
+ except IOError:
+ return 0
+
+ def clean_pidfile(self):
+ """ Remove a stale pidfile """
+ log("removing stale pidfile:", self.pidfile())
+ try:
+ os.unlink(self.pidfile())
+ except OSError, e:
+ log(self.pidfile(), e)
+
+class AcceptorHandler(DaemonHandler):
+ def __init__(self, port, net_type, send_mem, recv_mem, irq_aff):
+ DaemonHandler.__init__(self, "acceptor")
+ self.port = port
+ self.flags = ''
+ self.send_mem = send_mem
+ self.recv_mem = recv_mem
+
+ if irq_aff:
+ self.flags = self.flags + ' -i'
+
+ def pidfile(self):
+ return "/var/run/%s-%d.pid" % (self.command, self.port)
+
+ def command_line(self):
+ return string.join(map(str,('-s', self.send_mem, '-r', self.recv_mem, self.flags, self.port)))
+
+acceptors = {}
+
+# start the acceptors
+def run_acceptors():
+ if config.lctl_dump or config.record:
+ return
+ for port in acceptors.keys():
+ daemon = acceptors[port]
+ if not daemon.running():
+ daemon.start()
+
+def run_one_acceptor(port):
+ if config.lctl_dump or config.record:
+ return
+ if acceptors.has_key(port):
+ daemon = acceptors[port]
+ if not daemon.running():
+ daemon.start()
+ else:
+ panic("run_one_acceptor: No acceptor defined for port:", port)
+
+def stop_acceptor(port):
+ if acceptors.has_key(port):
+ daemon = acceptors[port]
+ if daemon.running():
+ daemon.stop()
# ============================================================
Initialize close by finding the lctl binary.
"""
self.lctl = find_prog(cmd)
+ self.save_file = ''
+ self.record_device = ''
if not self.lctl:
- if config.noexec():
+ if config.noexec:
debug('! lctl not found')
self.lctl = 'lctl'
else:
raise CommandError('lctl', "unable to find lctl binary.")
-
+
+ def use_save_file(self, file):
+ self.save_file = file
+
+ def record(self, dev_name, logname):
+ log("Recording log", logname, "on", dev_name)
+ self.record_device = dev_name
+ self.record_log = logname
+
+ def end_record(self):
+ log("End recording log", self.record_log, "on", self.record_device)
+ self.record_device = None
+ self.record_log = None
+
+ def set_nonblock(self, fd):
+ fl = fcntl.fcntl(fd, F_GETFL)
+ fcntl.fcntl(fd, F_SETFL, fl | os.O_NDELAY)
+
def run(self, cmds):
"""
run lctl
should modify command line to accept multiple commands, or
create complex command line options
"""
- debug("+", self.lctl, cmds)
- if config.noexec(): return (0, [])
- p = popen2.Popen3(self.lctl, 1)
- p.tochild.write(cmds + "\n")
- p.tochild.close()
- out = p.fromchild.readlines()
- err = p.childerr.readlines()
- ret = p.wait()
- if ret or len(err):
- raise CommandError(self.lctl, err, ret)
- return ret, out
+ cmd_line = self.lctl
+ if self.save_file:
+ cmds = '\n dump ' + self.save_file + '\n' + cmds
+ elif self.record_device:
+ cmds = """
+ device $%s
+ record %s
+ %s""" % (self.record_device, self.record_log, cmds)
+
+ debug("+", cmd_line, cmds)
+ if config.noexec: return (0, [])
+
+ child = popen2.Popen3(cmd_line, 1) # Capture stdout and stderr from command
+ child.tochild.write(cmds + "\n")
+ child.tochild.close()
+# print "LCTL:", cmds
+
+ # From "Python Cookbook" from O'Reilly
+ outfile = child.fromchild
+ outfd = outfile.fileno()
+ self.set_nonblock(outfd)
+ errfile = child.childerr
+ errfd = errfile.fileno()
+ self.set_nonblock(errfd)
+
+ outdata = errdata = ''
+ outeof = erreof = 0
+ while 1:
+ ready = select.select([outfd,errfd],[],[]) # Wait for input
+ if outfd in ready[0]:
+ outchunk = outfile.read()
+ if outchunk == '': outeof = 1
+ outdata = outdata + outchunk
+ if errfd in ready[0]:
+ errchunk = errfile.read()
+ if errchunk == '': erreof = 1
+ errdata = errdata + errchunk
+ if outeof and erreof: break
+ # end of "borrowed" code
+
+ ret = child.wait()
+ if os.WIFEXITED(ret):
+ rc = os.WEXITSTATUS(ret)
+ else:
+ rc = 0
+ if rc or len(errdata):
+ raise CommandError(self.lctl, errdata, rc)
+ return rc, outdata
+
+ def runcmd(self, *args):
+ """
+ run lctl using the command line
+ """
+ cmd = string.join(map(str,args))
+ debug("+", self.lctl, cmd)
+ rc, out = run(self.lctl, cmd)
+ if rc:
+ raise CommandError(self.lctl, out, rc)
+ return rc, out
+
+
+ def clear_log(self, dev, log):
+ """ clear an existing log """
+ cmds = """
+ device $%s
+ probe
+ clear_log %s
+ quit """ % (dev, log)
+ self.run(cmds)
-
def network(self, net, nid):
- """ initialized network and add "self" """
- # Idea: "mynid" could be used for all network types to add "self," and then
- # this special case would be gone and the "self" hack would be hidden.
- if net == 'tcp':
- cmds = """
+ """ set mynid """
+ cmds = """
network %s
mynid %s
- add_uuid self %s
- quit""" % (net, nid, nid)
- else:
- cmds = """
- network %s
- add_uuid self %s
- quit""" % (net, nid)
-
+ quit """ % (net, nid)
self.run(cmds)
# create a new connection
- def connect(self, net, nid, port, servuuid, send_mem, recv_mem):
- if net == 'tcp':
+ def add_uuid(self, net_type, uuid, nid):
+ cmds = "\n add_uuid %s %s %s" %(uuid, nid, net_type)
+ self.run(cmds)
+
+ def add_autoconn(self, net_type, send_mem, recv_mem, nid, hostaddr,
+ port, flags):
+ if net_type in ('tcp',) and not config.lctl_dump:
cmds = """
network %s
- add_uuid %s %s
send_mem %d
recv_mem %d
- connect %s %d
- quit""" % (net, servuuid, nid, send_mem, recv_mem, nid, port, )
- else:
- cmds = """
- network %s
- add_uuid %s %s
- connect %s %d
- quit""" % (net, servuuid, nid, nid, port, )
-
+ add_autoconn %s %s %d %s
+ quit""" % (net_type,
+ send_mem,
+ recv_mem,
+ nid, hostaddr, port, flags )
+ self.run(cmds)
+
+ def connect(self, srv):
+ self.add_uuid(srv.net_type, srv.nid_uuid, srv.nid)
+ if srv.net_type in ('tcp',) and not config.lctl_dump:
+ flags = 's'
+ if srv.irq_affinity:
+ flags = flags + 'i'
+ self.add_autoconn(srv.net_type, srv.send_mem, srv.recv_mem,
+ srv.nid, srv.hostaddr, srv.port, flags)
+
+ # Recover a device
+ def recover(self, dev_name, new_conn):
+ cmds = """
+ device $%s
+ recover %s""" %(dev_name, new_conn)
self.run(cmds)
-
+
# add a route to a range
def add_route(self, net, gw, lo, hi):
cmds = """
network %s
add_route %s %s %s
- quit """ % (net, gw, lo, hi)
- self.run(cmds)
+ quit """ % (net,
+ gw, lo, hi)
+ try:
+ self.run(cmds)
+ except CommandError, e:
+ log ("ignore: ")
+ e.dump()
-
- # add a route to a range
def del_route(self, net, gw, lo, hi):
cmds = """
+ ignore_errors
network %s
- del_route %s %s
- quit """ % (net, lo, hi)
+ del_route %s %s %s
+ quit """ % (net, gw, lo, hi)
self.run(cmds)
# add a route to a host
def add_route_host(self, net, uuid, gw, tgt):
+ self.add_uuid(net, uuid, tgt)
cmds = """
network %s
- add_uuid %s %s
add_route %s %s
- quit """ % (net, uuid, tgt, gw, tgt)
+ quit """ % (net,
+ gw, tgt)
+ try:
+ self.run(cmds)
+ except CommandError, e:
+ log ("ignore: ")
+ e.dump()
+
+ # add a route to a range
+ def del_route_host(self, net, uuid, gw, tgt):
+ self.del_uuid(uuid)
+ cmds = """
+ ignore_errors
+ network %s
+ del_route %s %s
+ quit """ % (net, gw, tgt)
self.run(cmds)
+
+ def del_autoconn(self, net_type, nid, hostaddr):
+ if net_type in ('tcp',) and not config.lctl_dump:
+ cmds = """
+ ignore_errors
+ network %s
+ del_autoconn %s %s s
+ quit""" % (net_type,
+ nid, hostaddr)
+ self.run(cmds)
+
# disconnect one connection
- def disconnect(self, net, nid, port, servuuid):
+ def disconnect(self, srv):
+ self.del_uuid(srv.nid_uuid)
+ if srv.net_type in ('tcp',) and not config.lctl_dump:
+ self.del_autoconn(srv.net_type, srv.nid, srv.hostaddr)
+
+ def del_uuid(self, uuid):
cmds = """
- network %s
- disconnect %s
+ ignore_errors
del_uuid %s
- quit""" % (net, nid, servuuid)
+ quit""" % (uuid,)
self.run(cmds)
- # disconnect all connections
+ # disconnect all
def disconnectAll(self, net):
cmds = """
+ ignore_errors
network %s
disconnect
- del_uuid self
quit""" % (net)
self.run(cmds)
- # create a new device with lctl
- def newdev(self, attach, setup = ""):
+ def attach(self, type, name, uuid):
+ cmds = """
+ attach %s %s %s
+ quit""" % (type, name, uuid)
+ self.run(cmds)
+
+ def setup(self, name, setup = ""):
cmds = """
- newdev
- attach %s
+ cfg_device %s
setup %s
- quit""" % (attach, setup)
+ quit""" % (name, setup)
self.run(cmds)
+
+ # create a new device with lctl
+ def newdev(self, type, name, uuid, setup = ""):
+ self.attach(type, name, uuid);
+ try:
+ self.setup(name, setup)
+ except CommandError, e:
+ self.cleanup(name, uuid, 0)
+ raise e
+
+
# cleanup a device
- def cleanup(self, name, uuid):
+ def cleanup(self, name, uuid, force, failover = 0):
+ if failover: force = 1
cmds = """
- device $%s
- cleanup
+ ignore_errors
+ cfg_device $%s
+ cleanup %s %s
detach
- quit""" % (name)
+ quit""" % (name, ('', 'force')[force],
+ ('', 'failover')[failover])
self.run(cmds)
# create an lov
- def lovconfig(self, uuid, mdsuuid, stripe_cnt, stripe_sz, stripe_off, pattern, devlist):
+ def lov_setup(self, name, uuid, desc_uuid, stripe_cnt,
+ stripe_sz, stripe_off, pattern):
+ cmds = """
+ attach lov %s %s
+ lov_setup %s %d %d %d %s
+ quit""" % (name, uuid, desc_uuid, stripe_cnt, stripe_sz, stripe_off, pattern)
+ self.run(cmds)
+
+ # add an OBD to a LOV
+ def lov_add_obd(self, name, uuid, obd_uuid, index, gen):
+ cmds = """
+ lov_modify_tgts add %s %s %s %s
+ quit""" % (name, obd_uuid, index, gen)
+ self.run(cmds)
+
+ # create an lmv
+ def lmv_setup(self, name, uuid, desc_uuid, devlist):
+ cmds = """
+ attach lmv %s %s
+ lmv_setup %s %s
+ quit""" % (name, uuid, desc_uuid, devlist)
+ self.run(cmds)
+
+ # delete an OBD from a LOV
+ def lov_del_obd(self, name, uuid, obd_uuid, index, gen):
+ cmds = """
+ lov_modify_tgts del %s %s %s %s
+ quit""" % (name, obd_uuid, index, gen)
+ self.run(cmds)
+
+ # deactivate an OBD
+ def deactivate(self, name):
cmds = """
device $%s
- probe
- lovconfig %s %d %d %d %s %s
- quit""" % (mdsuuid, uuid, stripe_cnt, stripe_sz, stripe_off, pattern, devlist)
+ deactivate
+ quit""" % (name)
self.run(cmds)
+ # dump the log file
+ def dump(self, dump_file):
+ cmds = """
+ debug_kernel %s 1
+ quit""" % (dump_file)
+ self.run(cmds)
+
+ # get list of devices
+ def device_list(self):
+ devices = '/proc/fs/lustre/devices'
+ ret = []
+ if os.access(devices, os.R_OK):
+ try:
+ fp = open(devices, 'r')
+ ret = fp.readlines()
+ fp.close()
+ except IOError, e:
+ log(e)
+ return ret
+
+ # get lustre version
+ def lustre_version(self):
+ rc, out = self.runcmd('version')
+ return out
+
+ # dump mount options
+ def mount_option(self, profile, osc, mdc):
+ cmds = """
+ mount_option %s %s %s
+ quit""" % (profile, osc, mdc)
+ self.run(cmds)
+
+ # delete mount options
+ def del_mount_option(self, profile):
+ cmds = """
+ del_mount_option %s
+ quit""" % (profile,)
+ self.run(cmds)
+
+ def set_timeout(self, timeout):
+ cmds = """
+ set_timeout %s
+ quit""" % (timeout,)
+ self.run(cmds)
+
+ def set_lustre_upcall(self, upcall):
+ cmds = """
+ set_lustre_upcall %s
+ quit""" % (upcall,)
+ self.run(cmds)
# ============================================================
# Various system-level functions
# (ideally moved to their own module)
# Run a command and return the output and status.
# stderr is sent to /dev/null, could use popen3 to
# save it if necessary
-def run(*args):
- cmd = string.join(map(str,args))
+def runcmd(cmd):
debug ("+", cmd)
- if config.noexec(): return (0, [])
+ if config.noexec: return (0, [])
f = os.popen(cmd + ' 2>&1')
out = f.readlines()
ret = f.close()
ret = 0
return (ret, out)
+def run(*args):
+ cmd = string.join(map(str,args))
+ return runcmd(cmd)
+
# Run a command in the background.
def run_daemon(*args):
cmd = string.join(map(str,args))
debug ("+", cmd)
- if config.noexec(): return 0
+ if config.noexec: return 0
f = os.popen(cmd + ' 2>&1')
ret = f.close()
if ret:
syspath = string.split(os.environ['PATH'], ':')
cmdpath = os.path.dirname(sys.argv[0])
syspath.insert(0, cmdpath);
- syspath.insert(0, os.path.join(cmdpath, '../../portals/linux/utils/'))
+ if config.portals:
+ syspath.insert(0, os.path.join(config.portals, 'utils/'))
for d in syspath:
prog = os.path.join(d,cmd)
if os.access(prog, os.X_OK):
if module:
return module
-def find_module(src_dir, modname):
- mod = '%s.o' % (modname)
- search = (src_dir + "/lustre", src_dir + "/portals/linux")
- for d in search:
+def find_module(src_dir, dev_dir, modname):
+ modbase = src_dir +'/'+ dev_dir +'/'+ modname
+ for modext in '.ko', '.o':
+ module = modbase + modext
try:
- module = do_find_file(d, mod)
- if module:
+ if os.access(module, os.R_OK):
return module
except OSError:
pass
except OSError:
return 0
return stat.S_ISBLK(s[stat.ST_MODE])
-
+
# build fs according to type
# fixme: dangerous
-def mkfs(fstype, dev):
- if(fstype in ('ext3', 'extN')):
- mkfs = 'mkfs.ext2 -j -b 4096'
- else:
- print 'unsupported fs type: ', fstype
- if not is_block(dev):
- force = '-F'
+def mkfs(dev, devsize, fstype, jsize, isize, mkfsoptions, isblock=1):
+ block_cnt = ''
+ jopt = ''
+ iopt = ''
+ if devsize:
+ if devsize < 8000:
+ panic("size of filesystem on '%s' must be larger than 8MB, but is set to %s"%
+ (dev, devsize))
+ # devsize is in 1k, and fs block count is in 4k
+ block_cnt = devsize/4
+
+ if fstype in ('ext3', 'extN', 'ldiskfs'):
+ # ext3 journal size is in megabytes
+ if jsize == 0:
+ if devsize == 0:
+ if not is_block(dev):
+ ret, out = runcmd("ls -l %s" %dev)
+ devsize = int(string.split(out[0])[4]) / 1024
+ else:
+ ret, out = runcmd("sfdisk -s %s" %dev)
+ devsize = int(out[0])
+ if devsize > 1024 * 1024:
+ jsize = ((devsize / 102400) * 4)
+ if jsize > 400:
+ jsize = 400
+ if jsize: jopt = "-J size=%d" %(jsize,)
+ if isize: iopt = "-I %d" %(isize,)
+ mkfs = 'mkfs.ext2 -j -b 4096 '
+ if not isblock or config.force:
+ mkfs = mkfs + ' -F '
+ elif fstype == 'reiserfs':
+ # reiserfs journal size is in blocks
+ if jsize: jopt = "--journal_size %d" %(jsize,)
+ mkfs = 'mkreiserfs -ff'
else:
- force = ''
- (ret, out) = run (mkfs, force, dev)
+ panic('unsupported fs type: ', fstype)
+
+ if config.mkfsoptions != None:
+ mkfs = mkfs + ' ' + config.mkfsoptions
+ if mkfsoptions != None:
+ mkfs = mkfs + ' ' + mkfsoptions
+ (ret, out) = run (mkfs, jopt, iopt, dev, block_cnt)
if ret:
- panic("Unable to build fs:", dev)
- # enable hash tree indexing on fs
- if fstype == 'extN':
+ panic("Unable to build fs:", dev, string.join(out))
+ # enable hash tree indexing on fsswe
+ if fstype in ('ext3', 'extN', 'ldiskfs'):
htree = 'echo "feature FEATURE_C5" | debugfs -w'
(ret, out) = run (htree, dev)
if ret:
panic ("can't access loop devices")
return loop
-# find loop device assigned to thefile
-def find_loop(file):
+# find loop device assigned to the file
+def find_assigned_loop(file):
loop = loop_base()
for n in xrange(0, MAX_LOOP_DEVICES):
dev = loop + str(n)
if os.access(dev, os.R_OK):
(stat, out) = run('losetup', dev)
- if (out and stat == 0):
+ if out and stat == 0:
m = re.search(r'\((.*)\)', out[0])
if m and file == m.group(1):
return dev
return ''
# create file if necessary and assign the first free loop device
-def init_loop(file, size, fstype):
- dev = find_loop(file)
+def init_loop(file, size, fstype, journal_size, inode_size,
+ mkfsoptions, reformat, autoformat, backfstype, backfile):
+ if fstype == 'smfs':
+ realfile = backfile
+ realfstype = backfstype
+ if is_block(backfile):
+ if reformat or (need_format(realfstype, backfile) and autoformat == 'yes'):
+ mkfs(realfile, size, realfstype, journal_size, inode_size, mkfsoptions, isblock=0)
+ return realfile
+ else:
+ realfile = file
+ realfstype = fstype
+
+ dev = find_assigned_loop(realfile)
if dev:
- print 'WARNING file:', file, 'already mapped to', dev
+ print 'WARNING file:', realfile, 'already mapped to', dev
return dev
- if not os.access(file, os.R_OK | os.W_OK):
- run("dd if=/dev/zero bs=1k count=0 seek=%d of=%s" %(size, file))
+
+ if reformat or not os.access(realfile, os.R_OK | os.W_OK):
+ if size < 8000:
+ panic("size of loopback file '%s' must be larger than 8MB, but is set to %s" % (realfile, size))
+ (ret, out) = run("dd if=/dev/zero bs=1k count=0 seek=%d of=%s" %(size, realfile))
+ if ret:
+ panic("Unable to create backing store:", realfile)
+
+ mkfs(realfile, size, realfstype, journal_size, inode_size,
+ mkfsoptions, isblock=0)
+
loop = loop_base()
# find next free loop
for n in xrange(0, MAX_LOOP_DEVICES):
dev = loop + str(n)
if os.access(dev, os.R_OK):
(stat, out) = run('losetup', dev)
- if (stat):
- run('losetup', dev, file)
+ if stat:
+ run('losetup', dev, realfile)
return dev
else:
print "out of loop devices"
# undo loop assignment
def clean_loop(file):
- dev = find_loop(file)
+ dev = find_assigned_loop(file)
if dev:
ret, out = run('losetup -d', dev)
if ret:
# determine if dev is formatted as a <fstype> filesystem
def need_format(fstype, dev):
- # FIXME don't know how to implement this
+ # FIXME don't know how to implement this
return 0
# initialize a block device if needed
-def block_dev(dev, size, fstype, format):
- if config.noexec(): return dev
- if not is_block(dev):
- dev = init_loop(dev, size, fstype)
- if config.reformat() or (need_format(fstype, dev) and format == 'yes'):
- mkfs(fstype, dev)
-
+def block_dev(dev, size, fstype, reformat, autoformat, journal_size,
+ inode_size, mkfsoptions, backfstype, backdev):
+ if config.noexec:
+ return dev
+
+ if fstype == 'smfs' or not is_block(dev):
+ dev = init_loop(dev, size, fstype, journal_size, inode_size,
+ mkfsoptions, reformat, autoformat, backfstype, backdev)
+ elif reformat or (need_format(fstype, dev) and autoformat == 'yes'):
+ mkfs(dev, size, fstype, journal_size, inode_size, mkfsoptions,
+ isblock=0)
# else:
# panic("device:", dev,
# "not prepared, and autoformat is not set.\n",
# "Rerun with --reformat option to format ALL filesystems")
-
+
return dev
-def get_local_address(net_type):
+def if2addr(iface):
+ """lookup IP address for an interface"""
+ rc, out = run("/sbin/ifconfig", iface)
+ if rc or not out:
+ return None
+ addr = string.split(out[1])[1]
+ ip = string.split(addr, ':')[1]
+ return ip
+
+def def_mount_options(fstype, target):
+ """returns deafult mount options for passed fstype and target (mds, ost)"""
+ if fstype == 'ext3' or fstype == 'ldiskfs':
+ mountfsoptions = "errors=remount-ro"
+ if target == 'ost' and sys_get_branch() == '2.4':
+ mountfsoptions = "%s,asyncdel" % (mountfsoptions)
+ return mountfsoptions
+ return ""
+
+def sys_get_elan_position_file():
+ procfiles = ["/proc/elan/device0/position",
+ "/proc/qsnet/elan4/device0/position",
+ "/proc/qsnet/elan3/device0/position"]
+ for p in procfiles:
+ if os.access(p, os.R_OK):
+ return p
+ return ""
+
+def sys_get_local_nid(net_type, wildcard, cluster_id):
+ """Return the local nid."""
+ local = ""
+ if sys_get_elan_position_file():
+ local = sys_get_local_address('elan', '*', cluster_id)
+ else:
+ local = sys_get_local_address(net_type, wildcard, cluster_id)
+ return local
+
+def sys_get_local_address(net_type, wildcard, cluster_id):
"""Return the local address for the network type."""
local = ""
- if net_type == 'tcp':
- # host `hostname`
- host = socket.gethostname()
- local = socket.gethostbyname(host)
+ if net_type in ('tcp',):
+ if ':' in wildcard:
+ iface, star = string.split(wildcard, ':')
+ local = if2addr(iface)
+ if not local:
+ panic ("unable to determine ip for:", wildcard)
+ else:
+ host = socket.gethostname()
+ local = socket.gethostbyname(host)
elif net_type == 'elan':
- # awk '/NodeId/ { print $2 }' '/proc/elan/device0/position'
+ # awk '/NodeId/ { print $2 }' 'sys_get_elan_position_file()'
+ f = sys_get_elan_position_file()
+ if not f:
+ panic ("unable to determine local Elan ID")
try:
- fp = open('/proc/elan/device0/position', 'r')
+ fp = open(f, 'r')
lines = fp.readlines()
fp.close()
for l in lines:
a = string.split(l)
if a[0] == 'NodeId':
- local = a[1]
+ elan_id = a[1]
break
+ try:
+ nid = my_int(cluster_id) + my_int(elan_id)
+ local = "%d" % (nid)
+ except ValueError, e:
+ local = elan_id
except IOError, e:
log(e)
elif net_type == 'gm':
fixme("automatic local address for GM")
+
return local
-
-
-# ============================================================
-# Classes to prepare and cleanup the various objects
-#
-class Module:
- """ Base class for the rest of the modules. The default cleanup method is
- defined here, as well as some utilitiy funcs.
- """
- def __init__(self, module_name, dom_node):
- self.dom_node = dom_node
- self.module_name = module_name
- self.name = get_attr(dom_node, 'name')
- self.uuid = get_attr(dom_node, 'uuid')
- self.kmodule_list = []
- self._server = None
- self._connected = 0
+def sys_get_branch():
+ """Returns kernel release"""
+ try:
+ fp = open('/proc/sys/kernel/osrelease')
+ lines = fp.readlines()
+ fp.close()
- def info(self, *args):
- msg = string.join(map(str,args))
- print self.module_name + ":", self.name, self.uuid, msg
-
-
- def lookup_server(self, srv_uuid):
- """ Lookup a server's network information """
- net = get_ost_net(self.dom_node.parentNode, srv_uuid)
- self._server = Network(net)
-
- def get_server(self):
- return self._server
-
- def cleanup(self):
- """ default cleanup, used for most modules """
- self.info()
- srv = self.get_server()
- if srv:
- try:
- lctl.disconnect(srv.net_type, srv.nid, srv.port, srv.uuid)
- except CommandError, e:
- log(self.module_name, "disconnect failed: ", self.name)
- e.dump()
- try:
- lctl.cleanup(self.name, self.uuid)
- except CommandError, e:
- log(self.module_name, "cleanup failed: ", self.name)
- e.dump()
+ for l in lines:
+ version = string.split(l)
+ a = string.split(version[0], '.')
+ return a[0] + '.' + a[1]
+ except IOError, e:
+ log(e)
+ return ""
- def add_module(self, modname):
- """Append a module to list of modules to load."""
- self.kmodule_list.append(modname)
- def mod_loaded(self, modname):
- """Check if a module is already loaded. Look in /proc/modules for it."""
+def mod_loaded(modname):
+ """Check if a module is already loaded. Look in /proc/modules for it."""
+ try:
fp = open('/proc/modules')
lines = fp.readlines()
fp.close()
ret = filter(lambda word, mod=modname: word == mod,
map(lambda line: string.split(line)[0], lines))
return ret
+ except Exception, e:
+ return 0
+
+# XXX: instead of device_list, ask for $name and see what we get
+def is_prepared(name):
+ """Return true if a device exists for the name"""
+ if config.lctl_dump:
+ return 0
+ if (config.noexec or config.record) and config.cleanup:
+ return 1
+ try:
+ # expect this format:
+ # 1 UP ldlm ldlm ldlm_UUID 2
+ out = lctl.device_list()
+ for s in out:
+ if name == string.split(s)[3]:
+ return 1
+ except CommandError, e:
+ e.dump()
+ return 0
+
+def is_network_prepared():
+ """If the any device exists, then assume that all networking
+ has been configured"""
+ out = lctl.device_list()
+ return len(out) > 0
+
+def fs_is_mounted(path):
+ """Return true if path is a mounted lustre filesystem"""
+ try:
+ fp = open('/proc/mounts')
+ lines = fp.readlines()
+ fp.close()
+ for l in lines:
+ a = string.split(l)
+ if a[1] == path and a[2] == 'lustre_lite':
+ return 1
+ except IOError, e:
+ log(e)
+ return 0
+
+
+class kmod:
+ """Manage kernel modules"""
+ def __init__(self, lustre_dir, portals_dir):
+ self.lustre_dir = lustre_dir
+ self.portals_dir = portals_dir
+ self.kmodule_list = []
+
+ def add_portals_module(self, dev_dir, modname):
+ """Append a module to list of modules to load."""
+ self.kmodule_list.append((self.portals_dir, dev_dir, modname))
+
+ def add_lustre_module(self, dev_dir, modname):
+ """Append a module to list of modules to load."""
+ self.kmodule_list.append((self.lustre_dir, dev_dir, modname))
def load_module(self):
"""Load all the modules in the list in the order they appear."""
- for mod in self.kmodule_list:
- # (rc, out) = run ('/sbin/lsmod | grep -s', mod)
- if self.mod_loaded(mod) and not config.noexec():
+ for src_dir, dev_dir, mod in self.kmodule_list:
+ if mod_loaded(mod) and not config.noexec:
continue
- log ('loading module:', mod)
- if config.src_dir():
- module = find_module(config.src_dir(), mod)
+ log ('loading module:', mod, 'srcdir', src_dir, 'devdir', dev_dir)
+ if src_dir:
+ module = find_module(src_dir, dev_dir, mod)
if not module:
panic('module not found:', mod)
(rc, out) = run('/sbin/insmod', module)
(rc, out) = run('/sbin/modprobe', mod)
if rc:
raise CommandError('modprobe', out, rc)
-
+
def cleanup_module(self):
"""Unload the modules in the list in reverse order."""
rev = self.kmodule_list
rev.reverse()
- for mod in rev:
- if not self.mod_loaded(mod):
+ for src_dir, dev_dir, mod in rev:
+ if not mod_loaded(mod) and not config.noexec:
continue
+ # debug hack
+ if mod == 'portals' and config.dump:
+ lctl.dump(config.dump)
log('unloading module:', mod)
- if config.noexec():
- continue
(rc, out) = run('/sbin/rmmod', mod)
if rc:
log('! unable to unload module:', mod)
logall(out)
-
-class Network(Module):
- def __init__(self,dom_node):
- Module.__init__(self, 'NETWORK', dom_node)
- self.net_type = get_attr(dom_node,'type')
- self.nid = get_text(dom_node, 'server', '*')
- self.port = get_text_int(dom_node, 'port', 0)
- self.send_mem = get_text_int(dom_node, 'send_mem', 65536)
- self.recv_mem = get_text_int(dom_node, 'recv_mem', 65536)
- if self.nid == '*':
- self.nid = get_local_address(self.net_type)
- if not self.nid:
- panic("unable to set nid for", self.net_type)
-
- self.add_module('portals')
- if node_needs_router():
- self.add_module('kptlrouter')
- if self.net_type == 'tcp':
- self.add_module('ksocknal')
- if self.net_type == 'elan':
- self.add_module('kqswnal')
- if self.net_type == 'gm':
- self.add_module('kgmnal')
- self.add_module('obdclass')
- self.add_module('ptlrpc')
+# ============================================================
+# Classes to prepare and cleanup the various objects
+#
+class Module:
+ """ Base class for the rest of the modules. The default cleanup method is
+ defined here, as well as some utilitiy funcs.
+ """
+ def __init__(self, module_name, db):
+ self.db = db
+ self.module_name = module_name
+ self.name = self.db.getName()
+ self.uuid = self.db.getUUID()
+ self._server = None
+ self._connected = 0
+ self.kmod = kmod(config.lustre, config.portals)
- def prepare(self):
- self.info(self.net_type, self.nid, self.port)
- if self.net_type == 'tcp':
- ret = run_daemon(TCP_ACCEPTOR, '-s', self.send_mem, '-r', self.recv_mem, self.port)
- if ret:
- raise CommandError(TCP_ACCEPTOR, 'failed', ret)
- ret = self.dom_node.getElementsByTagName('route_tbl')
- for a in ret:
- for r in a.getElementsByTagName('route'):
- lo = get_attr(r, 'lo')
- hi = get_attr(r,'hi', '')
- lctl.add_route(self.net_type, self.nid, lo, hi)
- if self.net_type == 'tcp' and hi == '':
- srv = nid2server(self.dom_node.parentNode.parentNode, lo)
- if not srv:
- panic("no server for nid", lo)
- else:
- lctl.connect(srv.net_type, srv.nid, srv.port, srv.uuid, srv.send_mem, srv.recv_mem)
-
-
- lctl.network(self.net_type, self.nid)
- lctl.newdev(attach = "ptlrpc RPCDEV")
+ def info(self, *args):
+ msg = string.join(map(str,args))
+ print self.module_name + ":", self.name, self.uuid, msg
def cleanup(self):
- self.info(self.net_type, self.nid, self.port)
- ret = self.dom_node.getElementsByTagName('route_tbl')
- for a in ret:
- for r in a.getElementsByTagName('route'):
- lo = get_attr(r, 'lo')
- hi = get_attr(r,'hi', '')
- if self.net_type == 'tcp' and hi == '':
- srv = nid2server(self.dom_node.parentNode.parentNode, lo)
- if not srv:
- panic("no server for nid", lo)
- else:
- lctl.disconnect(srv.net_type, srv.nid, srv.port, srv.uuid)
- lctl.del_route(self.net_type, self.nid, lo, hi)
- try:
- lctl.cleanup("RPCDEV", "")
- except CommandError, e:
- print "cleanup failed: ", self.name
- e.dump()
+ """ default cleanup, used for most modules """
+ self.info()
try:
- lctl.disconnectAll(self.net_type)
+ lctl.cleanup(self.name, self.uuid, config.force)
except CommandError, e:
- print "disconnectAll failed: ", self.name
+ log(self.module_name, "cleanup failed: ", self.name)
e.dump()
+ cleanup_error(e.rc)
+
+ def add_portals_module(self, dev_dir, modname):
+ """Append a module to list of modules to load."""
+ self.kmod.add_portals_module(dev_dir, modname)
+
+ def add_lustre_module(self, dev_dir, modname):
+ """Append a module to list of modules to load."""
+ self.kmod.add_lustre_module(dev_dir, modname)
+
+ def load_module(self):
+ """Load all the modules in the list in the order they appear."""
+ self.kmod.load_module()
+
+ def cleanup_module(self):
+ """Unload the modules in the list in reverse order."""
+ if self.safe_to_clean():
+ self.kmod.cleanup_module()
+
+ def safe_to_clean(self):
+ return 1
+
+ def safe_to_clean_modules(self):
+ return self.safe_to_clean()
+
+class Network(Module):
+ def __init__(self,db):
+ Module.__init__(self, 'NETWORK', db)
+ self.net_type = self.db.get_val('nettype')
+ self.nid = self.db.get_val('nid', '*')
+ self.cluster_id = self.db.get_val('clusterid', "0")
+ self.port = self.db.get_val_int('port', 0)
+ self.send_mem = self.db.get_val_int('sendmem', DEFAULT_TCPBUF)
+ self.recv_mem = self.db.get_val_int('recvmem', DEFAULT_TCPBUF)
+ self.irq_affinity = self.db.get_val_int('irqaffinity', 0)
+
+ if '*' in self.nid:
+ self.nid = sys_get_local_nid(self.net_type, self.nid, self.cluster_id)
+ if not self.nid:
+ panic("unable to set nid for", self.net_type, self.nid, cluster_id)
+ self.generic_nid = 1
+ debug("nid:", self.nid)
+ else:
+ self.generic_nid = 0
+
+ self.nid_uuid = self.nid_to_uuid(self.nid)
+
+ self.hostaddr = self.db.get_val('hostaddr', self.nid)
+ if '*' in self.hostaddr:
+ self.hostaddr = sys_get_local_address(self.net_type, self.hostaddr, self.cluster_id)
+ if not self.hostaddr:
+ panic("unable to set hostaddr for", self.net_type, self.hostaddr, self.cluster_id)
+ debug("hostaddr:", self.hostaddr)
+
+ self.add_portals_module("libcfs", 'libcfs')
+ self.add_portals_module("portals", 'portals')
+ if node_needs_router():
+ self.add_portals_module("router", 'kptlrouter')
if self.net_type == 'tcp':
- # yikes, this ugly! need to save pid in /var/something
- run("killall acceptor")
+ self.add_portals_module("knals/socknal", 'ksocknal')
+ if self.net_type == 'elan':
+ self.add_portals_module("knals/qswnal", 'kqswnal')
+ if self.net_type == 'gm':
+ self.add_portals_module("knals/gmnal", 'kgmnal')
+
+ def nid_to_uuid(self, nid):
+ return "NID_%s_UUID" %(nid,)
-class LDLM(Module):
- def __init__(self,dom_node):
- Module.__init__(self, 'LDLM', dom_node)
- self.add_module('ldlm')
def prepare(self):
+ if not config.record and is_network_prepared():
+ return
+ self.info(self.net_type, self.nid, self.port)
+ if not (config.record and self.generic_nid):
+ lctl.network(self.net_type, self.nid)
+ if self.net_type == 'tcp':
+ sys_tweak_socknal()
+ if self.net_type == 'elan':
+ sys_optimize_elan()
+ if self.port and node_is_router():
+ run_one_acceptor(self.port)
+ self.connect_peer_gateways()
+
+ def connect_peer_gateways(self):
+ for router in self.db.lookup_class('node'):
+ if router.get_val_int('router', 0):
+ for netuuid in router.get_networks():
+ net = self.db.lookup(netuuid)
+ gw = Network(net)
+ if (gw.cluster_id == self.cluster_id and
+ gw.net_type == self.net_type):
+ if gw.nid != self.nid:
+ lctl.connect(gw)
+
+ def disconnect_peer_gateways(self):
+ for router in self.db.lookup_class('node'):
+ if router.get_val_int('router', 0):
+ for netuuid in router.get_networks():
+ net = self.db.lookup(netuuid)
+ gw = Network(net)
+ if (gw.cluster_id == self.cluster_id and
+ gw.net_type == self.net_type):
+ if gw.nid != self.nid:
+ try:
+ lctl.disconnect(gw)
+ except CommandError, e:
+ print "disconnect failed: ", self.name
+ e.dump()
+ cleanup_error(e.rc)
+
+ def safe_to_clean(self):
+ return not is_network_prepared()
+
+ def cleanup(self):
+ self.info(self.net_type, self.nid, self.port)
+ if self.port:
+ stop_acceptor(self.port)
+ if node_is_router():
+ self.disconnect_peer_gateways()
+
+ def correct_level(self, level, op=None):
+ return level
+
+class RouteTable(Module):
+ def __init__(self,db):
+ Module.__init__(self, 'ROUTES', db)
+
+ def server_for_route(self, net_type, gw, gw_cluster_id, tgt_cluster_id,
+ lo, hi):
+ # only setup connections for tcp NALs
+ srvdb = None
+ if not net_type in ('tcp',):
+ return None
+
+ # connect to target if route is to single node and this node is the gw
+ if lo == hi and local_interface(net_type, gw_cluster_id, gw):
+ if not local_cluster(net_type, tgt_cluster_id):
+ panic("target", lo, " not on the local cluster")
+ srvdb = self.db.nid2server(lo, net_type, gw_cluster_id)
+ # connect to gateway if this node is not the gw
+ elif (local_cluster(net_type, gw_cluster_id)
+ and not local_interface(net_type, gw_cluster_id, gw)):
+ srvdb = self.db.nid2server(gw, net_type, gw_cluster_id)
+ else:
+ return None
+
+ if not srvdb:
+ panic("no server for nid", lo)
+ return None
+
+ return Network(srvdb)
+
+ def prepare(self):
+ if not config.record and is_network_prepared():
+ return
self.info()
- lctl.newdev(attach="ldlm %s %s" % (self.name, self.uuid),
- setup ="")
+ for net_type, gw, gw_cluster_id, tgt_cluster_id, lo, hi in self.db.get_route_tbl():
+ lctl.add_route(net_type, gw, lo, hi)
+ srv = self.server_for_route(net_type, gw, gw_cluster_id, tgt_cluster_id, lo, hi)
+ if srv:
+ lctl.connect(srv)
-class LOV(Module):
- def __init__(self,dom_node):
- Module.__init__(self, 'LOV', dom_node)
- self.stripe_sz = get_attr_int(dom_node, 'stripesize', 65536)
- self.stripe_off = get_attr_int(dom_node, 'stripeoffset', 0)
- self.pattern = get_attr_int(dom_node, 'pattern', 0)
- self.mdsuuid = get_first_ref(dom_node, 'mds')
- mds= lookup(dom_node.parentNode, self.mdsuuid)
- self.mdsname = getName(mds)
- self.devlist = get_all_refs(dom_node, 'osc')
- self.stripe_cnt = len(self.devlist)
+ def safe_to_clean(self):
+ return not is_network_prepared()
+
+ def cleanup(self):
+ if is_network_prepared():
+ # the network is still being used, don't clean it up
+ return
+ for net_type, gw, gw_cluster_id, tgt_cluster_id, lo, hi in self.db.get_route_tbl():
+ srv = self.server_for_route(net_type, gw, gw_cluster_id, tgt_cluster_id, lo, hi)
+ if srv:
+ try:
+ lctl.disconnect(srv)
+ except CommandError, e:
+ print "disconnect failed: ", self.name
+ e.dump()
+ cleanup_error(e.rc)
+
+ try:
+ lctl.del_route(net_type, gw, lo, hi)
+ except CommandError, e:
+ print "del_route failed: ", self.name
+ e.dump()
+ cleanup_error(e.rc)
+
+class Management(Module):
+ def __init__(self, db):
+ Module.__init__(self, 'MGMT', db)
+ self.add_lustre_module('lvfs', 'lvfs')
+ self.add_lustre_module('obdclass', 'obdclass')
+ self.add_lustre_module('ptlrpc', 'ptlrpc')
+ self.add_lustre_module('mgmt', 'mgmt_svc')
def prepare(self):
- self.info(self.mdsuuid, self.stripe_cnt, self.stripe_sz, self.stripe_off, self.pattern,
- self.devlist, self.mdsname)
- lctl.lovconfig(self.uuid, self.mdsname, self.stripe_cnt,
- self.stripe_sz, self.stripe_off, self.pattern,
- string.join(self.devlist))
+ if not config.record and is_prepared(self.name):
+ return
+ self.info()
+ lctl.newdev("mgmt", self.name, self.uuid)
+ def safe_to_clean(self):
+ return 1
-class MDS(Module):
- def __init__(self,dom_node):
- Module.__init__(self, 'MDS', dom_node)
- self.devname, self.size = get_device(dom_node)
- self.fstype = get_text(dom_node, 'fstype')
- self.format = get_text(dom_node, 'autoformat', "no")
- if self.fstype == 'extN':
- self.add_module('extN')
- self.add_module('mds')
- self.add_module('mds_%s' % (self.fstype))
+ def cleanup(self):
+ if is_prepared(self.name):
+ Module.cleanup(self)
+
+ def correct_level(self, level, op=None):
+ return level
+
+# This is only needed to load the modules; the LDLM device
+# is now created automatically.
+class LDLM(Module):
+ def __init__(self,db):
+ Module.__init__(self, 'LDLM', db)
+ self.add_lustre_module('lvfs', 'lvfs')
+ self.add_lustre_module('obdclass', 'obdclass')
+ self.add_lustre_module('ptlrpc', 'ptlrpc')
+
+ def prepare(self):
+ return
+
+ def cleanup(self):
+ return
+
+ def correct_level(self, level, op=None):
+ return level
+
+
+class LOV(Module):
+ def __init__(self, db, uuid, fs_name, name_override = None, config_only = None):
+ Module.__init__(self, 'LOV', db)
+ if name_override != None:
+ self.name = "lov_%s" % name_override
+ self.add_lustre_module('lov', 'lov')
+ self.mds_uuid = self.db.get_first_ref('mds')
+ self.stripe_sz = self.db.get_val_int('stripesize', 1048576)
+ self.stripe_off = self.db.get_val_int('stripeoffset', 0)
+ self.pattern = self.db.get_val_int('stripepattern', 0)
+ self.devlist = self.db.get_lov_tgts('lov_tgt')
+ self.stripe_cnt = self.db.get_val_int('stripecount', len(self.devlist))
+ self.osclist = []
+ self.desc_uuid = self.uuid
+ self.uuid = generate_client_uuid(self.name)
+ self.fs_name = fs_name
+ if config_only:
+ self.config_only = 1
+ return
+ self.config_only = None
+ mds = self.db.lookup(self.mds_uuid)
+ self.mds_name = mds.getName()
+ for (obd_uuid, index, gen, active) in self.devlist:
+ if obd_uuid == '':
+ continue
+ obd = self.db.lookup(obd_uuid)
+ osc = get_osc(obd, self.uuid, fs_name)
+ if osc:
+ self.osclist.append((osc, index, gen, active))
+ else:
+ panic('osc not found:', obd_uuid)
+ def get_uuid(self):
+ return self.uuid
+ def get_name(self):
+ return self.name
+ def prepare(self):
+ if not config.record and is_prepared(self.name):
+ return
+ self.info(self.mds_uuid, self.stripe_cnt, self.stripe_sz,
+ self.stripe_off, self.pattern, self.devlist,
+ self.mds_name)
+ lctl.lov_setup(self.name, self.uuid, self.desc_uuid, self.stripe_cnt,
+ self.stripe_sz, self.stripe_off, self.pattern)
+ for (osc, index, gen, active) in self.osclist:
+ target_uuid = osc.target_uuid
+ try:
+ # Only ignore connect failures with --force, which
+ # isn't implemented here yet.
+ osc.active = active
+ osc.prepare(ignore_connect_failure=0)
+ except CommandError, e:
+ print "Error preparing OSC %s\n" % osc.uuid
+ raise e
+ lctl.lov_add_obd(self.name, self.uuid, target_uuid, index, gen)
+
+ def cleanup(self):
+ for (osc, index, gen, active) in self.osclist:
+ target_uuid = osc.target_uuid
+ osc.cleanup()
+ if is_prepared(self.name):
+ Module.cleanup(self)
+ if self.config_only:
+ panic("Can't clean up config_only LOV ", self.name)
+
+ def load_module(self):
+ if self.config_only:
+ panic("Can't load modules for config_only LOV ", self.name)
+ for (osc, index, gen, active) in self.osclist:
+ osc.load_module()
+ break
+ Module.load_module(self)
+
+ def cleanup_module(self):
+ if self.config_only:
+ panic("Can't cleanup modules for config_only LOV ", self.name)
+ Module.cleanup_module(self)
+ for (osc, index, gen, active) in self.osclist:
+ if active:
+ osc.cleanup_module()
+ break
+
+ def correct_level(self, level, op=None):
+ return level
+
+class LMV(Module):
+ def __init__(self, db, uuid, fs_name, name_override = None):
+ Module.__init__(self, 'LMV', db)
+ if name_override != None:
+ self.name = "lmv_%s" % name_override
+ self.add_lustre_module('lmv', 'lmv')
+ self.devlist = self.db.get_refs('mds')
+ self.mdclist = []
+ self.desc_uuid = self.uuid
+ self.uuid = uuid
+ self.fs_name = fs_name
+ for mds_uuid in self.devlist:
+ mds = self.db.lookup(mds_uuid)
+ if not mds:
+ panic("MDS not found!")
+ mdc = MDC(mds, self.uuid, fs_name)
+ if mdc:
+ self.mdclist.append(mdc)
+ else:
+ panic('mdc not found:', mds_uuid)
def prepare(self):
- self.info(self.devname, self.fstype, self.format)
- blkdev = block_dev(self.devname, self.size, self.fstype, self.format)
- lctl.newdev(attach="mds %s %s" % (self.name, self.uuid),
- setup ="%s %s" %(blkdev, self.fstype))
+ if is_prepared(self.name):
+ return
+ for mdc in self.mdclist:
+ try:
+ # Only ignore connect failures with --force, which
+ # isn't implemented here yet.
+ mdc.prepare(ignore_connect_failure=0)
+ except CommandError, e:
+ print "Error preparing LMV %s\n" % mdc.uuid
+ raise e
+ lctl.lmv_setup(self.name, self.uuid, self.desc_uuid,
+ string.join(self.devlist))
+
def cleanup(self):
- Module.cleanup(self)
- clean_loop(self.devname)
+ for mdc in self.mdclist:
+ mdc.cleanup()
+ if is_prepared(self.name):
+ Module.cleanup(self)
+
+ def load_module(self):
+ for mdc in self.mdclist:
+ mdc.load_module()
+ break
+ Module.load_module(self)
+
+ def cleanup_module(self):
+ Module.cleanup_module(self)
+ for mdc in self.mdclist:
+ mdc.cleanup_module()
+ break
-class MDC(Module):
- def __init__(self,dom_node):
- Module.__init__(self, 'MDC', dom_node)
- self.mds_uuid = get_first_ref(dom_node, 'mds')
- self.lookup_server(self.mds_uuid)
- self.add_module('mdc')
+ def correct_level(self, level, op=None):
+ return level
+
+class MDSDEV(Module):
+ def __init__(self,db):
+ Module.__init__(self, 'MDSDEV', db)
+ self.devpath = self.db.get_val('devpath','')
+ self.backdevpath = self.db.get_val('backdevpath','')
+ self.size = self.db.get_val_int('devsize', 0)
+ self.journal_size = self.db.get_val_int('journalsize', 0)
+ self.fstype = self.db.get_val('fstype', '')
+ self.backfstype = self.db.get_val('backfstype', '')
+ self.nspath = self.db.get_val('nspath', '')
+ self.mkfsoptions = self.db.get_val('mkfsoptions', '')
+ self.mountfsoptions = self.db.get_val('mountfsoptions', '')
+ self.cachetype = self.db.get_val('cachetype', '')
+ # overwrite the orignal MDSDEV name and uuid with the MDS name and uuid
+ target_uuid = self.db.get_first_ref('target')
+ mds = self.db.lookup(target_uuid)
+ self.name = mds.getName()
+ self.filesystem_uuids = mds.get_refs('filesystem')
+ self.lmv_uuid = ''
+ self.lmv = ''
+ self.master_mds = ""
+ if not self.filesystem_uuids:
+ self.lmv_uuid = self.db.get_first_ref('lmv')
+ if not self.lmv_uuid:
+ panic("ALERT: can't find lvm uuid")
+ if self.lmv_uuid:
+ self.lmv = self.db.lookup(self.lmv_uuid)
+ if self.lmv:
+ self.filesystem_uuids = self.lmv.get_refs('filesystem')
+ self.master_mds = self.lmv_uuid
+ # FIXME: if fstype not set, then determine based on kernel version
+ self.format = self.db.get_val('autoformat', "no")
+ if mds.get_val('failover', 0):
+ self.failover_mds = 'f'
+ else:
+ self.failover_mds = 'n'
+ active_uuid = get_active_target(mds)
+ if not active_uuid:
+ panic("No target device found:", target_uuid)
+ if active_uuid == self.uuid:
+ self.active = 1
+ else:
+ self.active = 0
+ if self.active and config.group and config.group != mds.get_val('group'):
+ self.active = 0
+
+ self.inode_size = self.db.get_val_int('inodesize', 0)
+ if self.inode_size == 0:
+ # find the LOV for this MDS
+ lovconfig_uuid = mds.get_first_ref('lovconfig')
+ if not lovconfig_uuid:
+ if not self.lmv_uuid:
+ panic("No LOV found for lovconfig ", lovconfig.name)
+
+ if not self.lmv:
+ panic("No LMV initialized and not lovconfig_uuid found")
+
+ lovconfig_uuid = self.lmv.get_first_ref('lovconfig')
+ lovconfig = self.lmv.lookup(lovconfig_uuid)
+ lov_uuid = lovconfig.get_first_ref('lov')
+ if not lov_uuid:
+ panic("No LOV found for lovconfig ", lovconfig.name)
+ else:
+ lovconfig = mds.lookup(lovconfig_uuid)
+ lov_uuid = lovconfig.get_first_ref('lov')
+ if not lov_uuid:
+ panic("No LOV found for lovconfig ", lovconfig.name)
+
+ if self.lmv:
+ lovconfig_uuid = self.lmv.get_first_ref('lovconfig')
+ lovconfig = self.lmv.lookup(lovconfig_uuid)
+ lov_uuid = lovconfig.get_first_ref('lov')
+
+ lov = LOV(self.db.lookup(lov_uuid), lov_uuid, 'FS_name', config_only = 1)
+
+ # default stripe count controls default inode_size
+ stripe_count = lov.stripe_cnt
+ if stripe_count > 77:
+ self.inode_size = 4096
+ elif stripe_count > 35:
+ self.inode_size = 2048
+ elif stripe_count > 13:
+ self.inode_size = 1024
+ elif stripe_count > 3:
+ self.inode_size = 512
+ else:
+ self.inode_size = 256
+
+ self.target_dev_uuid = self.uuid
+ self.uuid = target_uuid
+ # setup LMV
+ if self.master_mds:
+ client_uuid = generate_client_uuid(self.name)
+ client_uuid = self.name + "_lmv_" + "UUID"
+ self.master = LMV(self.db.lookup(self.lmv_uuid), client_uuid, self.name, self.name)
+ self.master_mds = self.master.name
+
+ # modules
+ self.add_lustre_module('mdc', 'mdc')
+ self.add_lustre_module('osc', 'osc')
+ self.add_lustre_module('lov', 'lov')
+ self.add_lustre_module('lmv', 'lmv')
+ self.add_lustre_module('ost', 'ost')
+ self.add_lustre_module('mds', 'mds')
+
+ if self.fstype == 'smfs':
+ self.add_lustre_module('smfs', 'smfs')
+
+ if self.fstype == 'ldiskfs':
+ self.add_lustre_module('ldiskfs', 'ldiskfs')
+
+ if self.fstype:
+ self.add_lustre_module('lvfs', 'fsfilt_%s' % (self.fstype))
+
+ # if fstype is smfs, then we should also take care about backing
+ # store fs.
+ if self.fstype == 'smfs':
+ self.add_lustre_module('lvfs', 'fsfilt_%s' % (self.backfstype))
+
+ for options in string.split(self.mountfsoptions, ','):
+ if options == 'snap':
+ if not self.fstype == 'smfs':
+ panic("mountoptions with snap, but fstype is not smfs\n")
+ self.add_lustre_module('lvfs', 'fsfilt_snap_%s' % (self.fstype))
+ self.add_lustre_module('lvfs', 'fsfilt_snap_%s' % (self.backfstype))
+ def load_module(self):
+ if self.active:
+ Module.load_module(self)
def prepare(self):
- self.info(self.mds_uuid)
- srv = self.get_server()
- lctl.connect(srv.net_type, srv.nid, srv.port, srv.uuid, srv.send_mem, srv.recv_mem)
- lctl.newdev(attach="mdc %s %s" % (self.name, self.uuid),
- setup ="%s %s" %(self.mds_uuid, srv.uuid))
+ if not config.record and is_prepared(self.name):
+ return
+ if not self.active:
+ debug(self.uuid, "not active")
+ return
+ if config.reformat:
+ # run write_conf automatically, if --reformat used
+ self.write_conf()
+ self.info(self.devpath, self.fstype, self.size, self.format)
+ run_acceptors()
+ # prepare LMV
+ if self.master_mds:
+ self.master.prepare()
+ # never reformat here
+ blkdev = block_dev(self.devpath, self.size, self.fstype, 0,
+ self.format, self.journal_size, self.inode_size,
+ self.mkfsoptions, self.backfstype, self.backdevpath)
+
+ if not is_prepared('MDT'):
+ lctl.newdev("mdt", 'MDT', 'MDT_UUID', setup ="")
+ try:
+ mountfsoptions = def_mount_options(self.fstype, 'mds')
+
+ if config.mountfsoptions:
+ if mountfsoptions:
+ mountfsoptions = mountfsoptions + ',' + config.mountfsoptions
+ else:
+ mountfsoptions = config.mountfsoptions
+ if self.mountfsoptions:
+ mountfsoptions = mountfsoptions + ',' + self.mountfsoptions
+ else:
+ if self.mountfsoptions:
+ if mountfsoptions:
+ mountfsoptions = mountfsoptions + ',' + self.mountfsoptions
+ else:
+ mountfsoptions = self.mountfsoptions
-class OBD(Module):
- def __init__(self, dom_node):
- Module.__init__(self, 'OBD', dom_node)
- self.obdtype = get_attr(dom_node, 'type')
- self.devname, self.size = get_device(dom_node)
- self.fstype = get_text(dom_node, 'fstype')
- self.format = get_text(dom_node, 'autoformat', 'yes')
- if self.fstype == 'extN':
- self.add_module('extN')
- self.add_module(self.obdtype)
+ if self.fstype == 'smfs':
+ realdev = self.fstype
+
+ if mountfsoptions:
+ mountfsoptions = "%s,type=%s,dev=%s" % (mountfsoptions,
+ self.backfstype,
+ blkdev)
+ else:
+ mountfsoptions = "type=%s,dev=%s" % (self.backfstype,
+ blkdev)
+ else:
+ realdev = blkdev
+
+ print 'MDS mount options: ' + mountfsoptions
+
+ if not self.master_mds:
+ self.master_mds = 'dumb'
+ if not self.cachetype:
+ self.cachetype = 'dumb'
+ lctl.newdev("mds", self.name, self.uuid,
+ setup ="%s %s %s %s %s %s" %(realdev, self.fstype,
+ self.name, mountfsoptions,
+ self.master_mds, self.cachetype))
+ except CommandError, e:
+ if e.rc == 2:
+ panic("MDS is missing the config log. Need to run " +
+ "lconf --write_conf.")
+ else:
+ raise e
+
+ def write_conf(self):
+ do_cleanup = 0
+ if not is_prepared(self.name):
+ self.info(self.devpath, self.fstype, self.format)
+
+ blkdev = block_dev(self.devpath, self.size, self.fstype,
+ config.reformat, self.format, self.journal_size,
+ self.inode_size, self.mkfsoptions,
+ self.backfstype, self.backdevpath)
+
+ # Even for writing logs we mount mds with supplied mount options
+ # because it will not mount smfs (if used) otherwise.
+
+ mountfsoptions = def_mount_options(self.fstype, 'mds')
+
+ if config.mountfsoptions:
+ if mountfsoptions:
+ mountfsoptions = mountfsoptions + ',' + config.mountfsoptions
+ else:
+ mountfsoptions = config.mountfsoptions
+ if self.mountfsoptions:
+ mountfsoptions = mountfsoptions + ',' + self.mountfsoptions
+ else:
+ if self.mountfsoptions:
+ if mountfsoptions:
+ mountfsoptions = mountfsoptions + ',' + self.mountfsoptions
+ else:
+ mountfsoptions = self.mountfsoptions
+
+ if self.fstype == 'smfs':
+ realdev = self.fstype
+
+ if mountfsoptions:
+ mountfsoptions = "%s,type=%s,dev=%s" % (mountfsoptions,
+ self.backfstype,
+ blkdev)
+ else:
+ mountfsoptions = "type=%s,dev=%s" % (self.backfstype,
+ blkdev)
+ else:
+ realdev = blkdev
+
+ print 'MDS mount options: ' + mountfsoptions
+
+ # As mount options are passed by 4th param to config tool, we need
+ # to pass something in 3rd param. But we do not want this 3rd param
+ # be counted as a profile name for reading log on MDS setup, thus,
+ # we pass there some predefined sign like 'dumb', which will be
+ # checked in MDS code and skipped. Probably there is more nice way
+ # like pass empty string and check it in config tool and pass null
+ # as 4th param.
+ lctl.newdev("mds", self.name, self.uuid,
+ setup ="%s %s %s %s" %(realdev, self.fstype,
+ 'dumb', mountfsoptions))
+ do_cleanup = 1
+
+ # record logs for the MDS lov
+ for uuid in self.filesystem_uuids:
+ log("recording clients for filesystem:", uuid)
+ fs = self.db.lookup(uuid)
+
+ # this is ugly, should be organized nice later.
+ target_uuid = self.db.get_first_ref('target')
+ mds = self.db.lookup(target_uuid)
+
+ lovconfig_uuid = mds.get_first_ref('lovconfig')
+ if lovconfig_uuid:
+ lovconfig = mds.lookup(lovconfig_uuid)
+ obd_uuid = lovconfig.get_first_ref('lov')
+ else:
+ obd_uuid = fs.get_first_ref('obd')
+
+ client_uuid = generate_client_uuid(self.name)
+ client = VOSC(self.db.lookup(obd_uuid), client_uuid, self.name,
+ self.name)
+ config.record = 1
+ lctl.clear_log(self.name, self.name)
+ lctl.record(self.name, self.name)
+ client.prepare()
+ lctl.mount_option(self.name, client.get_name(), "")
+ lctl.end_record()
+ process_updates(self.db, self.name, self.name, client)
+
+ config.cleanup = 1
+ lctl.clear_log(self.name, self.name + '-clean')
+ lctl.record(self.name, self.name + '-clean')
+ client.cleanup()
+ lctl.del_mount_option(self.name)
+ lctl.end_record()
+ process_updates(self.db, self.name, self.name + '-clean', client)
+ config.cleanup = 0
+ config.record = 0
+
+ # record logs for each client
+ if config.noexec:
+ noexec_opt = '-n'
+ else:
+ noexec_opt = ''
+ if config.ldapurl:
+ config_options = "--ldapurl " + config.ldapurl + " --config " + config.config
+ else:
+ config_options = CONFIG_FILE
+
+ for node_db in self.db.lookup_class('node'):
+ client_name = node_db.getName()
+ for prof_uuid in node_db.get_refs('profile'):
+ prof_db = node_db.lookup(prof_uuid)
+ # refactor this into a funtion to test "clientness"
+ # of a node.
+ for ref_class, ref_uuid in prof_db.get_all_refs():
+ if ref_class in ('mountpoint','echoclient'):
+ debug("recording", client_name)
+ old_noexec = config.noexec
+ config.noexec = 0
+ ret, out = run (sys.argv[0], noexec_opt,
+ " -v --record --nomod",
+ "--record_log", client_name,
+ "--record_device", self.name,
+ "--node", client_name,
+ config_options)
+ if config.verbose:
+ for s in out: log("record> ", string.strip(s))
+ ret, out = run (sys.argv[0], noexec_opt,
+ "--cleanup -v --record --nomod",
+ "--record_log", client_name + "-clean",
+ "--record_device", self.name,
+ "--node", client_name,
+ config_options)
+ if config.verbose:
+ for s in out: log("record> ", string.strip(s))
+ config.noexec = old_noexec
+ if do_cleanup:
+ try:
+ lctl.cleanup(self.name, self.uuid, 0, 0)
+ except CommandError, e:
+ log(self.module_name, "cleanup failed: ", self.name)
+ e.dump()
+ cleanup_error(e.rc)
+ Module.cleanup(self)
+
+ if self.fstype == 'smfs':
+ clean_loop(self.backdevpath)
+ else:
+ clean_loop(self.devpath)
+
+ def msd_remaining(self):
+ out = lctl.device_list()
+ for s in out:
+ if string.split(s)[2] in ('mds',):
+ return 1
+
+ def safe_to_clean(self):
+ return self.active
+
+ def safe_to_clean_modules(self):
+ return not self.msd_remaining()
+
+ def cleanup(self):
+ if not self.active:
+ debug(self.uuid, "not active")
+ return
+ self.info()
+ if is_prepared(self.name):
+ try:
+ lctl.cleanup(self.name, self.uuid, config.force,
+ config.failover)
+ except CommandError, e:
+ log(self.module_name, "cleanup failed: ", self.name)
+ e.dump()
+ cleanup_error(e.rc)
+ Module.cleanup(self)
+ # cleanup LMV
+ if self.master_mds:
+ self.master.cleanup()
+ if not self.msd_remaining() and is_prepared('MDT'):
+ try:
+ lctl.cleanup("MDT", "MDT_UUID", config.force,
+ config.failover)
+ except CommandError, e:
+ print "cleanup failed: ", self.name
+ e.dump()
+ cleanup_error(e.rc)
+
+ if self.fstype == 'smfs':
+ clean_loop(self.backdevpath)
+ else:
+ clean_loop(self.devpath)
+
+ def correct_level(self, level, op=None):
+ #if self.master_mds:
+ # level = level + 2
+ return level
+
+class OSD(Module):
+ def __init__(self, db):
+ Module.__init__(self, 'OSD', db)
+ self.osdtype = self.db.get_val('osdtype')
+ self.devpath = self.db.get_val('devpath', '')
+ self.backdevpath = self.db.get_val('backdevpath', '')
+ self.size = self.db.get_val_int('devsize', 0)
+ self.journal_size = self.db.get_val_int('journalsize', 0)
+ self.inode_size = self.db.get_val_int('inodesize', 0)
+ self.mkfsoptions = self.db.get_val('mkfsoptions', '')
+ self.mountfsoptions = self.db.get_val('mountfsoptions', '')
+ self.fstype = self.db.get_val('fstype', '')
+ self.backfstype = self.db.get_val('backfstype', '')
+ self.nspath = self.db.get_val('nspath', '')
+ target_uuid = self.db.get_first_ref('target')
+ ost = self.db.lookup(target_uuid)
+ self.name = ost.getName()
+ self.format = self.db.get_val('autoformat', 'yes')
+ if ost.get_val('failover', 0):
+ self.failover_ost = 'f'
+ else:
+ self.failover_ost = 'n'
+
+ active_uuid = get_active_target(ost)
+ if not active_uuid:
+ panic("No target device found:", target_uuid)
+ if active_uuid == self.uuid:
+ self.active = 1
+ else:
+ self.active = 0
+ if self.active and config.group and config.group != ost.get_val('group'):
+ self.active = 0
+
+ self.target_dev_uuid = self.uuid
+ self.uuid = target_uuid
+ # modules
+ self.add_lustre_module('ost', 'ost')
+ if self.fstype == 'smfs':
+ self.add_lustre_module('smfs', 'smfs')
+ # FIXME: should we default to ext3 here?
+ if self.fstype == 'ldiskfs':
+ self.add_lustre_module('ldiskfs', 'ldiskfs')
+ if self.fstype:
+ self.add_lustre_module('lvfs' , 'fsfilt_%s' % (self.fstype))
+ if self.fstype == 'smfs':
+ self.add_lustre_module('lvfs' , 'fsfilt_%s' % (self.backfstype))
+
+ for options in self.mountfsoptions:
+ if options == 'snap':
+ if not self.fstype == 'smfs':
+ panic("mountoptions with snap, but fstype is not smfs\n")
+ self.add_lustre_module('lvfs', 'fsfilt_snap_%s' % (self.fstype))
+ self.add_lustre_module('lvfs', 'fsfilt_snap_%s' % (self.backfstype))
+
+ self.add_lustre_module(self.osdtype, self.osdtype)
+
+ def load_module(self):
+ if self.active:
+ Module.load_module(self)
# need to check /proc/mounts and /etc/mtab before
# formatting anything.
# FIXME: check if device is already formatted.
def prepare(self):
- self.info(self.obdtype, self.devname, self.size, self.fstype, self.format)
- if self.obdtype == 'obdecho':
+ if is_prepared(self.name):
+ return
+ if not self.active:
+ debug(self.uuid, "not active")
+ return
+ self.info(self.osdtype, self.devpath, self.size, self.fstype,
+ self.format, self.journal_size, self.inode_size)
+ run_acceptors()
+ if self.osdtype == 'obdecho':
blkdev = ''
else:
- blkdev = block_dev(self.devname, self.size, self.fstype, self.format)
- lctl.newdev(attach="%s %s %s" % (self.obdtype, self.name, self.uuid),
- setup ="%s %s" %(blkdev, self.fstype))
+ blkdev = block_dev(self.devpath, self.size, self.fstype,
+ config.reformat, self.format, self.journal_size,
+ self.inode_size, self.mkfsoptions, self.backfstype,
+ self.backdevpath)
+
+ mountfsoptions = def_mount_options(self.fstype, 'ost')
+
+ if config.mountfsoptions:
+ if mountfsoptions:
+ mountfsoptions = mountfsoptions + ',' + config.mountfsoptions
+ else:
+ mountfsoptions = config.mountfsoptions
+ if self.mountfsoptions:
+ mountfsoptions = mountfsoptions + ',' + self.mountfsoptions
+ else:
+ if self.mountfsoptions:
+ if mountfsoptions:
+ mountfsoptions = mountfsoptions + ',' + self.mountfsoptions
+ else:
+ mountfsoptions = self.mountfsoptions
+
+ if self.fstype == 'smfs':
+ realdev = self.fstype
+
+ if mountfsoptions:
+ mountfsoptions = "%s,type=%s,dev=%s" % (mountfsoptions,
+ self.backfstype,
+ blkdev)
+ else:
+ mountfsoptions = "type=%s,dev=%s" % (self.backfstype,
+ blkdev)
+ else:
+ realdev = blkdev
+
+ print 'OSD mount options: ' + mountfsoptions
+
+ lctl.newdev(self.osdtype, self.name, self.uuid,
+ setup ="%s %s %s %s" %(realdev, self.fstype,
+ self.failover_ost,
+ mountfsoptions))
+ if not is_prepared('OSS'):
+ lctl.newdev("ost", 'OSS', 'OSS_UUID', setup ="")
+
+ def osd_remaining(self):
+ out = lctl.device_list()
+ for s in out:
+ if string.split(s)[2] in ('obdfilter', 'obdecho'):
+ return 1
+
+ def safe_to_clean(self):
+ return self.active
+
+ def safe_to_clean_modules(self):
+ return not self.osd_remaining()
+
def cleanup(self):
- Module.cleanup(self)
- if not self.obdtype == 'obdecho':
- clean_loop(self.devname)
+ if not self.active:
+ debug(self.uuid, "not active")
+ return
+ if is_prepared(self.name):
+ self.info()
+ try:
+ lctl.cleanup(self.name, self.uuid, config.force,
+ config.failover)
+ except CommandError, e:
+ log(self.module_name, "cleanup failed: ", self.name)
+ e.dump()
+ cleanup_error(e.rc)
+ if not self.osd_remaining() and is_prepared('OSS'):
+ try:
+ lctl.cleanup("OSS", "OSS_UUID", config.force,
+ config.failover)
+ except CommandError, e:
+ print "cleanup failed: ", self.name
+ e.dump()
+ cleanup_error(e.rc)
+ if not self.osdtype == 'obdecho':
+ if self.fstype == 'smfs':
+ clean_loop(self.backdevpath)
+ else:
+ clean_loop(self.devpath)
+
+ def correct_level(self, level, op=None):
+ return level
+
+def mgmt_uuid_for_fs(mtpt_name):
+ if not mtpt_name:
+ return ''
+ mtpt_db = toplustreDB.lookup_name(mtpt_name)
+ fs_uuid = mtpt_db.get_first_ref('filesystem')
+ fs = toplustreDB.lookup(fs_uuid)
+ if not fs:
+ return ''
+ return fs.get_first_ref('mgmt')
+
+# Generic client module, used by OSC and MDC
+class Client(Module):
+ def __init__(self, tgtdb, uuid, module, fs_name, self_name=None,
+ module_dir=None):
+ self.target_name = tgtdb.getName()
+ self.target_uuid = tgtdb.getUUID()
+ self.db = tgtdb
+ self.active = 1
+
+ self.tgt_dev_uuid = get_active_target(tgtdb)
+ if not self.tgt_dev_uuid:
+ panic("No target device found for target(1):", self.target_name)
+
+ self.kmod = kmod(config.lustre, config.portals)
+ self._server = None
+ self._connected = 0
-class OST(Module):
- def __init__(self,dom_node):
- Module.__init__(self, 'OST', dom_node)
- self.obd_uuid = get_first_ref(dom_node, 'obd')
- self.add_module('ost')
+ self.module = module
+ self.module_name = string.upper(module)
+ if not self_name:
+ self.name = '%s_%s_%s_%s' % (self.module_name, socket.gethostname(),
+ self.target_name, fs_name)
+ else:
+ self.name = self_name
+ self.uuid = uuid
+ self.lookup_server(self.tgt_dev_uuid)
+ mgmt_uuid = mgmt_uuid_for_fs(fs_name)
+ if mgmt_uuid:
+ self.mgmt_name = mgmtcli_name_for_uuid(mgmt_uuid)
+ else:
+ self.mgmt_name = ''
+ self.fs_name = fs_name
+ if not module_dir:
+ module_dir = module
+ self.add_lustre_module(module_dir, module)
+ def lookup_server(self, srv_uuid):
+ """ Lookup a server's network information """
+ self._server_nets = get_ost_net(self.db, srv_uuid)
+ if len(self._server_nets) == 0:
+ panic ("Unable to find a server for:", srv_uuid)
+ def get_name(self):
+ return self.name
+ def get_servers(self):
+ return self._server_nets
+
+ def prepare(self, ignore_connect_failure = 0):
+ self.info(self.target_uuid)
+ if not config.record and is_prepared(self.name):
+ self.cleanup()
+ try:
+ srv = choose_local_server(self.get_servers())
+ if srv:
+ lctl.connect(srv)
+ else:
+ routes = find_route(self.get_servers())
+ if len(routes) == 0:
+ panic ("no route to", self.target_uuid)
+ for (srv, r) in routes:
+ lctl.add_route_host(r[0], srv.nid_uuid, r[1], r[3])
+ except CommandError, e:
+ if not ignore_connect_failure:
+ raise e
+ if srv:
+ if self.permits_inactive() and (self.target_uuid in config.inactive or self.active == 0):
+ debug("%s inactive" % self.target_uuid)
+ inactive_p = "inactive"
+ else:
+ debug("%s active" % self.target_uuid)
+ inactive_p = ""
+ lctl.newdev(self.module, self.name, self.uuid,
+ setup ="%s %s %s %s" % (self.target_uuid, srv.nid_uuid,
+ inactive_p, self.mgmt_name))
+
+ def cleanup(self):
+ if is_prepared(self.name):
+ Module.cleanup(self)
+ try:
+ srv = choose_local_server(self.get_servers())
+ if srv:
+ lctl.disconnect(srv)
+ else:
+ for (srv, r) in find_route(self.get_servers()):
+ lctl.del_route_host(r[0], srv.nid_uuid, r[1], r[3])
+ except CommandError, e:
+ log(self.module_name, "cleanup failed: ", self.name)
+ e.dump()
+ cleanup_error(e.rc)
+
+ def correct_level(self, level, op=None):
+ return level
+
+ def deactivate(self):
+ try:
+ lctl.deactivate(self.name)
+ except CommandError, e:
+ log(self.module_name, "deactivate failed: ", self.name)
+ e.dump()
+ cleanup_error(e.rc)
+
+class MDC(Client):
+ def __init__(self, db, uuid, fs_name):
+ Client.__init__(self, db, uuid, 'mdc', fs_name)
+
+ def permits_inactive(self):
+ return 0
+
+class OSC(Client):
+ def __init__(self, db, uuid, fs_name):
+ Client.__init__(self, db, uuid, 'osc', fs_name)
+
+ def permits_inactive(self):
+ return 1
+
+def mgmtcli_name_for_uuid(uuid):
+ return 'MGMTCLI_%s' % uuid
+
+class ManagementClient(Client):
+ def __init__(self, db, uuid):
+ Client.__init__(self, db, uuid, 'mgmt_cli', '',
+ self_name = mgmtcli_name_for_uuid(db.getUUID()),
+ module_dir = 'mgmt')
+class VLOV(Module):
+ def __init__(self, db, uuid, fs_name, name_override = None, config_only = None):
+ Module.__init__(self, 'VLOV', db)
+ if name_override != None:
+ self.name = "lov_%s" % name_override
+ self.add_lustre_module('lov', 'lov')
+ self.stripe_sz = 65536
+ self.stripe_off = 0
+ self.pattern = 0
+ self.stripe_cnt = 1
+ self.desc_uuid = self.uuid
+ self.uuid = generate_client_uuid(self.name)
+ self.fs_name = fs_name
+ self.osc = get_osc(db, self.uuid, fs_name)
+ if not self.osc:
+ panic('osc not found:', self.uuid)
+ if config_only:
+ self.config_only = 1
+ return
+ self.config_only = None
+ def get_uuid(self):
+ return self.uuid
+ def get_name(self):
+ return self.name
def prepare(self):
- self.info(self.obd_uuid)
- lctl.newdev(attach="ost %s %s" % (self.name, self.uuid),
- setup ="%s" % (self.obd_uuid))
+ if not config.record and is_prepared(self.name):
+ return
+ lctl.lov_setup(self.name, self.uuid, self.desc_uuid, self.stripe_cnt,
+ self.stripe_sz, self.stripe_off, self.pattern)
+ target_uuid = self.osc.target_uuid
+ try:
+ self.osc.active = 1
+ self.osc.prepare(ignore_connect_failure=0)
+ except CommandError, e:
+ print "Error preparing OSC %s\n" % osc.uuid
+ raise e
+ lctl.lov_add_obd(self.name, self.uuid, target_uuid, 0, 1)
+
+ def cleanup(self):
+ target_uuid = self.osc.target_uuid
+ self.osc.cleanup()
+ if is_prepared(self.name):
+ Module.cleanup(self)
+ if self.config_only:
+ panic("Can't clean up config_only LOV ", self.name)
-class OSC(Module):
- def __init__(self,dom_node):
- Module.__init__(self, 'OSC', dom_node)
- self.obd_uuid = get_first_ref(dom_node, 'obd')
- self.ost_uuid = get_first_ref(dom_node, 'ost')
- self.lookup_server(self.ost_uuid)
- self.add_module('osc')
+ def load_module(self):
+ if self.config_only:
+ panic("Can't load modules for config_only LOV ", self.name)
+ self.osc.load_module()
+ Module.load_module(self)
+ def cleanup_module(self):
+ if self.config_only:
+ panic("Can't cleanup modules for config_only LOV ", self.name)
+ Module.cleanup_module(self)
+ self.osc.cleanup_module()
+
+ def correct_level(self, level, op=None):
+ return level
+
+class CMOBD(Module):
+ def __init__(self,db):
+ Module.__init__(self, 'CMOBD', db)
+ self.name = self.db.getName();
+ self.uuid = generate_client_uuid(self.name)
+ self.master_uuid = self.db.get_first_ref('masterobd')
+ self.cache_uuid = self.db.get_first_ref('cacheobd')
+ self.add_lustre_module('cmobd', 'cmobd')
+ master_obd = self.db.lookup(self.master_uuid)
+ if not master_obd:
+ panic('master obd not found:', self.master_uuid)
+ cache_obd = self.db.lookup(self.cache_uuid)
+ if not cache_obd:
+ panic('cache obd not found:', self.cache_uuid)
+
+ if master_obd.get_class() == 'ost':
+ self.client_uuid = generate_client_uuid(self.name)
+ self.master= VLOV(master_obd, self.client_uuid, self.name,
+ "%s_master" % (self.name))
+ self.master_uuid = self.master.get_uuid()
+ else:
+ self.master = get_mdc(db, self.name, self.master_uuid)
+ # need to check /proc/mounts and /etc/mtab before
+ # formatting anything.
+ # FIXME: check if device is already formatted.
def prepare(self):
- self.info(self.obd_uuid, self.ost_uuid)
- srv = self.get_server()
- if local_net(srv):
- lctl.connect(srv.net_type, srv.nid, srv.port, srv.uuid, srv.send_mem, srv.recv_mem)
+ self.master.prepare()
+ if not config.record and is_prepared(self.name):
+ return
+ self.info(self.master_uuid, self.cache_uuid)
+ lctl.newdev("cmobd", self.name, self.uuid,
+ setup ="%s %s" %(self.master_uuid,
+ self.cache_uuid))
+
+ def cleanup(self):
+ if is_prepared(self.name):
+ Module.cleanup(self)
+ self.master.cleanup()
+
+ def load_module(self):
+ self.master.load_module()
+ Module.load_module(self)
+
+ def cleanup_module(self):
+ Module.cleanup_module(self)
+ self.master.cleanup_module()
+
+ def correct_level(self, level, op=None):
+ return level
+
+class COBD(Module):
+ def __init__(self, db, uuid, name, type, name_override = None):
+ Module.__init__(self, 'COBD', db)
+ self.name = self.db.getName();
+ self.uuid = generate_client_uuid(self.name)
+ self.real_uuid = self.db.get_first_ref('realobd')
+ self.cache_uuid = self.db.get_first_ref('cacheobd')
+ self.add_lustre_module('cobd', 'cobd')
+ real_obd = self.db.lookup(self.real_uuid)
+ if not real_obd:
+ panic('real obd not found:', self.real_uuid)
+ cache_obd = self.db.lookup(self.cache_uuid)
+ if not cache_obd:
+ panic('cache obd not found:', self.cache_uuid)
+ if type == 'obd':
+ self.real = LOV(real_obd, self.real_uuid, name,
+ "%s_real" % (self.name));
+ self.cache = LOV(cache_obd, self.cache_uuid, name,
+ "%s_cache" % (self.name));
else:
- r = find_route(srv)
- if r:
- lctl.add_route_host(r[0], srv.uuid, r[1], r[2])
- else:
- panic ("no route to", srv.nid)
-
- lctl.newdev(attach="osc %s %s" % (self.name, self.uuid),
- setup ="%s %s" %(self.obd_uuid, srv.uuid))
+ self.real = get_mdc(db, name, self.real_uuid)
+ self.cache = get_mdc(db, name, self.cache_uuid)
+ # need to check /proc/mounts and /etc/mtab before
+ # formatting anything.
+ # FIXME: check if device is already formatted.
+ def get_uuid(self):
+ return self.uuid
+ def get_name(self):
+ return self.name
+ def get_real_name(self):
+ return self.real.name
+ def get_cache_name(self):
+ return self.cache.name
+ def prepare(self):
+ self.real.prepare()
+ self.cache.prepare()
+ if not config.record and is_prepared(self.name):
+ return
+ self.info(self.real_uuid, self.cache_uuid)
+ lctl.newdev("cobd", self.name, self.uuid,
+ setup ="%s %s" %(self.real.name,
+ self.cache.name))
+ def cleanup(self):
+ if is_prepared(self.name):
+ Module.cleanup(self)
+ self.real.cleanup()
+ self.cache.cleanup()
-class Mountpoint(Module):
- def __init__(self,dom_node):
- Module.__init__(self, 'MTPT', dom_node)
- self.path = get_text(dom_node, 'path')
- self.mdc_uuid = get_first_ref(dom_node, 'mdc')
- self.lov_uuid = get_first_ref(dom_node, 'osc')
- self.add_module('osc')
- # should add lov only if needed
- self.add_module('lov')
- self.add_module('llite')
+ def load_module(self):
+ self.real.load_module()
+ Module.load_module(self)
+ def cleanup_module(self):
+ Module.cleanup_module(self)
+ self.real.cleanup_module()
+
+# virtual interface for OSC and LOV
+class VOSC(Module):
+ def __init__(self, db, client_uuid, name, name_override = None):
+ Module.__init__(self, 'VOSC', db)
+ if db.get_class() == 'lov':
+ self.osc = LOV(db, client_uuid, name, name_override)
+ self.type = 'lov'
+ elif db.get_class() == 'cobd':
+ self.osc = COBD(db, client_uuid, name, 'obd')
+ self.type = 'cobd'
+ else:
+ self.osc = OSC(db, client_uuid, name)
+ self.type = 'osc'
+ def get_uuid(self):
+ return self.osc.get_uuid()
+ def get_name(self):
+ return self.osc.get_name()
def prepare(self):
- l = lookup(self.dom_node.parentNode, self.lov_uuid)
- if l.nodeName == 'lov':
- lov = LOV(l)
- for osc_uuid in lov.devlist:
- osc = lookup(self.dom_node.parentNode, osc_uuid)
- if osc:
- n = OSC(osc)
- n.prepare()
- else:
- panic('osc not found:', osc_uuid)
- lctl.newdev(attach="lov %s %s" % (lov.name, lov.uuid),
- setup ="%s" % (self.mdc_uuid))
+ self.osc.prepare()
+ def cleanup(self):
+ self.osc.cleanup()
+ def load_module(self):
+ self.osc.load_module()
+ def cleanup_module(self):
+ self.osc.cleanup_module()
+ def correct_level(self, level, op=None):
+ return self.osc.correct_level(level, op)
+
+# virtual interface for MDC and LMV
+class VMDC(Module):
+ def __init__(self, db, client_uuid, name, name_override = None):
+ Module.__init__(self, 'VMDC', db)
+ if db.get_class() == 'lmv':
+ self.mdc = LMV(db, client_uuid, name)
+ elif db.get_class() == 'cobd':
+ self.mdc = COBD(db, client_uuid, name, 'mds')
else:
- osc = OSC(l)
- osc.prepare()
+ self.mdc = MDC(db, client_uuid, name)
+ def get_uuid(self):
+ return self.mdc.uuid
+ def get_name(self):
+ return self.mdc.name
+ def prepare(self):
+ self.mdc.prepare()
+ def cleanup(self):
+ self.mdc.cleanup()
+ def load_module(self):
+ self.mdc.load_module()
+ def cleanup_module(self):
+ self.mdc.cleanup_module()
+ def correct_level(self, level, op=None):
+ return self.mdc.correct_level(level, op)
+
+class ECHO_CLIENT(Module):
+ def __init__(self,db):
+ Module.__init__(self, 'ECHO_CLIENT', db)
+ self.add_lustre_module('obdecho', 'obdecho')
+ self.obd_uuid = self.db.get_first_ref('obd')
+ obd = self.db.lookup(self.obd_uuid)
+ self.uuid = generate_client_uuid(self.name)
+ self.osc = VOSC(obd, self.uuid, self.name)
+
+ def prepare(self):
+ if not config.record and is_prepared(self.name):
+ return
+ run_acceptors()
+ self.osc.prepare() # XXX This is so cheating. -p
+ self.info(self.obd_uuid)
+
+ lctl.newdev("echo_client", self.name, self.uuid,
+ setup = self.osc.get_name())
+
+ def cleanup(self):
+ if is_prepared(self.name):
+ Module.cleanup(self)
+ self.osc.cleanup()
+
+ def load_module(self):
+ self.osc.load_module()
+ Module.load_module(self)
+
+ def cleanup_module(self):
+ Module.cleanup_module(self)
+ self.osc.cleanup_module()
+
+ def correct_level(self, level, op=None):
+ return level
+
+def generate_client_uuid(name):
+ client_uuid = '%05x_%.19s_%05x%05x' % (int(random.random() * 1048576),
+ name,
+ int(random.random() * 1048576),
+ int(random.random() * 1048576))
+ return client_uuid[:36]
+
+class Mountpoint(Module):
+ def __init__(self,db):
+ Module.__init__(self, 'MTPT', db)
+ self.path = self.db.get_val('path')
+ self.clientoptions = self.db.get_val('clientoptions', '')
+ self.fs_uuid = self.db.get_first_ref('filesystem')
+ fs = self.db.lookup(self.fs_uuid)
+ self.mds_uuid = fs.get_first_ref('lmv')
+ if not self.mds_uuid:
+ self.mds_uuid = fs.get_first_ref('mds')
+ self.obd_uuid = fs.get_first_ref('obd')
+ self.mgmt_uuid = fs.get_first_ref('mgmt')
+ client_uuid = generate_client_uuid(self.name)
+
+ ost = self.db.lookup(self.obd_uuid)
+ if not ost:
+ panic("no ost: ", self.obd_uuid)
- self.info(self.path, self.mdc_uuid,self.lov_uuid)
- cmd = "mount -t lustre_lite -o osc=%s,mdc=%s none %s" % \
- (self.lov_uuid, self.mdc_uuid, self.path)
+ mds = self.db.lookup(self.mds_uuid)
+ if not mds:
+ panic("no mds: ", self.mds_uuid)
+
+ self.add_lustre_module('mdc', 'mdc')
+ self.add_lustre_module('lmv', 'lmv')
+ self.add_lustre_module('llite', 'llite')
+
+ self.vosc = VOSC(ost, client_uuid, self.name)
+ self.vmdc = VMDC(mds, client_uuid, self.name)
+
+ if self.mgmt_uuid:
+ self.mgmtcli = ManagementClient(db.lookup(self.mgmt_uuid),
+ client_uuid)
+ else:
+ self.mgmtcli = None
+
+ def prepare(self):
+ if not config.record and fs_is_mounted(self.path):
+ log(self.path, "already mounted.")
+ return
+ run_acceptors()
+ if self.mgmtcli:
+ self.mgmtcli.prepare()
+ self.vosc.prepare()
+ self.vmdc.prepare()
+ vmdc_name = self.vmdc.get_name()
+
+ self.info(self.path, self.mds_uuid, self.obd_uuid)
+ if config.record or config.lctl_dump:
+ lctl.mount_option(local_node_name, self.vosc.get_name(), vmdc_name)
+ return
+
+ if config.clientoptions:
+ if self.clientoptions:
+ self.clientoptions = self.clientoptions + ',' + \
+ config.clientoptions
+ else:
+ self.clientoptions = config.clientoptions
+ if self.clientoptions:
+ self.clientoptions = ',' + self.clientoptions
+ # Linux kernel will deal with async and not pass it to ll_fill_super,
+ # so replace it with Lustre async
+ self.clientoptions = string.replace(self.clientoptions, "async",
+ "lasync")
+
+ cmd = "mount -t lustre_lite -o osc=%s,mdc=%s%s %s %s" % \
+ (self.vosc.get_name(), vmdc_name, self.clientoptions,
+ config.config, self.path)
run("mkdir", self.path)
ret, val = run(cmd)
if ret:
- panic("mount failed:", self.path)
- def cleanup(self):
- self.info(self.path, self.mdc_uuid,self.lov_uuid)
- (rc, out) = run("umount", self.path)
- if rc:
- log("umount failed, cleanup will most likely not work.")
- l = lookup(self.dom_node.parentNode, self.lov_uuid)
- if l.nodeName == 'lov':
- lov = LOV(l)
- for osc_uuid in lov.devlist:
- osc = lookup(self.dom_node.parentNode, osc_uuid)
- if osc:
- n = OSC(osc)
- n.cleanup()
- else:
- panic('osc not found:', osc_uuid)
- else:
- osc = OSC(l)
- osc.cleanup()
-
+ self.vmdc.cleanup()
+ self.vosc.cleanup()
+ panic("mount failed:", self.path, ":", string.join(val))
-# ============================================================
-# XML processing and query
-# TODO: Change query funcs to use XPath, which is muc cleaner
-
-def get_device(obd):
- list = obd.getElementsByTagName('device')
- if len(list) > 0:
- dev = list[0]
- dev.normalize();
- size = get_attr_int(dev, 'size', 0)
- return dev.firstChild.data, size
- return '', 0
-
-# Get the text content from the first matching child
-# If there is no content (or it is all whitespace), return
-# the default
-def get_text(dom_node, tag, default=""):
- list = dom_node.getElementsByTagName(tag)
- if len(list) > 0:
- dom_node = list[0]
- dom_node.normalize()
- if dom_node.firstChild:
- txt = string.strip(dom_node.firstChild.data)
- if txt:
- return txt
- return default
-
-def get_text_int(dom_node, tag, default=0):
- list = dom_node.getElementsByTagName(tag)
- n = default
- if len(list) > 0:
- dom_node = list[0]
- dom_node.normalize()
- if dom_node.firstChild:
- txt = string.strip(dom_node.firstChild.data)
- if txt:
- try:
- n = int(txt)
- except ValueError:
- panic("text value is not integer:", txt)
- return n
+ def cleanup(self):
+ self.info(self.path, self.mds_uuid,self.obd_uuid)
-def get_attr(dom_node, attr, default=""):
- v = dom_node.getAttribute(attr)
- if v:
- return v
- return default
+ if config.record or config.lctl_dump:
+ lctl.del_mount_option(local_node_name)
+ else:
+ if fs_is_mounted(self.path):
+ if config.force:
+ (rc, out) = run("umount", "-f", self.path)
+ else:
+ (rc, out) = run("umount", self.path)
+ if rc:
+ raise CommandError('umount', out, rc)
-def get_attr_int(dom_node, attr, default=0):
- n = default
- v = dom_node.getAttribute(attr)
- if v:
- try:
- n = int(v)
- except ValueError:
- panic("attr value is not integer", v)
- return n
+ if fs_is_mounted(self.path):
+ panic("fs is still mounted:", self.path)
-def get_first_ref(dom_node, tag):
- """ Get the first uuidref of the type TAG. Used one only
- one is expected. Returns the uuid."""
- uuid = None
- refname = '%s_ref' % tag
- list = dom_node.getElementsByTagName(refname)
- if len(list) > 0:
- uuid = getRef(list[0])
- return uuid
-
-def get_all_refs(dom_node, tag):
- """ Get all the refs of type TAG. Returns list of uuids. """
- uuids = []
- refname = '%s_ref' % tag
- list = dom_node.getElementsByTagName(refname)
- if len(list) > 0:
- for i in list:
- uuids.append(getRef(i))
- return uuids
-
-def get_ost_net(dom_node, uuid):
- ost = lookup(dom_node, uuid)
- uuid = get_first_ref(ost, 'network')
- if not uuid:
- return None
- return lookup(dom_node, uuid)
-
-def nid2server(dom_node, nid):
- netlist = dom_node.getElementsByTagName('network')
- for net_node in netlist:
- if get_text(net_node, 'server') == nid:
- return Network(net_node)
- return None
-
-def lookup(dom_node, uuid):
- for n in dom_node.childNodes:
- if n.nodeType == n.ELEMENT_NODE:
- if getUUID(n) == uuid:
- return n
- else:
- n = lookup(n, uuid)
- if n: return n
- return None
-
-# Get name attribute of dom_node
-def getName(dom_node):
- return dom_node.getAttribute('name')
+ self.vmdc.cleanup()
+ self.vosc.cleanup()
+ if self.mgmtcli:
+ self.mgmtcli.cleanup()
-def getRef(dom_node):
- return dom_node.getAttribute('uuidref')
+ def load_module(self):
+ if self.mgmtcli:
+ self.mgmtcli.load_module()
+ self.vosc.load_module()
+ Module.load_module(self)
-# Get name attribute of dom_node
-def getUUID(dom_node):
- return dom_node.getAttribute('uuid')
+ def cleanup_module(self):
+ Module.cleanup_module(self)
+ self.vosc.cleanup_module()
+ if self.mgmtcli:
+ self.mgmtcli.cleanup_module()
-# the tag name is the service type
-# fixme: this should do some checks to make sure the dom_node is a service
-def getServiceType(dom_node):
- return dom_node.nodeName
+ def correct_level(self, level, op=None):
+ return level
-#
-# determine what "level" a particular node is at.
-# the order of iniitailization is based on level.
-def getServiceLevel(dom_node):
- type = getServiceType(dom_node)
+# ============================================================
+# misc query functions
+
+def get_ost_net(self, osd_uuid):
+ srv_list = []
+ if not osd_uuid:
+ return srv_list
+ osd = self.lookup(osd_uuid)
+ node_uuid = osd.get_first_ref('node')
+ node = self.lookup(node_uuid)
+ if not node:
+ panic("unable to find node for osd_uuid:", osd_uuid,
+ " node_ref:", node_uuid_)
+ for net_uuid in node.get_networks():
+ db = node.lookup(net_uuid)
+ srv_list.append(Network(db))
+ return srv_list
+
+
+# the order of iniitailization is based on level.
+def getServiceLevel(self):
+ type = self.get_class()
+ ret=0;
if type in ('network',):
- return 10
- elif type in ('device', 'ldlm'):
- return 20
- elif type in ('obd', 'mdd'):
- return 30
- elif type in ('mds','ost'):
- return 40
- elif type in ('mdc','osc'):
- return 50
- elif type in ('lov',):
- return 60
- elif type in ('mountpoint',):
- return 70
- return 0
+ ret = 5
+ elif type in ('routetbl',):
+ ret = 6
+ elif type in ('ldlm',):
+ ret = 20
+ elif type in ('mgmt',):
+ ret = 25
+ elif type in ('osd', 'cobd'):
+ ret = 30
+ elif type in ('mdsdev',):
+ ret = 40
+ elif type in ('lmv',):
+ ret = 45
+ elif type in ('cmobd',):
+ ret = 50
+ elif type in ('mountpoint', 'echoclient'):
+ ret = 70
+ else:
+ panic("Unknown type: ", type)
+
+ if ret < config.minlevel or ret > config.maxlevel:
+ ret = 0
+ return ret
#
# return list of services in a profile. list is a list of tuples
-# [(level, dom_node),]
-def getServices(lustreNode, profileNode):
+# [(level, db_object),]
+def getServices(self):
list = []
- for n in profileNode.childNodes:
- if n.nodeType == n.ELEMENT_NODE:
- servNode = lookup(lustreNode, getRef(n))
- if not servNode:
- print n
- panic('service not found: ' + getRef(n))
- level = getServiceLevel(servNode)
- list.append((level, servNode))
+ for ref_class, ref_uuid in self.get_all_refs():
+ servdb = self.lookup(ref_uuid)
+ if servdb:
+ level = getServiceLevel(servdb)
+ if level > 0:
+ list.append((level, servdb))
+ else:
+ panic('service not found: ' + ref_uuid)
+
list.sort()
return list
-def getByName(lustreNode, name, tag):
- ndList = lustreNode.getElementsByTagName(tag)
- for nd in ndList:
- if getName(nd) == name:
- return nd
- return None
-
-
-
############################################################
-# routing ("rooting")
+# MDC UUID hack -
+# FIXME: clean this mess up!
#
-routes = []
-local_node = []
-router_flag = 0
-
-def init_node(dom_node):
- global local_node, router_flag
- netlist = dom_node.getElementsByTagName('network')
- for dom_net in netlist:
- type = get_attr(dom_net, 'type')
- gw = get_text(dom_net, 'server')
- local_node.append((type, gw))
+# OSC is no longer in the xml, so we have to fake it.
+# this is getting ugly and begging for another refactoring
+def get_osc(ost_db, uuid, fs_name):
+ osc = OSC(ost_db, uuid, fs_name)
+ return osc
+
+def get_mdc(db, fs_name, mds_uuid):
+ mds_db = db.lookup(mds_uuid);
+ if not mds_db:
+ error("no mds:", mds_uuid)
+ mdc = MDC(mds_db, mds_uuid, fs_name)
+ return mdc
+############################################################
+# routing ("rooting")
+# list of (nettype, cluster_id, nid)
+local_clusters = []
+
+def find_local_clusters(node_db):
+ global local_clusters
+ for netuuid in node_db.get_networks():
+ net = node_db.lookup(netuuid)
+ srv = Network(net)
+ debug("add_local", netuuid)
+ local_clusters.append((srv.net_type, srv.cluster_id, srv.nid))
+ if srv.port > 0:
+ if acceptors.has_key(srv.port):
+ panic("duplicate port:", srv.port)
+ acceptors[srv.port] = AcceptorHandler(srv.port, srv.net_type,
+ srv.send_mem, srv.recv_mem,
+ srv.irq_affinity)
+
+# This node is a gateway.
+is_router = 0
+def node_is_router():
+ return is_router
+
+# If there are any routers found in the config, then this will be true
+# and all nodes will load kptlrouter.
+needs_router = 0
def node_needs_router():
- return router_flag
-
-def get_routes(type, gw, dom_net):
- """ Return the routes as a list of tuples of the form:
- [(type, gw, lo, hi),]"""
- res = []
- tbl = dom_net.getElementsByTagName('route_tbl')
- routes = tbl[0].getElementsByTagName('route')
- for r in routes:
- lo = get_attr(r, 'lo')
- hi = get_attr(r, 'hi', '')
- res.append((type, gw, lo, hi))
- return res
-
+ return needs_router or is_router
+
+# list of (nettype, gw, tgt_cluster_id, lo, hi)
+# Currently, these local routes are only added to kptlrouter route
+# table if they are needed to connect to a specific server. This
+# should be changed so all available routes are loaded, and the
+# ptlrouter can make all the decisions.
+local_routes = []
-def init_route_config(lustre):
- """ Scan the lustre config looking for routers. Build list of
+def find_local_routes(lustre):
+ """ Scan the lustre config looking for routers . Build list of
routes. """
- global routes, router_flag
- routes = []
- list = lustre.getElementsByTagName('node')
- for node in list:
- if get_attr(node, 'router'):
- router_flag = 1
- for (local_type, local_nid) in local_node:
+ global local_routes, needs_router
+ local_routes = []
+ list = lustre.lookup_class('node')
+ for router in list:
+ if router.get_val_int('router', 0):
+ needs_router = 1
+ for (local_type, local_cluster_id, local_nid) in local_clusters:
gw = None
- netlist = node.getElementsByTagName('network')
- for dom_net in netlist:
- if local_type == get_attr(dom_net, 'type'):
- gw = get_text(dom_net, 'server')
+ for netuuid in router.get_networks():
+ db = router.lookup(netuuid)
+ if (local_type == db.get_val('nettype') and
+ local_cluster_id == db.get_val('clusterid')):
+ gw = db.get_val('nid')
break
- if not gw:
- continue
- for dom_net in netlist:
- if local_type != get_attr(dom_net, 'type'):
- for route in get_routes(local_type, gw, dom_net):
- routes.append(route)
-
+ if gw:
+ debug("find_local_routes: gw is", gw)
+ for route in router.get_local_routes(local_type, gw):
+ local_routes.append(route)
+ debug("find_local_routes:", local_routes)
+
+
+def choose_local_server(srv_list):
+ for srv in srv_list:
+ if local_cluster(srv.net_type, srv.cluster_id):
+ return srv
-def local_net(net):
- global local_node
- for iface in local_node:
- if net.net_type == iface[0]:
+def local_cluster(net_type, cluster_id):
+ for cluster in local_clusters:
+ if net_type == cluster[0] and cluster_id == cluster[1]:
return 1
return 0
-def find_route(net):
- global local_node, routes
- frm_type = local_node[0][0]
- to_type = net.net_type
- to = net.nid
- debug ('looking for route to', to_type,to)
- for r in routes:
- if r[2] == to:
- return r
- return None
-
-
-
+def local_interface(net_type, cluster_id, nid):
+ for cluster in local_clusters:
+ if (net_type == cluster[0] and cluster_id == cluster[1]
+ and nid == cluster[2]):
+ return 1
+ return 0
+
+def find_route(srv_list):
+ result = []
+ frm_type = local_clusters[0][0]
+ for srv in srv_list:
+ debug("find_route: srv:", srv.nid, "type: ", srv.net_type)
+ to_type = srv.net_type
+ to = srv.nid
+ cluster_id = srv.cluster_id
+ debug ('looking for route to', to_type, to)
+ for r in local_routes:
+ debug("find_route: ", r)
+ if (r[3] <= to and to <= r[4]) and cluster_id == r[2]:
+ result.append((srv, r))
+ return result
+
+def get_active_target(db):
+ target_uuid = db.getUUID()
+ target_name = db.getName()
+ node_name = get_select(target_name)
+ if node_name:
+ tgt_dev_uuid = db.get_node_tgt_dev(node_name, target_uuid)
+ else:
+ tgt_dev_uuid = db.get_first_ref('active')
+ return tgt_dev_uuid
+
+def get_server_by_nid_uuid(db, nid_uuid):
+ for n in db.lookup_class("network"):
+ net = Network(n)
+ if net.nid_uuid == nid_uuid:
+ return net
+
############################################################
# lconf level logic
# Start a service.
-def startService(dom_node, module_flag):
- type = getServiceType(dom_node)
- debug('Service:', type, getName(dom_node), getUUID(dom_node))
- # there must be a more dynamic way of doing this...
+def newService(db):
+ type = db.get_class()
+ debug('Service:', type, db.getName(), db.getUUID())
n = None
if type == 'ldlm':
- n = LDLM(dom_node)
+ n = LDLM(db)
elif type == 'lov':
- n = LOV(dom_node)
+ n = LOV(db, "YOU_SHOULD_NEVER_SEE_THIS_UUID")
elif type == 'network':
- n = Network(dom_node)
- elif type == 'obd':
- n = OBD(dom_node)
- elif type == 'ost':
- n = OST(dom_node)
- elif type == 'mds':
- n = MDS(dom_node)
- elif type == 'osc':
- n = OSC(dom_node)
- elif type == 'mdc':
- n = MDC(dom_node)
+ n = Network(db)
+ elif type == 'routetbl':
+ n = RouteTable(db)
+ elif type == 'osd':
+ n = OSD(db)
+ elif type == 'cobd':
+ n = COBD(db, "YOU_SHOULD_NEVER_SEE_THIS_UUID")
+ elif type == 'cmobd':
+ n = CMOBD(db)
+ elif type == 'mdsdev':
+ n = MDSDEV(db)
elif type == 'mountpoint':
- n = Mountpoint(dom_node)
+ n = Mountpoint(db)
+ elif type == 'echoclient':
+ n = ECHO_CLIENT(db)
+ elif type == 'mgmt':
+ n = Management(db)
+ elif type == 'lmv':
+ n = LMV(db)
else:
panic ("unknown service type:", type)
-
- if module_flag:
- if config.nomod():
- return
- if config.cleanup():
- n.cleanup_module()
- else:
- n.load_module()
- else:
- if config.nosetup():
- return
- if config.cleanup():
- n.cleanup()
- else:
- n.prepare()
+ return n
#
# Prepare the system to run lustre using a particular profile
-# in a the configuration.
+# in a the configuration.
# * load & the modules
# * setup networking for the current node
# * make sure partitions are in place and prepared
# * initialize devices with lctl
# Levels is important, and needs to be enforced.
-def startProfile(lustreNode, profileNode, module_flag):
- if not profileNode:
- panic("profile:", profile, "not found.")
- services = getServices(lustreNode, profileNode)
- if config.cleanup():
- services.reverse()
+def for_each_profile(db, prof_list, operation):
+ for prof_uuid in prof_list:
+ prof_db = db.lookup(prof_uuid)
+ if not prof_db:
+ panic("profile:", profile, "not found.")
+ services = getServices(prof_db)
+ operation(services)
+
+def magic_get_osc(db, rec, lov):
+ if lov:
+ lov_uuid = lov.get_uuid()
+ lov_name = lov.osc.fs_name
+ else:
+ lov_uuid = rec.getAttribute('lov_uuidref')
+ # FIXME: better way to find the mountpoint?
+ filesystems = db.root_node.getElementsByTagName('filesystem')
+ fsuuid = None
+ for fs in filesystems:
+ ref = fs.getElementsByTagName('obd_ref')
+ if ref[0].getAttribute('uuidref') == lov_uuid:
+ fsuuid = fs.getAttribute('uuid')
+ break
+
+ if not fsuuid:
+ panic("malformed xml: lov uuid '" + lov_uuid + "' referenced in 'add' record is not used by any filesystems.")
+
+ mtpts = db.root_node.getElementsByTagName('mountpoint')
+ lov_name = None
+ for fs in mtpts:
+ ref = fs.getElementsByTagName('filesystem_ref')
+ if ref[0].getAttribute('uuidref') == fsuuid:
+ lov_name = fs.getAttribute('name')
+ break
+
+ if not lov_name:
+ panic("malformed xml: 'add' record references lov uuid '" + lov_uuid + "', which references filesystem uuid '" + fsuuid + "', which does not reference a mountpoint.")
+
+ print "lov_uuid: " + lov_uuid + "; lov_name: " + lov_name
+
+ ost_uuid = rec.getAttribute('ost_uuidref')
+ obd = db.lookup(ost_uuid)
+
+ if not obd:
+ panic("malformed xml: 'add' record references ost uuid '" + ost_uuid + "' which cannot be found.")
+
+ osc = get_osc(obd, lov_uuid, lov_name)
+ if not osc:
+ panic('osc not found:', obd_uuid)
+ return osc
+
+# write logs for update records. sadly, logs of all types -- and updates in
+# particular -- are something of an afterthought. lconf needs rewritten with
+# these as core concepts. so this is a pretty big hack.
+def process_update_record(db, update, lov):
+ for rec in update.childNodes:
+ if rec.nodeType != rec.ELEMENT_NODE:
+ continue
+
+ log("found "+rec.nodeName+" record in update version " +
+ str(update.getAttribute('version')))
+
+ lov_uuid = rec.getAttribute('lov_uuidref')
+ ost_uuid = rec.getAttribute('ost_uuidref')
+ index = rec.getAttribute('index')
+ gen = rec.getAttribute('generation')
+
+ if not lov_uuid or not ost_uuid or not index or not gen:
+ panic("malformed xml: 'update' record requires lov_uuid, ost_uuid, index, and generation.")
+
+ if not lov:
+ tmplov = db.lookup(lov_uuid)
+ if not tmplov:
+ panic("malformed xml: 'delete' record contains lov UUID '" + lov_uuid + "', which cannot be located.")
+ lov_name = tmplov.getName()
+ else:
+ lov_name = lov.osc.name
+
+ # ------------------------------------------------------------- add
+ if rec.nodeName == 'add':
+ if config.cleanup:
+ lctl.lov_del_obd(lov_name, lov_uuid, ost_uuid, index, gen)
+ continue
+
+ osc = magic_get_osc(db, rec, lov)
+
+ try:
+ # Only ignore connect failures with --force, which
+ # isn't implemented here yet.
+ osc.prepare(ignore_connect_failure=0)
+ except CommandError, e:
+ print "Error preparing OSC %s\n" % osc.uuid
+ raise e
+
+ lctl.lov_add_obd(lov_name, lov_uuid, ost_uuid, index, gen)
+
+ # ------------------------------------------------------ deactivate
+ elif rec.nodeName == 'deactivate':
+ if config.cleanup:
+ continue
+
+ osc = magic_get_osc(db, rec, lov)
+
+ try:
+ osc.deactivate()
+ except CommandError, e:
+ print "Error deactivating OSC %s\n" % osc.uuid
+ raise e
+
+ # ---------------------------------------------------------- delete
+ elif rec.nodeName == 'delete':
+ if config.cleanup:
+ continue
+
+ osc = magic_get_osc(db, rec, lov)
+
+ try:
+ config.cleanup = 1
+ osc.cleanup()
+ config.cleanup = 0
+ except CommandError, e:
+ print "Error cleaning up OSC %s\n" % osc.uuid
+ raise e
+
+ lctl.lov_del_obd(lov_name, lov_uuid, ost_uuid, index, gen)
+
+def process_updates(db, log_device, log_name, lov = None):
+ updates = db.root_node.getElementsByTagName('update')
+ for u in updates:
+ if not u.childNodes:
+ log("ignoring empty update record (version " +
+ str(u.getAttribute('version')) + ")")
+ continue
+
+ version = u.getAttribute('version')
+ real_name = "%s-%s" % (log_name, version)
+ lctl.clear_log(log_device, real_name)
+ lctl.record(log_device, real_name)
+
+ process_update_record(db, u, lov)
+
+ lctl.end_record()
+
+def doWriteconf(services):
+ #if config.nosetup:
+ # return
for s in services:
- startService(s[1], module_flag)
+ if s[1].get_class() == 'mdsdev':
+ n = newService(s[1])
+ n.write_conf()
+def doSetup(services):
+ if config.nosetup:
+ return
+ slist = []
+ for s in services:
+ n = newService(s[1])
+ n.level = s[0]
+ slist.append((n.level, n))
+ nlist = []
+ for n in slist:
+ nl = n[1].correct_level(n[0])
+ nlist.append((nl, n[1]))
+ nlist.sort()
+ for n in nlist:
+ n[1].prepare()
+
+def doModules(services):
+ if config.nomod:
+ return
+ for s in services:
+ n = newService(s[1])
+ n.load_module()
+
+def doCleanup(services):
+ if config.nosetup:
+ return
+ slist = []
+ for s in services:
+ n = newService(s[1])
+ n.level = s[0]
+ slist.append((n.level, n))
+ nlist = []
+ for n in slist:
+ nl = n[1].correct_level(n[0])
+ nlist.append((nl, n[1]))
+ nlist.sort()
+ nlist.reverse()
+ for n in nlist:
+ if n[1].safe_to_clean():
+ n[1].cleanup()
+
+def doUnloadModules(services):
+ if config.nomod:
+ return
+ services.reverse()
+ for s in services:
+ n = newService(s[1])
+ if n.safe_to_clean_modules():
+ n.cleanup_module()
#
-# Load profile for
-def doHost(lustreNode, hosts):
- global routes
- dom_node = None
+# Load profile for
+def doHost(lustreDB, hosts):
+ global is_router, local_node_name
+ node_db = None
for h in hosts:
- dom_node = getByName(lustreNode, h, 'node')
- if dom_node:
+ node_db = lustreDB.lookup_name(h, 'node')
+ if node_db:
break
+ if not node_db:
+ panic('No host entry found.')
- if not dom_node:
- print 'No host entry found.'
- return
+ local_node_name = node_db.get_val('name', 0)
+ is_router = node_db.get_val_int('router', 0)
+ lustre_upcall = node_db.get_val('lustreUpcall', '')
+ portals_upcall = node_db.get_val('portalsUpcall', '')
+ timeout = node_db.get_val_int('timeout', 0)
+ ptldebug = node_db.get_val('ptldebug', '')
+ subsystem = node_db.get_val('subsystem', '')
- if not get_attr(dom_node, 'router'):
- init_node(dom_node)
- init_route_config(lustreNode)
- else:
- global router_flag
- router_flag = 1
+ find_local_clusters(node_db)
+ if not is_router:
+ find_local_routes(lustreDB)
# Two step process: (1) load modules, (2) setup lustre
# if not cleaning, load modules first.
- module_flag = not config.cleanup()
- reflist = dom_node.getElementsByTagName('profile')
- for profile in reflist:
- startProfile(lustreNode, profile, module_flag)
+ prof_list = node_db.get_refs('profile')
+
+ if config.write_conf:
+ lustreDB.close()
+ for_each_profile(node_db, prof_list, doModules)
+ sys_make_devices()
+ for_each_profile(node_db, prof_list, doWriteconf)
+ for_each_profile(node_db, prof_list, doUnloadModules)
+
+ elif config.recover:
+ if not (config.tgt_uuid and config.client_uuid and config.conn_uuid):
+ raise Lustre.LconfError( "--recovery requires --tgt_uuid <UUID> " +
+ "--client_uuid <UUID> --conn_uuid <UUID>")
+ doRecovery(lustreDB, lctl, config.tgt_uuid, config.client_uuid,
+ config.conn_uuid)
+ elif config.cleanup:
+ if config.force:
+ # the command line can override this value
+ timeout = 5
+ # ugly hack, only need to run lctl commands for --dump
+ if config.lctl_dump or config.record:
+ for_each_profile(node_db, prof_list, doCleanup)
+ return
+
+ sys_set_timeout(timeout)
+ sys_set_ptldebug(ptldebug)
+ sys_set_subsystem(subsystem)
+ sys_set_lustre_upcall(lustre_upcall)
+ sys_set_portals_upcall(portals_upcall)
+
+ for_each_profile(node_db, prof_list, doCleanup)
+ for_each_profile(node_db, prof_list, doUnloadModules)
+ lustreDB.close()
+
+ else:
+ # ugly hack, only need to run lctl commands for --dump
+ if config.lctl_dump or config.record:
+ sys_set_timeout(timeout)
+ sys_set_lustre_upcall(lustre_upcall)
+ for_each_profile(node_db, prof_list, doSetup)
+ return
+
+ sys_make_devices()
+ sys_set_netmem_max('/proc/sys/net/core/rmem_max', MAXTCPBUF)
+ sys_set_netmem_max('/proc/sys/net/core/wmem_max', MAXTCPBUF)
+
+ for_each_profile(node_db, prof_list, doModules)
- if not config.cleanup():
sys_set_debug_path()
- script = config.gdb_script()
+ sys_set_ptldebug(ptldebug)
+ sys_set_subsystem(subsystem)
+ script = config.gdb_script
run(lctl.lctl, ' modules >', script)
- if config.gdb():
- # dump /tmp/ogdb and sleep/pause here
+ if config.gdb:
log ("The GDB module script is in", script)
+ # pause, so user has time to break and
+ # load the script
time.sleep(5)
-
- module_flag = not module_flag
- for profile in reflist:
- startProfile(lustreNode, profile, module_flag)
-
-############################################################
-# Command line processing
-#
-def parse_cmdline(argv):
- short_opts = "hdnv"
- long_opts = ["ldap", "reformat", "lustre=", "verbose", "gdb",
- "portals=", "makeldiff", "cleanup", "noexec",
- "help", "node=", "get=", "nomod", "nosetup"]
- opts = []
- args = []
+ sys_set_timeout(timeout)
+ sys_set_lustre_upcall(lustre_upcall)
+ sys_set_portals_upcall(portals_upcall)
+
+ for_each_profile(node_db, prof_list, doSetup)
+ lustreDB.close()
+
+def doRecovery(lustreDB, lctl, tgt_uuid, client_uuid, nid_uuid):
+ tgt = lustreDB.lookup(tgt_uuid)
+ if not tgt:
+ raise Lustre.LconfError("doRecovery: "+ tgt_uuid +" not found.")
+ new_uuid = get_active_target(tgt)
+ if not new_uuid:
+ raise Lustre.LconfError("doRecovery: no active target found for: " +
+ tgt_uuid)
+ net = choose_local_server(get_ost_net(lustreDB, new_uuid))
+ if not net:
+ raise Lustre.LconfError("Unable to find a connection to:" + new_uuid)
+
+ log("Reconnecting", tgt_uuid, " to ", net.nid_uuid);
try:
- opts, args = getopt.getopt(argv, short_opts, long_opts)
- except getopt.error:
- print "invalid opt"
- usage()
-
- for o, a in opts:
- if o in ("-h", "--help"):
- usage()
- if o in ("-d","--cleanup"):
- config.cleanup(1)
- if o in ("-v", "--verbose"):
- config.verbose(1)
- if o in ("-n", "--noexec"):
- config.noexec(1)
- config.verbose(1)
- if o == "--portals":
- config.portals = a
- if o == "--lustre":
- config.lustre = a
- if o == "--reformat":
- config.reformat(1)
- if o == "--node":
- config.node(a)
- if o == "--get":
- config.url(a)
- if o == "--gdb":
- config.gdb(1)
- if o == "--nomod":
- config.nomod(1)
- if o == "--nosetup":
- config.nosetup(1)
- return args
-
-def fetch(url):
- import urllib
- data = ""
+ oldnet = get_server_by_nid_uuid(lustreDB, nid_uuid)
+ lustreDB.close()
+ if oldnet:
+ lctl.disconnect(oldnet)
+ except CommandError, e:
+ log("recover: disconnect", nid_uuid, "failed: ")
+ e.dump()
+
try:
- s = urllib.urlopen(url)
- data = s.read()
- except:
- usage()
- return data
+ lctl.connect(net)
+ except CommandError, e:
+ log("recover: connect failed")
+ e.dump()
-def setupModulePath(cmd):
- base = os.path.dirname(cmd)
- if os.access(base+"/Makefile", os.R_OK):
- config.src_dir(base + "/../../")
+ lctl.recover(client_uuid, net.nid_uuid)
-def sys_set_debug_path():
- debug("debug path: ", config.debug_path())
- if config.noexec():
+
+def setupModulePath(cmd, portals_dir = PORTALS_DIR):
+ base = os.path.dirname(cmd)
+ if development_mode():
+ if not config.lustre:
+ debug('using objdir module paths')
+ config.lustre = (os.path.join(base, ".."))
+ # normalize the portals dir, using command line arg if set
+ if config.portals:
+ portals_dir = config.portals
+ dir = os.path.join(config.lustre, portals_dir)
+ config.portals = dir
+ debug('config.portals', config.portals)
+ elif config.lustre and config.portals:
+ # production mode
+ # if --lustre and --portals, normalize portals
+ # can ignore POTRALS_DIR here, since it is probly useless here
+ config.portals = os.path.join(config.lustre, config.portals)
+ debug('config.portals B', config.portals)
+
+def sysctl(path, val):
+ debug("+ sysctl", path, val)
+ if config.noexec:
return
try:
- fp = open('/proc/sys/portals/debug_path', 'w')
- fp.write(config.debug_path())
+ fp = open(os.path.join('/proc/sys', path), 'w')
+ fp.write(str(val))
fp.close()
except IOError, e:
- print e
-
-#/proc/sys/net/core/rmem_max
-#/proc/sys/net/core/wmem_max
+ panic(str(e))
+
+
+def sys_set_debug_path():
+ sysctl('portals/debug_path', config.debug_path)
+
+def sys_set_lustre_upcall(upcall):
+ # the command overrides the value in the node config
+ if config.lustre_upcall:
+ upcall = config.lustre_upcall
+ elif config.upcall:
+ upcall = config.upcall
+ if upcall:
+ lctl.set_lustre_upcall(upcall)
+
+def sys_set_portals_upcall(upcall):
+ # the command overrides the value in the node config
+ if config.portals_upcall:
+ upcall = config.portals_upcall
+ elif config.upcall:
+ upcall = config.upcall
+ if upcall:
+ sysctl('portals/upcall', upcall)
+
+def sys_set_timeout(timeout):
+ # the command overrides the value in the node config
+ if config.timeout and config.timeout > 0:
+ timeout = config.timeout
+ if timeout != None and timeout > 0:
+ lctl.set_timeout(timeout)
+
+def sys_tweak_socknal ():
+ if config.single_socket:
+ sysctl("socknal/typed", 0)
+
+def sys_optimize_elan ():
+ procfiles = ["/proc/elan/config/eventint_punt_loops",
+ "/proc/qsnet/elan3/config/eventint_punt_loops",
+ "/proc/qsnet/elan4/config/elan4_mainint_punt_loops"]
+ for p in procfiles:
+ if os.access(p, os.R_OK):
+ run ("echo 1 > " + p)
+
+def sys_set_ptldebug(ptldebug):
+ if config.ptldebug:
+ ptldebug = config.ptldebug
+ if ptldebug:
+ try:
+ val = eval(ptldebug, ptldebug_names)
+ val = "0x%x" % (val)
+ sysctl('portals/debug', val)
+ except NameError, e:
+ panic(str(e))
+
+def sys_set_subsystem(subsystem):
+ if config.subsystem:
+ subsystem = config.subsystem
+ if subsystem:
+ try:
+ val = eval(subsystem, subsystem_names)
+ val = "0x%x" % (val)
+ sysctl('portals/subsystem_debug', val)
+ except NameError, e:
+ panic(str(e))
+
def sys_set_netmem_max(path, max):
debug("setting", path, "to at least", max)
- if config.noexec():
+ if config.noexec:
return
fp = open(path)
str = fp.readline()
- fp.close
+ fp.close()
cur = int(str)
if max > cur:
fp = open(path, 'w')
fp.write('%d\n' %(max))
fp.close()
-
-
+
+
def sys_make_devices():
if not os.access('/dev/portals', os.R_OK):
run('mknod /dev/portals c 10 240')
if not os.access('/dev/obd', os.R_OK):
run('mknod /dev/obd c 10 241')
-# Initialize or shutdown lustre according to a configuration file
-# * prepare the system for lustre
-# * configure devices with lctl
-# Shutdown does steps in reverse
-#
+
+# Add dir to the global PATH, if not already there.
+def add_to_path(new_dir):
+ syspath = string.split(os.environ['PATH'], ':')
+ if new_dir in syspath:
+ return
+ os.environ['PATH'] = os.environ['PATH'] + ':' + new_dir
+
+def default_debug_path():
+ path = '/tmp/lustre-log'
+ if os.path.isdir('/r'):
+ return '/r' + path
+ else:
+ return path
+
+def default_gdb_script():
+ script = '/tmp/ogdb'
+ if os.path.isdir('/r'):
+ return '/r' + script
+ else:
+ return script
+
+
+DEFAULT_PATH = ('/sbin', '/usr/sbin', '/bin', '/usr/bin')
+# ensure basic elements are in the system path
+def sanitise_path():
+ for dir in DEFAULT_PATH:
+ add_to_path(dir)
+
+# global hack for the --select handling
+tgt_select = {}
+def init_select(args):
+ # args = [service=nodeA,service2=nodeB service3=nodeC]
+ global tgt_select
+ for arg in args:
+ list = string.split(arg, ',')
+ for entry in list:
+ srv, node = string.split(entry, '=')
+ tgt_select[srv] = node
+
+def get_select(srv):
+ if tgt_select.has_key(srv):
+ return tgt_select[srv]
+ return None
+
+
+FLAG = Lustre.Options.FLAG
+PARAM = Lustre.Options.PARAM
+INTPARAM = Lustre.Options.INTPARAM
+PARAMLIST = Lustre.Options.PARAMLIST
+lconf_options = [
+ ('verbose,v', "Print system commands as they are run"),
+ ('ldapurl',"LDAP server URL, eg. ldap://localhost", PARAM),
+ ('config', "Cluster config name used for LDAP query", PARAM),
+ ('select', "service=nodeA,service2=nodeB ", PARAMLIST),
+ ('node', "Load config for <nodename>", PARAM),
+ ('cleanup,d', "Cleans up config. (Shutdown)"),
+ ('force,f', "Forced unmounting and/or obd detach during cleanup",
+ FLAG, 0),
+ ('single_socket', "socknal option: only use one socket instead of bundle",
+ FLAG, 0),
+ ('failover',"""Used to shut down without saving state.
+ This will allow this node to "give up" a service to a
+ another node for failover purposes. This will not
+ be a clean shutdown.""",
+ FLAG, 0),
+ ('gdb', """Prints message after creating gdb module script
+ and sleeps for 5 seconds."""),
+ ('noexec,n', """Prints the commands and steps that will be run for a
+ config without executing them. This can used to check if a
+ config file is doing what it should be doing"""),
+ ('nomod', "Skip load/unload module step."),
+ ('nosetup', "Skip device setup/cleanup step."),
+ ('reformat', "Reformat all devices (without question)"),
+ ('mkfsoptions', "Additional options for the mk*fs command line", PARAM),
+ ('mountfsoptions', "Additional options for mount fs command line", PARAM),
+ ('clientoptions', "Additional options for Lustre", PARAM),
+ ('dump', "Dump the kernel debug log to file before portals is unloaded",
+ PARAM),
+ ('write_conf', "Save all the client config information on mds."),
+ ('record', "Write config information on mds."),
+ ('record_log', "Name of config record log.", PARAM),
+ ('record_device', "MDS device name that will record the config commands",
+ PARAM),
+ ('minlevel', "Minimum level of services to configure/cleanup",
+ INTPARAM, 0),
+ ('maxlevel', """Maximum level of services to configure/cleanup
+ Levels are aproximatly like:
+ 10 - netwrk
+ 20 - device, ldlm
+ 30 - osd, mdd
+ 40 - mds, ost
+ 70 - mountpoint, echo_client, osc, mdc, lov""",
+ INTPARAM, 100),
+ ('lustre', """Base directory of lustre sources. This parameter will
+ cause lconf to load modules from a source tree.""", PARAM),
+ ('portals', """Portals source directory. If this is a relative path,
+ then it is assumed to be relative to lustre. """, PARAM),
+ ('timeout', "Set recovery timeout", INTPARAM),
+ ('upcall', "Set both portals and lustre upcall script", PARAM),
+ ('lustre_upcall', "Set lustre upcall script", PARAM),
+ ('portals_upcall', "Set portals upcall script", PARAM),
+ ('lctl_dump', "Save lctl ioctls to the dumpfile argument", PARAM),
+ ('ptldebug', "Set the portals debug level", PARAM),
+ ('subsystem', "Set the portals debug subsystem", PARAM),
+ ('gdb_script', "Fullname of gdb debug script", PARAM, default_gdb_script()),
+ ('debug_path', "Path to save debug dumps", PARAM, default_debug_path()),
+# Client recovery options
+ ('recover', "Recover a device"),
+ ('group', "The group of devices to configure or cleanup", PARAM),
+ ('tgt_uuid', "The failed target (required for recovery)", PARAM),
+ ('client_uuid', "The failed client (required for recovery)", PARAM),
+ ('conn_uuid', "The failed connection (required for recovery)", PARAM),
+
+ ('inactive', """The name of an inactive service, to be ignored during
+ mounting (currently OST-only). Can be repeated.""",
+ PARAMLIST),
+ ]
+
def main():
- global TCP_ACCEPTOR, lctl, MAXTCPBUF
- args = parse_cmdline(sys.argv[1:])
+ global lctl, config, toplustreDB, CONFIG_FILE
+
+ # in the upcall this is set to SIG_IGN
+ signal.signal(signal.SIGCHLD, signal.SIG_DFL)
+
+ cl = Lustre.Options("lconf", "config.xml", lconf_options)
+ try:
+ config, args = cl.parse(sys.argv[1:])
+ except Lustre.OptionError, e:
+ print e
+ sys.exit(1)
+
+ setupModulePath(sys.argv[0])
+
+ host = socket.gethostname()
+
+ # the PRNG is normally seeded with time(), which is not so good for starting
+ # time-synchronized clusters
+ input = open('/dev/urandom', 'r')
+ if not input:
+ print 'Unable to open /dev/urandom!'
+ sys.exit(1)
+ seed = input.read(32)
+ input.close()
+ random.seed(seed)
+
+ sanitise_path()
+
+ init_select(config.select)
+
if len(args) > 0:
- if not os.access(args[0], os.R_OK | os.W_OK):
- print 'File not found:', args[0]
+ # allow config to be fetched via HTTP, but only with python2
+ if sys.version[0] != '1' and args[0].startswith('http://'):
+ import urllib2
+ try:
+ config_file = urllib2.urlopen(args[0])
+ except (urllib2.URLError, socket.error), err:
+ if hasattr(err, 'args'):
+ err = err.args[1]
+ print "Could not access '%s': %s" %(args[0], err)
+ sys.exit(1)
+ elif not os.access(args[0], os.R_OK):
+ print 'File not found or readable:', args[0]
sys.exit(1)
- dom = xml.dom.minidom.parse(args[0])
- elif config.url():
- xmldata = fetch(config.url())
- dom = xml.dom.minidom.parseString(xmldata)
+ else:
+ # regular file
+ config_file = open(args[0], 'r')
+ try:
+ dom = xml.dom.minidom.parse(config_file)
+ except Exception:
+ panic("%s does not appear to be a config file." % (args[0]))
+ sys.exit(1) # make sure to die here, even in debug mode.
+ config_file.close()
+ CONFIG_FILE = args[0]
+ lustreDB = Lustre.LustreDB_XML(dom.documentElement, dom.documentElement)
+ if not config.config:
+ config.config = os.path.basename(args[0])# use full path?
+ if config.config[-4:] == '.xml':
+ config.config = config.config[:-4]
+ elif config.ldapurl:
+ if not config.config:
+ panic("--ldapurl requires --config name")
+ dn = "config=%s,fs=lustre" % (config.config)
+ lustreDB = Lustre.LustreDB_LDAP('', {}, base=dn, url = config.ldapurl)
+ elif config.ptldebug or config.subsystem:
+ sys_set_ptldebug(None)
+ sys_set_subsystem(None)
+ sys.exit(0)
else:
- usage()
+ print 'Missing config file or ldap URL.'
+ print 'see lconf --help for command summary'
+ sys.exit(1)
+
+ toplustreDB = lustreDB
+
+ ver = lustreDB.get_version()
+ if not ver:
+ panic("No version found in config data, please recreate.")
+ if ver != Lustre.CONFIG_VERSION:
+ panic("Config version", ver, "does not match lconf version",
+ Lustre.CONFIG_VERSION)
node_list = []
- if config.node():
- node_list.append(config.node())
+ if config.node:
+ node_list.append(config.node)
else:
- host = socket.gethostname()
if len(host) > 0:
node_list.append(host)
node_list.append('localhost')
+
debug("configuring for host: ", node_list)
- TCP_ACCEPTOR = find_prog('acceptor')
- if not TCP_ACCEPTOR:
- if config.noexec():
- TCP_ACCEPTOR = 'acceptor'
- debug('! acceptor not found')
- else:
- panic('acceptor not found')
+ if len(host) > 0:
+ config.debug_path = config.debug_path + '-' + host
+ config.gdb_script = config.gdb_script + '-' + host
lctl = LCTLInterface('lctl')
- setupModulePath(sys.argv[0])
- sys_make_devices()
- sys_set_netmem_max('/proc/sys/net/core/rmem_max', MAXTCPBUF)
- sys_set_netmem_max('/proc/sys/net/core/wmem_max', MAXTCPBUF)
- doHost(dom.documentElement, node_list)
+ if config.lctl_dump:
+ lctl.use_save_file(config.lctl_dump)
+
+ if config.record:
+ if not (config.record_device and config.record_log):
+ panic("When recording, both --record_log and --record_device must be specified.")
+ lctl.clear_log(config.record_device, config.record_log)
+ lctl.record(config.record_device, config.record_log)
+
+ doHost(lustreDB, node_list)
+
+ if not config.record:
+ return
+
+ lctl.end_record()
+
+ process_updates(lustreDB, config.record_device, config.record_log)
if __name__ == "__main__":
try:
main()
- except LconfError, e:
+ except Lustre.LconfError, e:
print e
+# traceback.print_exc(file=sys.stdout)
+ sys.exit(1)
except CommandError, e:
e.dump()
+ sys.exit(e.rc)
+
+ if first_cleanup_error:
+ sys.exit(first_cleanup_error)
git://git.whamcloud.com / fs / lustre-release.git / blobdiff