import Lustre
# Global parameters
-MAXTCPBUF = 1048576
-DEFAULT_TCPBUF = 1048576
+MAXTCPBUF = 16777216
+DEFAULT_TCPBUF = 8388608
+DEFAULT_PORT = 988
#
# Maximum number of devices to search for.
# (the /dev/loop* nodes need to be created beforehand)
MAX_LOOP_DEVICES = 256
PORTALS_DIR = 'portals'
+# Needed to call lconf --record
+CONFIG_FILE = ""
# Please keep these in sync with the values in portals/kp30.h
ptldebug_names = {
log(self.pidfile(), e)
class AcceptorHandler(DaemonHandler):
- def __init__(self, port, net_type, send_mem, recv_mem, irq_aff, nid_xchg):
+ def __init__(self, port, net_type, send_mem, recv_mem, irq_aff):
DaemonHandler.__init__(self, "acceptor")
self.port = port
self.flags = ''
self.flags = self.flags + ' -N 4'
if irq_aff:
self.flags = self.flags + ' -i'
- if nid_xchg:
- self.flags = self.flags + ' -x'
def pidfile(self):
return "/var/run/%s-%d.pid" % (self.command, self.port)
# start the acceptors
def run_acceptors():
+ if config.lctl_dump or config.record:
+ return
for port in acceptors.keys():
daemon = acceptors[port]
if not daemon.running():
daemon.start()
def run_one_acceptor(port):
+ if config.lctl_dump or config.record:
+ return
if acceptors.has_key(port):
daemon = acceptors[port]
if not daemon.running():
"""
self.lctl = find_prog(cmd)
self.save_file = ''
+ self.record_device = ''
if not self.lctl:
if config.noexec:
debug('! lctl not found')
def use_save_file(self, file):
self.save_file = file
+ def record(self, dev_name, logname):
+ log("Recording log", logname, "on", dev_name)
+ self.record_device = dev_name
+ self.record_log = logname
+
+ def end_record(self):
+ log("End recording log", self.record_log, "on", self.record_device)
+ self.record_device = None
+ self.record_log = None
+
def set_nonblock(self, fd):
fl = fcntl.fcntl(fd, F_GETFL)
fcntl.fcntl(fd, F_SETFL, fl | os.O_NDELAY)
"""
cmd_line = self.lctl
if self.save_file:
- cmds = '\n dump ' + self.save_file + cmds
-
+ cmds = '\n dump ' + self.save_file + '\n' + cmds
+ elif self.record_device:
+ cmds = """
+ device $%s
+ probe
+ record %s
+ %s""" % (self.record_device, self.record_log, cmds)
+
debug("+", cmd_line, cmds)
if config.noexec: return (0, [])
def network(self, net, nid):
- """ initialized network and add "self" """
+ """ set mynid """
cmds = """
network %s
mynid %s
self.run(cmds)
# create a new connection
+ def add_uuid(self, net_type, uuid, nid):
+ cmds = "\n add_uuid %s %s %s" %(uuid, nid, net_type)
+ self.run(cmds)
+
+ def add_autoconn(self, net_type, send_mem, recv_mem, nid, hostaddr,
+ port, flags):
+ if net_type in ('tcp', 'toe') and not config.lctl_dump:
+ cmds = """
+ network %s
+ send_mem %d
+ recv_mem %d
+ add_autoconn %s %s %d %s
+ quit""" % (net_type,
+ send_mem,
+ recv_mem,
+ nid, hostaddr, port, flags )
+ self.run(cmds)
+
def connect(self, srv):
- cmds = "\n add_uuid %s %s %s" % (srv.uuid, srv.nid, srv.net_type)
+ self.add_uuid(srv.net_type, srv.nid_uuid, srv.nid)
if srv.net_type in ('tcp', 'toe') and not config.lctl_dump:
flags = 's'
if srv.irq_affinity:
flags = flags + 'i'
- if srv.nid_exchange:
- flags = flags + 'x'
- cmds = """%s
- network %s
- send_mem %d
- recv_mem %d
- add_autoconn %s %s %d %s""" % (cmds, srv.net_type,
- srv.send_mem,
- srv.recv_mem,
- srv.nid, srv.hostaddr, srv.port, flags )
-
- cmds = cmds + "\n quit"
- self.run(cmds)
+ self.add_autoconn(srv.net_type, srv.send_mem, srv.recv_mem,
+ srv.nid, srv.hostaddr, srv.port, flags)
# Recover a device
def recover(self, dev_name, new_conn):
# add a route to a host
def add_route_host(self, net, uuid, gw, tgt):
+ self.add_uuid(net, uuid, tgt)
cmds = """
network %s
- add_uuid %s %s %s
add_route %s %s
quit """ % (net,
- uuid, tgt, net,
gw, tgt)
try:
self.run(cmds)
# add a route to a range
def del_route_host(self, net, uuid, gw, tgt):
+ self.del_uuid(uuid)
cmds = """
ignore_errors
network %s
- del_uuid %s
del_route %s %s
- quit """ % (net, uuid, gw, tgt)
+ quit """ % (net, gw, tgt)
self.run(cmds)
+
+ def del_autoconn(self, net_type, nid, hostaddr):
+ if net_type in ('tcp', 'toe') and not config.lctl_dump:
+ cmds = """
+ ignore_errors
+ network %s
+ del_autoconn %s %s s
+ quit""" % (net_type,
+ nid, hostaddr)
+ self.run(cmds)
+
# disconnect one connection
def disconnect(self, srv):
- cmds = " ignore_errors\n del_uuid %s" % (srv.uuid)
+ self.del_uuid(srv.nid_uuid)
if srv.net_type in ('tcp', 'toe') and not config.lctl_dump:
- cmds = """%s
- network %s
- del_autoconn %s %s s""" % (cmds,
- srv.net_type,
- srv.nid, srv.hostaddr)
- cmds = cmds + "\n quit"
-
- self.run(cmds)
+ self.del_autoconn(srv.net_type, srv.nid, srv.hostaddr)
- def del_uuid(self, servuuid):
+ def del_uuid(self, uuid):
cmds = """
ignore_errors
del_uuid %s
- quit""" % (servuuid,)
+ quit""" % (uuid,)
self.run(cmds)
# disconnect all
quit""" % (net)
self.run(cmds)
- # create a new device with lctl
- def newdev(self, attach, setup = ""):
+ def attach(self, type, name, uuid):
cmds = """
- newdev
- attach %s
+ attach %s %s %s
+ quit""" % (type, name, uuid)
+ self.run(cmds)
+
+ def setup(self, name, setup = ""):
+ cmds = """
+ cfg_device %s
setup %s
- quit""" % (attach, setup)
+ quit""" % (name, setup)
self.run(cmds)
+
+
+ # create a new device with lctl
+ def newdev(self, type, name, uuid, setup = ""):
+ self.attach(type, name, uuid);
+ try:
+ self.setup(name, setup)
+ except CommandError, e:
+ self.cleanup(name, uuid, 0)
+ raise e
+
# cleanup a device
def cleanup(self, name, uuid, force, failover = 0):
if failover: force = 1
cmds = """
ignore_errors
- device $%s
+ cfg_device $%s
cleanup %s %s
detach
quit""" % (name, ('', 'force')[force],
self.run(cmds)
# create an lov
+ def lov_setup(self, name, uuid, desc_uuid, mdsuuid, stripe_cnt,
+ stripe_sz, stripe_off,
+ pattern, devlist):
+ cmds = """
+ attach lov %s %s
+ lov_setup %s %d %d %d %s %s
+ quit""" % (name, uuid, desc_uuid, stripe_cnt, stripe_sz, stripe_off,
+ pattern, devlist)
+ self.run(cmds)
+
+ # create an lov
def lov_setconfig(self, uuid, mdsuuid, stripe_cnt, stripe_sz, stripe_off,
pattern, devlist):
cmds = """
- device $%s
- probe
+ cfg_device $%s
lov_setconfig %s %d %d %d %s %s
quit""" % (mdsuuid, uuid, stripe_cnt, stripe_sz, stripe_off, pattern, devlist)
self.run(cmds)
# get list of devices
def device_list(self):
- try:
- rc, out = self.runcmd('device_list')
- except CommandError, e:
- if config.cleanup:
- out = []
- else:
- raise e
- return out
+ devices = '/proc/fs/lustre/devices'
+ ret = []
+ if os.access(devices, os.R_OK):
+ try:
+ fp = open(devices, 'r')
+ ret = fp.readlines()
+ fp.close()
+ except IOError, e:
+ log(e)
+ return ret
# get lustre version
def lustre_version(self):
return out
# dump mount options
- def mount_option(self, option):
+ def mount_option(self, profile, osc, mdc):
cmds = """
- mount_option %s
- quit""" % (option)
+ mount_option %s %s %s
+ quit""" % (profile, osc, mdc)
+ self.run(cmds)
+
+ # delete mount options
+ def del_mount_option(self, profile):
+ cmds = """
+ del_mount_option %s
+ quit""" % (profile,)
+ self.run(cmds)
+
+ def set_timeout(self, timeout):
+ cmds = """
+ set_timeout %s
+ quit""" % (timeout,)
+ self.run(cmds)
+
+ # delete mount options
+ def set_lustre_upcall(self, upcall):
+ cmds = """
+ set_lustre_upcall %s
+ quit""" % (upcall,)
self.run(cmds)
# ============================================================
# Various system-level functions
# build fs according to type
# fixme: dangerous
-def mkfs(dev, devsize, fstype,jsize):
+def mkfs(dev, devsize, fstype, jsize, mkfsoptions, isblock=1):
block_cnt = ''
jopt = ''
if devsize:
if fstype in ('ext3', 'extN'):
# ext3 journal size is in megabytes
if jsize: jopt = "-J size=%d" %(jsize,)
- mkfs = 'mkfs.ext2 -j -b 4096 -F '
+ mkfs = 'mkfs.ext2 -j -b 4096 '
+ if not isblock or config.force:
+ mkfs = mkfs + ' -F '
elif fstype == 'reiserfs':
# reiserfs journal size is in blocks
if jsize: jopt = "--journal_size %d" %(jsize,)
mkfs = 'mkreiserfs -ff'
else:
- print 'unsupported fs type: ', fstype
+ panic('unsupported fs type: ', fstype)
+ if config.mkfsoptions != None:
+ mkfs = mkfs + ' ' + config.mkfsoptions
+ if mkfsoptions != None:
+ mkfs = mkfs + ' ' + mkfsoptions
(ret, out) = run (mkfs, jopt, dev, block_cnt)
if ret:
panic("Unable to build fs:", dev, string.join(out))
return ''
# create file if necessary and assign the first free loop device
-def init_loop(file, size, fstype, journal_size):
+def init_loop(file, size, fstype, journal_size, mkfsoptions, reformat):
dev = find_loop(file)
if dev:
print 'WARNING file:', file, 'already mapped to', dev
return dev
- if config.reformat or not os.access(file, os.R_OK | os.W_OK):
+ if reformat or not os.access(file, os.R_OK | os.W_OK):
if size < 8000:
panic("size of loopback file '%s' must be larger than 8MB, but is set to %s" % (file,size))
(ret, out) = run("dd if=/dev/zero bs=1k count=0 seek=%d of=%s" %(size,
file))
if ret:
panic("Unable to create backing store:", file)
- mkfs(file, size, fstype, journal_size)
+ mkfs(file, size, fstype, journal_size, mkfsoptions, isblock=0)
loop = loop_base()
# find next free loop
return 0
# initialize a block device if needed
-def block_dev(dev, size, fstype, format, journal_size):
+def block_dev(dev, size, fstype, reformat, autoformat, journal_size,
+ mkfsoptions):
if config.noexec: return dev
if not is_block(dev):
- dev = init_loop(dev, size, fstype, journal_size)
- elif config.reformat or (need_format(fstype, dev) and format == 'yes'):
- mkfs(dev, size, fstype, journal_size)
+ dev = init_loop(dev, size, fstype, journal_size, mkfsoptions, reformat)
+ elif reformat or (need_format(fstype, dev) and autoformat == 'yes'):
+ mkfs(dev, size, fstype, journal_size, mkfsoptions, isblock=0)
# else:
# panic("device:", dev,
return local
+def mod_loaded(modname):
+ """Check if a module is already loaded. Look in /proc/modules for it."""
+ try:
+ fp = open('/proc/modules')
+ lines = fp.readlines()
+ fp.close()
+ # please forgive my tired fingers for this one
+ ret = filter(lambda word, mod=modname: word == mod,
+ map(lambda line: string.split(line)[0], lines))
+ return ret
+ except Exception, e:
+ return 0
+
# XXX: instead of device_list, ask for $name and see what we get
def is_prepared(name):
"""Return true if a device exists for the name"""
if config.lctl_dump:
return 0
- if config.noexec and config.cleanup:
+ if (config.noexec or config.record) and config.cleanup:
return 1
try:
# expect this format:
return 0
def is_network_prepared():
- """If the LDLM device exists, then assume that all networking
+ """If the any device exists, then assume that all networking
has been configured"""
- return is_prepared('ldlm')
-
+ out = lctl.device_list()
+ return len(out) > 0
+
def fs_is_mounted(path):
"""Return true if path is a mounted lustre filesystem"""
try:
return 0
+class kmod:
+ """Manage kernel modules"""
+ def __init__(self, lustre_dir, portals_dir):
+ self.lustre_dir = lustre_dir
+ self.portals_dir = portals_dir
+ self.kmodule_list = []
+
+ def add_portals_module(self, dev_dir, modname):
+ """Append a module to list of modules to load."""
+ self.kmodule_list.append((self.portals_dir, dev_dir, modname))
+
+ def add_lustre_module(self, dev_dir, modname):
+ """Append a module to list of modules to load."""
+ self.kmodule_list.append((self.lustre_dir, dev_dir, modname))
+
+ def load_module(self):
+ """Load all the modules in the list in the order they appear."""
+ for src_dir, dev_dir, mod in self.kmodule_list:
+ if mod_loaded(mod) and not config.noexec:
+ continue
+ log ('loading module:', mod, 'srcdir', src_dir, 'devdir', dev_dir)
+ if src_dir:
+ module = find_module(src_dir, dev_dir, mod)
+ if not module:
+ panic('module not found:', mod)
+ (rc, out) = run('/sbin/insmod', module)
+ if rc:
+ raise CommandError('insmod', out, rc)
+ else:
+ (rc, out) = run('/sbin/modprobe', mod)
+ if rc:
+ raise CommandError('modprobe', out, rc)
+
+ def cleanup_module(self):
+ """Unload the modules in the list in reverse order."""
+ rev = self.kmodule_list
+ rev.reverse()
+ for src_dir, dev_dir, mod in rev:
+ if not mod_loaded(mod) and not config.noexec:
+ continue
+ # debug hack
+ if mod == 'portals' and config.dump:
+ lctl.dump(config.dump)
+ log('unloading module:', mod)
+ (rc, out) = run('/sbin/rmmod', mod)
+ if rc:
+ log('! unable to unload module:', mod)
+ logall(out)
+
# ============================================================
# Classes to prepare and cleanup the various objects
#
self.module_name = module_name
self.name = self.db.getName()
self.uuid = self.db.getUUID()
- self.kmodule_list = []
self._server = None
self._connected = 0
+ self.kmod = kmod(config.lustre, config.portals)
def info(self, *args):
msg = string.join(map(str,args))
def add_portals_module(self, dev_dir, modname):
"""Append a module to list of modules to load."""
- self.kmodule_list.append((config.portals, dev_dir, modname))
+ self.kmod.add_portals_module(dev_dir, modname)
def add_lustre_module(self, dev_dir, modname):
"""Append a module to list of modules to load."""
- self.kmodule_list.append((config.lustre, dev_dir, modname))
-
- def mod_loaded(self, modname):
- """Check if a module is already loaded. Look in /proc/modules for it."""
- fp = open('/proc/modules')
- lines = fp.readlines()
- fp.close()
- # please forgive my tired fingers for this one
- ret = filter(lambda word, mod=modname: word == mod,
- map(lambda line: string.split(line)[0], lines))
- return ret
+ self.kmod.add_lustre_module(dev_dir, modname)
def load_module(self):
"""Load all the modules in the list in the order they appear."""
- for src_dir, dev_dir, mod in self.kmodule_list:
- # (rc, out) = run ('/sbin/lsmod | grep -s', mod)
- if self.mod_loaded(mod) and not config.noexec:
- continue
- log ('loading module:', mod, 'srcdir', src_dir, 'devdir', dev_dir)
- if src_dir:
- module = find_module(src_dir, dev_dir, mod)
- if not module:
- panic('module not found:', mod)
- (rc, out) = run('/sbin/insmod', module)
- if rc:
- raise CommandError('insmod', out, rc)
- else:
- (rc, out) = run('/sbin/modprobe', mod)
- if rc:
- raise CommandError('modprobe', out, rc)
+ self.kmod.load_module()
def cleanup_module(self):
"""Unload the modules in the list in reverse order."""
- if not self.safe_to_clean():
- return
- rev = self.kmodule_list
- rev.reverse()
- for src_dir, dev_dir, mod in rev:
- if not self.mod_loaded(mod) and not config.noexec:
- continue
- # debug hack
- if mod == 'portals' and config.dump:
- lctl.dump(config.dump)
- log('unloading module:', mod)
- (rc, out) = run('/sbin/rmmod', mod)
- if rc:
- log('! unable to unload module:', mod)
- logall(out)
+ if self.safe_to_clean():
+ self.kmod.cleanup_module()
def safe_to_clean(self):
return 1
self.send_mem = self.db.get_val_int('sendmem', DEFAULT_TCPBUF)
self.recv_mem = self.db.get_val_int('recvmem', DEFAULT_TCPBUF)
self.irq_affinity = self.db.get_val_int('irqaffinity', 0)
- self.nid_exchange = self.db.get_val_int('nidexchange', 0)
if '*' in self.nid:
- if self.nid_exchange:
- self.nid = sys_get_local_nid(self.net_type, self.nid, self.cluster_id)
- else:
- self.nid = sys_get_local_address(self.net_type, self.nid, self.cluster_id)
+ self.nid = sys_get_local_nid(self.net_type, self.nid, self.cluster_id)
if not self.nid:
panic("unable to set nid for", self.net_type, self.nid, cluster_id)
+ self.generic_nid = 1
debug("nid:", self.nid)
+ else:
+ self.generic_nid = 0
+
+ self.nid_uuid = self.nid_to_uuid(self.nid)
self.hostaddr = self.db.get_val('hostaddr', self.nid)
if '*' in self.hostaddr:
if self.net_type == 'scimac':
self.add_portals_module("knals/scimacnal", 'kscimacnal')
+ def nid_to_uuid(self, nid):
+ return "NID_%s_UUID" %(nid,)
+
def prepare(self):
if is_network_prepared():
return
self.info(self.net_type, self.nid, self.port)
- lctl.network(self.net_type, self.nid)
+ if not (config.record and self.generic_nid):
+ lctl.network(self.net_type, self.nid)
+ if self.net_type == 'tcp':
+ sys_tweak_socknal()
+ if self.net_type == 'elan':
+ sys_optimize_elan()
if self.port and node_is_router():
run_one_acceptor(self.port)
self.connect_peer_gateways()
if node_is_router():
self.disconnect_peer_gateways()
- try:
- lctl.disconnectAll(self.net_type)
- except CommandError, e:
- print "disconnectAll failed: ", self.name
- e.dump()
- cleanup_error(e.rc)
-
class RouteTable(Module):
def __init__(self,db):
Module.__init__(self, 'ROUTES', db)
- def server_for_route(self, net_type, gw, gw_cluster_id, tgt_cluster_id, lo, hi):
+ def server_for_route(self, net_type, gw, gw_cluster_id, tgt_cluster_id,
+ lo, hi):
# only setup connections for tcp NALs
srvdb = None
if not net_type in ('tcp', 'toe'):
if lo == hi and local_interface(net_type, gw_cluster_id, gw):
if not local_cluster(net_type, tgt_cluster_id):
panic("target", lo, " not on the local cluster")
- srvdb = self.db.nid2server(lo, net_type)
+ srvdb = self.db.nid2server(lo, net_type, gw_cluster_id)
# connect to gateway if this node is not the gw
elif (local_cluster(net_type, gw_cluster_id)
and not local_interface(net_type, gw_cluster_id, gw)):
- srvdb = self.db.nid2server(gw, net_type)
+ srvdb = self.db.nid2server(gw, net_type, gw_cluster_id)
else:
return None
class Management(Module):
def __init__(self, db):
Module.__init__(self, 'MGMT', db)
+ self.add_lustre_module('lvfs', 'lvfs')
self.add_lustre_module('obdclass', 'obdclass')
self.add_lustre_module('ptlrpc', 'ptlrpc')
- self.add_lustre_module('ldlm', 'ldlm')
self.add_lustre_module('mgmt', 'mgmt_svc')
def prepare(self):
if is_prepared(self.name):
return
self.info()
- lctl.newdev(attach="mgmt %s %s" % (self.name, self.uuid))
+ lctl.newdev("mgmt", self.name, self.uuid)
def safe_to_clean(self):
return 1
if is_prepared(self.name):
Module.cleanup(self)
+# This is only needed to load the modules; the LDLM device
+# is now created automatically.
class LDLM(Module):
def __init__(self,db):
Module.__init__(self, 'LDLM', db)
+ self.add_lustre_module('lvfs', 'lvfs')
self.add_lustre_module('obdclass', 'obdclass')
self.add_lustre_module('ptlrpc', 'ptlrpc')
- self.add_lustre_module('ldlm', 'ldlm')
def prepare(self):
- if is_prepared(self.name):
- return
- self.info()
- lctl.newdev(attach="ldlm %s %s" % ('ldlm', 'ldlm_UUID'))
-
- def safe_to_clean(self):
- out = lctl.device_list()
- return len(out) <= 1
+ return
def cleanup(self):
- if is_prepared(self.name):
- Module.cleanup(self)
+ return
class LOV(Module):
- def __init__(self, db, uuid, fs_name):
+ def __init__(self, db, uuid, fs_name, name_override = None):
Module.__init__(self, 'LOV', db)
- self.add_lustre_module('mdc', 'mdc')
+ if name_override != None:
+ self.name = "lov_%s" % name_override
self.add_lustre_module('lov', 'lov')
self.mds_uuid = self.db.get_first_ref('mds')
mds= self.db.lookup(self.mds_uuid)
self.devlist = self.db.get_refs('obd')
self.stripe_cnt = self.db.get_val_int('stripecount', len(self.devlist))
self.osclist = []
- self.client_uuid = generate_client_uuid(self.name)
+ self.desc_uuid = self.uuid
+ self.uuid = generate_client_uuid(self.name)
self.fs_name = fs_name
- self.mdc_name = ''
- self.mdc = get_mdc(db, self.client_uuid, fs_name, self.mds_uuid)
for obd_uuid in self.devlist:
obd = self.db.lookup(obd_uuid)
- osc = get_osc(obd, self.client_uuid, fs_name)
+ osc = get_osc(obd, self.uuid, fs_name)
if osc:
self.osclist.append(osc)
else:
except CommandError, e:
print "Error preparing OSC %s\n" % osc.uuid
raise e
- self.mdc.prepare()
- self.mdc_name = self.mdc.name
self.info(self.mds_uuid, self.stripe_cnt, self.stripe_sz,
self.stripe_off, self.pattern, self.devlist, self.mds_name)
- lctl.newdev(attach="lov %s %s" % (self.name, self.uuid),
- setup ="%s" % (self.mdc_name))
+ lctl.lov_setup(self.name, self.uuid,
+ self.desc_uuid, self.mds_name, self.stripe_cnt,
+ self.stripe_sz, self.stripe_off, self.pattern,
+ string.join(self.devlist))
def cleanup(self):
if is_prepared(self.name):
Module.cleanup(self)
for osc in self.osclist:
osc.cleanup()
- mdc = get_mdc(self.db, self.client_uuid, self.fs_name, self.mds_uuid)
- mdc.cleanup()
def load_module(self):
for osc in self.osclist:
osc.cleanup_module()
break
-class LOVConfig(Module):
- def __init__(self, db):
- Module.__init__(self, 'LOVConfig', db)
-
- self.lov_uuid = self.db.get_first_ref('lov')
- l = self.db.lookup(self.lov_uuid)
- self.lov = LOV(l, "YOU_SHOULD_NEVER_SEE_THIS_UUID", '')
-
- def prepare(self):
- lov = self.lov
- self.info(lov.mds_uuid, lov.stripe_cnt, lov.stripe_sz, lov.stripe_off,
- lov.pattern, lov.devlist, lov.mds_name)
- lctl.lov_setconfig(lov.uuid, lov.mds_name, lov.stripe_cnt,
- lov.stripe_sz, lov.stripe_off, lov.pattern,
- string.join(lov.devlist))
-
- def cleanup(self):
- #nothing to do here
- pass
-
class MDSDEV(Module):
def __init__(self,db):
Module.__init__(self, 'MDSDEV', db)
self.journal_size = self.db.get_val_int('journalsize', 0)
self.fstype = self.db.get_val('fstype', '')
self.nspath = self.db.get_val('nspath', '')
+ self.mkfsoptions = self.db.get_val('mkfsoptions', '')
# overwrite the orignal MDSDEV name and uuid with the MDS name and uuid
target_uuid = self.db.get_first_ref('target')
mds = self.db.lookup(target_uuid)
self.name = mds.getName()
- self.lovconfig_uuids = mds.get_refs('lovconfig')
self.filesystem_uuids = mds.get_refs('filesystem')
# FIXME: if fstype not set, then determine based on kernel version
self.format = self.db.get_val('autoformat', "no")
self.target_dev_uuid = self.uuid
self.uuid = target_uuid
# modules
+ self.add_lustre_module('mdc', 'mdc')
+ self.add_lustre_module('osc', 'osc')
+ self.add_lustre_module('lov', 'lov')
self.add_lustre_module('mds', 'mds')
if self.fstype:
- self.add_lustre_module('obdclass', 'fsfilt_%s' % (self.fstype))
+ self.add_lustre_module('lvfs', 'fsfilt_%s' % (self.fstype))
+
def load_module(self):
if self.active:
if not self.active:
debug(self.uuid, "not active")
return
- self.info(self.devpath, self.fstype, self.format)
+ if config.reformat:
+ # run write_conf automatically, if --reformat used
+ self.write_conf()
+ self.info(self.devpath, self.fstype, self.size, self.format)
run_acceptors()
- blkdev = block_dev(self.devpath, self.size, self.fstype, self.format,
- self.journal_size)
+ # never reformat here
+ blkdev = block_dev(self.devpath, self.size, self.fstype, 0,
+ self.format, self.journal_size, self.mkfsoptions)
if not is_prepared('MDT'):
- lctl.newdev(attach="mdt %s %s" % ('MDT', 'MDT_UUID'),
- setup ="")
- if self.nspath:
- run ("mkdir", self.nspath)
- lctl.newdev(attach="mds %s %s" % (self.name, self.uuid),
- setup ="%s %s %s" %(blkdev, self.fstype, self.nspath))
- for uuid in self.lovconfig_uuids:
- db = self.db.lookup(uuid)
- lovconfig = LOVConfig(db)
- lovconfig.prepare()
- if config.mds_ost_conn:
- for uuid in self.filesystem_uuids:
- log("open clients for filesystem:", uuid)
- fs = self.db.lookup(uuid)
- obd_uuid = fs.get_first_ref('obd')
- client_uuid = generate_client_uuid(self.name)
- client = VOSC(client_uuid, self.db.lookup(obd_uuid), self.name)
- client.prepare()
+ lctl.newdev("mdt", 'MDT', 'MDT_UUID', setup ="")
+ try:
+ lctl.newdev("mds", self.name, self.uuid,
+ setup ="%s %s %s" %(blkdev, self.fstype, self.name))
+ except CommandError, e:
+ if e.rc == 2:
+ panic("MDS is missing the config log. Need to run " +
+ "lconf --write_conf.")
+ else:
+ raise e
+
+ def write_conf(self):
+ if is_prepared(self.name):
+ return
+ self.info(self.devpath, self.fstype, self.format)
+ blkdev = block_dev(self.devpath, self.size, self.fstype,
+ config.reformat, self.format, self.journal_size,
+ self.mkfsoptions)
+ lctl.newdev("mds", self.name, self.uuid,
+ setup ="%s %s" %(blkdev, self.fstype))
-
+ # record logs for the MDS lov
+ for uuid in self.filesystem_uuids:
+ log("recording clients for filesystem:", uuid)
+ fs = self.db.lookup(uuid)
+ obd_uuid = fs.get_first_ref('obd')
+ client_uuid = generate_client_uuid(self.name)
+ client = VOSC(self.db.lookup(obd_uuid), client_uuid, self.name,
+ self.name)
+ config.record = 1
+ lctl.record(self.name, self.name)
+ client.prepare()
+ lctl.mount_option(self.name, client.get_name(), "")
+ lctl.end_record()
+
+ config.cleanup = 1
+ lctl.record(self.name, self.name + '-clean')
+ client.cleanup()
+ lctl.del_mount_option(self.name)
+ lctl.end_record()
+ config.cleanup = 0
+ config.record = 0
+
+ # record logs for each client
+ if config.ldapurl:
+ config_options = "--ldapurl " + config.ldapurl + " --config " + config.config
+ else:
+ config_options = CONFIG_FILE
+
+ for node_db in self.db.lookup_class('node'):
+ client_name = node_db.getName()
+ for prof_uuid in node_db.get_refs('profile'):
+ prof_db = node_db.lookup(prof_uuid)
+ # refactor this into a funtion to test "clientness"
+ # of a node.
+ for ref_class, ref_uuid in prof_db.get_all_refs():
+ if ref_class in ('mountpoint','echoclient'):
+ debug("recording", client_name)
+ old_noexec = config.noexec
+ config.noexec = 0
+ noexec_opt = ('', '-n')
+ ret, out = run (sys.argv[0],
+ noexec_opt[old_noexec == 1],
+ " -v --record --nomod",
+ "--record_log", client_name,
+ "--record_device", self.name,
+ "--node", client_name,
+ config_options)
+ if config.verbose:
+ for s in out: log("record> ", string.strip(s))
+ ret, out = run (sys.argv[0],
+ noexec_opt[old_noexec == 1],
+ "--cleanup -v --record --nomod",
+ "--record_log", client_name + "-clean",
+ "--record_device", self.name,
+ "--node", client_name,
+ config_options)
+ if config.verbose:
+ for s in out: log("record> ", string.strip(s))
+ config.noexec = old_noexec
+ try:
+ lctl.cleanup(self.name, self.uuid, 0, 0)
+ except CommandError, e:
+ log(self.module_name, "cleanup failed: ", self.name)
+ e.dump()
+ cleanup_error(e.rc)
+ Module.cleanup(self)
+ clean_loop(self.devpath)
+
def msd_remaining(self):
out = lctl.device_list()
for s in out:
if not self.active:
debug(self.uuid, "not active")
return
+ self.info()
if is_prepared(self.name):
- self.info()
try:
lctl.cleanup(self.name, self.uuid, config.force,
config.failover)
e.dump()
cleanup_error(e.rc)
Module.cleanup(self)
- if config.mds_ost_conn:
- for uuid in self.filesystem_uuids:
- log("clean clients for filesystem:", uuid)
- log("open clients for filesystem:", uuid)
- fs = self.db.lookup(uuid)
- obd_uuid = fs.get_first_ref('obd')
- client = VOSC(self.db.lookup(obd_uuid), self.name)
- client.cleanup()
if not self.msd_remaining() and is_prepared('MDT'):
try:
lctl.cleanup("MDT", "MDT_UUID", config.force,
self.devpath = self.db.get_val('devpath', '')
self.size = self.db.get_val_int('devsize', 0)
self.journal_size = self.db.get_val_int('journalsize', 0)
+ self.mkfsoptions = self.db.get_val_int('mkfsoptions', '')
self.fstype = self.db.get_val('fstype', '')
self.nspath = self.db.get_val('nspath', '')
target_uuid = self.db.get_first_ref('target')
self.add_lustre_module('ost', 'ost')
# FIXME: should we default to ext3 here?
if self.fstype:
- self.add_lustre_module('obdclass' , 'fsfilt_%s' % (self.fstype))
+ self.add_lustre_module('lvfs' , 'fsfilt_%s' % (self.fstype))
self.add_lustre_module(self.osdtype, self.osdtype)
def load_module(self):
blkdev = ''
else:
blkdev = block_dev(self.devpath, self.size, self.fstype,
- self.format, self.journal_size)
- if self.nspath:
- run ("mkdir", self.nspath)
- lctl.newdev(attach="%s %s %s" % (self.osdtype, self.name, self.uuid),
- setup ="%s %s %s %s" %(blkdev, self.fstype,
- self.failover_ost, self.nspath))
+ config.reformat, self.format, self.journal_size,
+ self.mkfsoptions)
+ lctl.newdev(self.osdtype, self.name, self.uuid,
+ setup ="%s %s %s" %(blkdev, self.fstype,
+ self.failover_ost))
if not is_prepared('OSS'):
- lctl.newdev(attach="ost %s %s" % ('OSS', 'OSS_UUID'),
- setup ="")
+ lctl.newdev("ost", 'OSS', 'OSS_UUID', setup ="")
def osd_remaining(self):
out = lctl.device_list()
if not self.tgt_dev_uuid:
panic("No target device found for target:", self.target_name)
- self.kmodule_list = []
+ self.kmod = kmod(config.lustre, config.portals)
self._server = None
self._connected = 0
if srv:
lctl.connect(srv)
else:
- srv, r = find_route(self.get_servers())
- if srv:
- lctl.add_route_host(r[0], srv.uuid, r[1], r[3])
- else:
+ routes = find_route(self.get_servers())
+ if len(routes) == 0:
panic ("no route to", self.target_uuid)
+ for (srv, r) in routes:
+ lctl.add_route_host(r[0], srv.nid_uuid, r[1], r[3])
except CommandError, e:
if not ignore_connect_failure:
raise e
if srv:
- lctl.newdev(attach="%s %s %s" % (self.module, self.name, self.uuid),
- setup ="%s %s %s" % (self.target_uuid, srv.uuid,
- self.mgmt_name))
+ if self.target_uuid in config.inactive and self.permits_inactive():
+ debug("%s inactive" % self.target_uuid)
+ inactive_p = "inactive"
+ else:
+ debug("%s active" % self.target_uuid)
+ inactive_p = ""
+ lctl.newdev(self.module, self.name, self.uuid,
+ setup ="%s %s %s %s" % (self.target_uuid, srv.nid_uuid,
+ inactive_p, self.mgmt_name))
def cleanup(self):
if is_prepared(self.name):
if srv:
lctl.disconnect(srv)
else:
- srv, r = find_route(self.get_servers())
- if srv:
- lctl.del_route_host(r[0], srv.uuid, r[1], r[3])
+ for (srv, r) in find_route(self.get_servers()):
+ lctl.del_route_host(r[0], srv.nid_uuid, r[1], r[3])
except CommandError, e:
log(self.module_name, "cleanup failed: ", self.name)
e.dump()
def __init__(self, db, uuid, fs_name):
Client.__init__(self, db, uuid, 'mdc', fs_name)
+ def permits_inactive(self):
+ return 0
class OSC(Client):
def __init__(self, db, uuid, fs_name):
Client.__init__(self, db, uuid, 'osc', fs_name)
+ def permits_inactive(self):
+ return 1
+
def mgmtcli_name_for_uuid(uuid):
return 'MGMTCLI_%s' % uuid
if is_prepared(self.name):
return
self.info(self.real_uuid, self.cache_uuid)
- lctl.newdev(attach="cobd %s %s" % (self.name, self.uuid),
+ lctl.newdev("cobd", self.name, self.uuid,
setup ="%s %s" %(self.real_uuid, self.cache_uuid))
# virtual interface for OSC and LOV
class VOSC(Module):
- def __init__(self, db, uuid, fs_name):
+ def __init__(self, db, uuid, fs_name, name_override = None):
Module.__init__(self, 'VOSC', db)
if db.get_class() == 'lov':
- self.osc = LOV(db, uuid, fs_name)
+ self.osc = LOV(db, uuid, fs_name, name_override)
else:
self.osc = get_osc(db, uuid, fs_name)
def get_uuid(self):
self.osc.load_module()
def cleanup_module(self):
self.osc.cleanup_module()
- def need_mdc(self):
- return self.db.get_class() != 'lov'
- def get_mdc_name(self):
- if self.db.get_class() == 'lov':
- return self.osc.mdc_name
- return ''
class ECHO_CLIENT(Module):
self.osc.prepare() # XXX This is so cheating. -p
self.info(self.obd_uuid)
- lctl.newdev(attach="echo_client %s %s" % (self.name, self.uuid),
+ lctl.newdev("echo_client", self.name, self.uuid,
setup = self.osc.get_name())
def cleanup(self):
obd = self.db.lookup(self.obd_uuid)
client_uuid = generate_client_uuid(self.name)
self.vosc = VOSC(obd, client_uuid, self.name)
- if self.vosc.need_mdc():
- self.add_lustre_module('mdc', 'mdc')
- self.mdc = get_mdc(db, client_uuid, self.name, self.mds_uuid)
+ self.mdc = get_mdc(db, client_uuid, self.name, self.mds_uuid)
+
+ self.add_lustre_module('mdc', 'mdc')
self.add_lustre_module('llite', 'llite')
if self.mgmt_uuid:
self.mgmtcli = ManagementClient(db.lookup(self.mgmt_uuid),
if self.mgmtcli:
self.mgmtcli.prepare()
self.vosc.prepare()
- if self.vosc.need_mdc():
- self.mdc.prepare()
- mdc_name = self.mdc.name
- else:
- mdc_name = self.vosc.get_mdc_name()
- if not mdc_name:
- self.vosc.cleanup()
- panic("Unable to determine MDC name. Probably need to cleanup before re-mounting.")
+ self.mdc.prepare()
+ mdc_name = self.mdc.name
+
self.info(self.path, self.mds_uuid, self.obd_uuid)
- if config.lctl_dump:
- cmd = "osc=%s,mdc=%s" % (self.vosc.get_name(), mdc_name)
- lctl.mount_option(cmd)
+ if config.record or config.lctl_dump:
+ lctl.mount_option(local_node_name, self.vosc.get_name(), mdc_name)
return
cmd = "mount -t lustre_lite -o osc=%s,mdc=%s %s %s" % \
(self.vosc.get_name(), mdc_name, config.config, self.path)
run("mkdir", self.path)
ret, val = run(cmd)
if ret:
+ self.mdc.cleanup()
self.vosc.cleanup()
- if self.vosc.need_mdc():
- self.mdc.cleanup()
panic("mount failed:", self.path, ":", string.join(val))
def cleanup(self):
self.info(self.path, self.mds_uuid,self.obd_uuid)
- if fs_is_mounted(self.path):
- if config.force:
- (rc, out) = run("umount", "-f", self.path)
- else:
- (rc, out) = run("umount", self.path)
- if rc:
- raise CommandError('umount', out, rc)
- if fs_is_mounted(self.path):
- panic("fs is still mounted:", self.path)
+ if config.record or config.lctl_dump:
+ lctl.del_mount_option(local_node_name)
+ else:
+ if fs_is_mounted(self.path):
+ if config.force:
+ (rc, out) = run("umount", "-f", self.path)
+ else:
+ (rc, out) = run("umount", self.path)
+ if rc:
+ raise CommandError('umount', out, rc)
+ if fs_is_mounted(self.path):
+ panic("fs is still mounted:", self.path)
+
+ self.mdc.cleanup()
self.vosc.cleanup()
- if self.vosc.need_mdc():
- self.mdc.cleanup()
if self.mgmtcli:
self.mgmtcli.cleanup()
panic("duplicate port:", srv.port)
acceptors[srv.port] = AcceptorHandler(srv.port, srv.net_type,
srv.send_mem, srv.recv_mem,
- srv.irq_affinity,
- srv.nid_exchange)
+ srv.irq_affinity)
# This node is a gateway.
is_router = 0
return 0
def find_route(srv_list):
+ result = []
frm_type = local_clusters[0][0]
for srv in srv_list:
debug("find_route: srv:", srv.nid, "type: ", srv.net_type)
for r in local_routes:
debug("find_route: ", r)
if (r[3] <= to and to <= r[4]) and cluster_id == r[2]:
- return srv, r
- return None,None
+ result.append((srv, r))
+ return result
def get_active_target(db):
target_uuid = db.getUUID()
tgt_dev_uuid = db.get_first_ref('active')
return tgt_dev_uuid
+def get_server_by_nid_uuid(db, nid_uuid):
+ for n in db.lookup_class("network"):
+ net = Network(n)
+ if net.nid_uuid == nid_uuid:
+ return net
+
############################################################
# lconf level logic
#
# Load profile for
def doHost(lustreDB, hosts):
- global is_router
+ global is_router, local_node_name
node_db = None
for h in hosts:
node_db = lustreDB.lookup_name(h, 'node')
print 'No host entry found.'
return
+ local_node_name = node_db.get_val('name', 0)
is_router = node_db.get_val_int('router', 0)
lustre_upcall = node_db.get_val('lustreUpcall', '')
portals_upcall = node_db.get_val('portalsUpcall', '')
timeout = node_db.get_val_int('timeout', 0)
-
+
find_local_clusters(node_db)
if not is_router:
find_local_routes(lustreDB)
# if not cleaning, load modules first.
prof_list = node_db.get_refs('profile')
- if config.recover:
+ if config.write_conf:
+ for_each_profile(node_db, prof_list, doModules)
+ sys_make_devices()
+ for node_db in lustreDB.lookup_class('mdsdev'):
+ mds = MDSDEV(node_db)
+ mds.write_conf()
+ for_each_profile(node_db, prof_list, doUnloadModules)
+
+ elif config.recover:
if not (config.tgt_uuid and config.client_uuid and config.conn_uuid):
raise Lustre.LconfError( "--recovery requires --tgt_uuid <UUID> " +
"--client_uuid <UUID> --conn_uuid <UUID>")
# the command line can override this value
timeout = 5
# ugly hack, only need to run lctl commands for --dump
- if config.lctl_dump:
+ if config.lctl_dump or config.record:
for_each_profile(node_db, prof_list, doCleanup)
return
else:
# ugly hack, only need to run lctl commands for --dump
- if config.lctl_dump:
+ if config.lctl_dump or config.record:
+ sys_set_timeout(timeout)
+ sys_set_lustre_upcall(lustre_upcall)
for_each_profile(node_db, prof_list, doSetup)
return
for_each_profile(node_db, prof_list, doSetup)
-def doRecovery(db, lctl, tgt_uuid, client_uuid, conn_uuid):
+def doRecovery(db, lctl, tgt_uuid, client_uuid, nid_uuid):
tgt = db.lookup(tgt_uuid)
if not tgt:
raise Lustre.LconfError("doRecovery: "+ tgt_uuid +" not found.")
net = choose_local_server(get_ost_net(db, new_uuid))
if not net:
raise Lustre.LconfError("Unable to find a connection to:" + new_uuid)
- # XXX, better to do a full disconnect here
- log("Reconnecting", tgt_uuid, " to ", net.uuid);
- lctl.del_uuid(conn_uuid)
- lctl.connect(net)
- lctl.recover(client_uuid, net.uuid)
+
+ log("Reconnecting", tgt_uuid, " to ", net.nid_uuid);
+ try:
+ oldnet = get_server_by_nid_uuid(db, nid_uuid)
+ if oldnet:
+ lctl.disconnect(oldnet)
+ except CommandError, e:
+ log("recover: disconnect", nid_uuid, "failed: ")
+ e.dump()
+
+ try:
+ lctl.connect(net)
+ except CommandError, e:
+ log("recover: connect failed")
+ e.dump()
+
+ lctl.recover(client_uuid, net.nid_uuid)
def setupModulePath(cmd, portals_dir = PORTALS_DIR):
elif config.upcall:
upcall = config.upcall
if upcall:
- sysctl('lustre/upcall', upcall)
+ lctl.set_lustre_upcall(upcall)
def sys_set_portals_upcall(upcall):
# the command overrides the value in the node config
def sys_set_timeout(timeout):
# the command overrides the value in the node config
- if config.timeout > 0:
+ if config.timeout and config.timeout > 0:
timeout = config.timeout
if timeout != None and timeout > 0:
- sysctl('lustre/timeout', timeout)
+ lctl.set_timeout(timeout)
+
+def sys_tweak_socknal ():
+ if config.single_socket:
+ sysctl("socknal/typed", 0)
+
+def sys_optimize_elan ():
+ run ("echo 0 > /proc/elan/config/eventint_punt_loops")
def sys_set_ptldebug():
if config.ptldebug != None:
return
fp = open(path)
str = fp.readline()
- fp.close
+ fp.close()
cur = int(str)
if max > cur:
fp = open(path, 'w')
# global hack for the --select handling
tgt_select = {}
-def init_select(arg):
- # arg = "service=nodeA,service2=nodeB"
+def init_select(args):
+ # args = [service=nodeA,service2=nodeB service3=nodeC]
global tgt_select
- if arg:
+ for arg in args:
list = string.split(arg, ',')
for entry in list:
srv, node = string.split(entry, '=')
return None
+FLAG = Lustre.Options.FLAG
PARAM = Lustre.Options.PARAM
INTPARAM = Lustre.Options.INTPARAM
+PARAMLIST = Lustre.Options.PARAMLIST
lconf_options = [
('verbose,v', "Print system commands as they are run"),
('ldapurl',"LDAP server URL, eg. ldap://localhost", PARAM),
('config', "Cluster config name used for LDAP query", PARAM),
- ('select', "service=nodeA,service2=nodeB ", PARAM),
+ ('select', "service=nodeA,service2=nodeB ", PARAMLIST),
('node', "Load config for <nodename>", PARAM),
('cleanup,d', "Cleans up config. (Shutdown)"),
('force,f', "Forced unmounting and/or obd detach during cleanup",
- Lustre.Options.FLAG, 0),
- ('mds_ost_conn', "Open connections to OSTs on the MDS"),
+ FLAG, 0),
+ ('single_socket', "socknal option: only use one socket instead of bundle",
+ FLAG, 0),
('failover',"""Used to shut down without saving state.
This will allow this node to "give up" a service to a
another node for failover purposes. This will not
be a clean shutdown.""",
- Lustre.Options.FLAG, 0),
+ FLAG, 0),
('gdb', """Prints message after creating gdb module script
and sleeps for 5 seconds."""),
('noexec,n', """Prints the commands and steps that will be run for a
('nomod', "Skip load/unload module step."),
('nosetup', "Skip device setup/cleanup step."),
('reformat', "Reformat all devices (without question)"),
+ ('mkfsoptions', "Additional options for the mk*fs command line", PARAM),
('dump', "Dump the kernel debug log to file before portals is unloaded",
PARAM),
+ ('write_conf', "Save all the client config information on mds."),
+ ('record', "Write config information on mds."),
+ ('record_log', "Name of config record log.", PARAM),
+ ('record_device', "MDS device name that will record the config commands",
+ PARAM),
('minlevel', "Minimum level of services to configure/cleanup",
INTPARAM, 0),
('maxlevel', """Maximum level of services to configure/cleanup
Levels are aproximatly like:
- 10 - network
+ 10 - netwrk
20 - device, ldlm
30 - osd, mdd
40 - mds, ost
cause lconf to load modules from a source tree.""", PARAM),
('portals', """Portals source directory. If this is a relative path,
then it is assumed to be relative to lustre. """, PARAM),
- ('timeout', "Set recovery timeout", PARAM),
+ ('timeout', "Set recovery timeout", INTPARAM),
('upcall', "Set both portals and lustre upcall script", PARAM),
('lustre_upcall', "Set lustre upcall script", PARAM),
('portals_upcall', "Set portals upcall script", PARAM),
('tgt_uuid', "The failed target (required for recovery)", PARAM),
('client_uuid', "The failed client (required for recovery)", PARAM),
('conn_uuid', "The failed connection (required for recovery)", PARAM),
+
+ ('inactive', """The name of an inactive service, to be ignored during
+ mounting (currently OST-only). Can be repeated.""",
+ PARAMLIST),
]
def main():
- global lctl, config, toplevel
+ global lctl, config, toplevel, CONFIG_FILE
# in the upcall this is set to SIG_IGN
signal.signal(signal.SIGCHLD, signal.SIG_DFL)
except Exception:
panic("%s does not appear to be a config file." % (args[0]))
sys.exit(1) # make sure to die here, even in debug mode.
+ CONFIG_FILE = args[0]
db = Lustre.LustreDB_XML(dom.documentElement, dom.documentElement)
if not config.config:
config.config = os.path.basename(args[0])# use full path?
if config.lctl_dump:
lctl.use_save_file(config.lctl_dump)
+ if config.record:
+ if not (config.record_device and config.record_log):
+ panic("When recording, both --record_log and --record_device must be specified.")
+ lctl.record(config.record_device, config.record_log)
+
doHost(db, node_list)
+ if config.record:
+ lctl.end_record()
+
if __name__ == "__main__":
try:
main()
git://git.whamcloud.com / fs / lustre-release.git / blobdiff