#!/usr/bin/env python
#
-# Copyright (C) 2002-2003 Cluster File Systems, Inc.
-# Authors: Robert Read <rread@clusterfs.com>
-# Mike Shaver <shaver@clusterfs.com>
-# This file is part of Lustre, http://www.lustre.org.
+# GPL HEADER START
#
-# Lustre is free software; you can redistribute it and/or
-# modify it under the terms of version 2 of the GNU General Public
-# License as published by the Free Software Foundation.
+# DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
#
-# Lustre is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-# GNU General Public License for more details.
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License version 2 only,
+# as published by the Free Software Foundation.
#
-# You should have received a copy of the GNU General Public License
-# along with Lustre; if not, write to the Free Software
-# Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+# This program is distributed in the hope that it will be useful, but
+# WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+# General Public License version 2 for more details (a copy is included
+# in the LICENSE file that accompanied this code).
+#
+# You should have received a copy of the GNU General Public License
+# version 2 along with this program; If not, see
+# http://www.sun.com/software/products/lustre/docs/GPLv2.pdf
+# copy of GPLv2].
+#
+# Please contact Sun Microsystems, Inc., 4150 Network Circle, Santa Clara,
+# CA 95054 USA or visit www.sun.com if you need additional information or
+# have any questions.
+#
+# GPL HEADER END
+#
+
+#
+# Copyright (c) 2002, 2010, Oracle and/or its affiliates. All rights reserved.
+# Use is subject to license terms.
+#
+
+#
+# This file is part of Lustre, http://www.lustre.org/
+# Lustre is a trademark of Sun Microsystems, Inc.
+#
+# Author: Robert Read <rread@clusterfs.com>
+# Author: Mike Shaver <shaver@clusterfs.com>
#
# lconf - lustre configuration tool
#
# lconf is the main driver script for starting and stopping
# lustre filesystem services.
#
-# Based in part on the XML obdctl modifications done by Brian Behlendorf
+# Based in part on the XML obdctl modifications done by Brian Behlendorf
-import sys, getopt, types
+import sys, getopt, types, errno
import string, os, stat, popen2, socket, time, random, fcntl, select
import re, exceptions, signal, traceback
import xml.dom.minidom
else:
from fcntl import F_GETFL, F_SETFL
-PYMOD_DIR = "/usr/lib/lustre/python"
+PYMOD_DIR = ["/usr/lib64/lustre/python", "/usr/lib/lustre/python"]
+PLATFORM = ''
+KEXTPATH = ''
+if string.find(sys.platform, 'linux') != -1:
+ PLATFORM='LINUX'
+elif string.find(sys.platform, 'darwin') != -1:
+ PLATFORM='DARWIN'
+ KEXTPATH='/System/Library/Extensions/'
+else:
+ PLATFORM='Unsupported'
def development_mode():
base = os.path.dirname(sys.argv[0])
if development_mode():
sys.path.append('../utils')
else:
- sys.path.append(PYMOD_DIR)
+ sys.path.extend(PYMOD_DIR)
import Lustre
# Global parameters
MAXTCPBUF = 16777216
-DEFAULT_TCPBUF = 8388608
-DEFAULT_PORT = 988
#
# Maximum number of devices to search for.
# (the /dev/loop* nodes need to be created beforehand)
MAX_LOOP_DEVICES = 256
-PORTALS_DIR = '../portals'
+PORTALS_DIR = '../lnet'
# Needed to call lconf --record
-CONFIG_FILE = ""
+CONFIG_FILE = ""
-# Please keep these in sync with the values in portals/kp30.h
-ptldebug_names = {
+# Please keep these in sync with the values in lnet/include/libcfs/libcfs.h
+ptldebug_names = {
"trace" : (1 << 0),
"inode" : (1 << 1),
"super" : (1 << 2),
"buffs" : (1 << 11),
"other" : (1 << 12),
"dentry" : (1 << 13),
- "portals" : (1 << 14),
+ "portals" : (1 << 14), # deprecated
+ "lnet" : (1 << 14),
"page" : (1 << 15),
"dlmtrace" : (1 << 16),
"error" : (1 << 17),
"console" : (1 << 25),
"quota" : (1 << 26),
"sec" : (1 << 27),
-}
+ }
subsystem_names = {
"undefined" : (1 << 0),
"log" : (1 << 6),
"llite" : (1 << 7),
"rpc" : (1 << 8),
- "mgmt" : (1 << 9),
- "portals" : (1 << 10),
- "nal" : (1 << 11),
+ "lnet" : (1 << 10),
+ "portals" : (1 << 10), # deprecated
+ "lnd" : (1 << 11),
+ "nal" : (1 << 11), # deprecated
"pinger" : (1 << 12),
"filter" : (1 << 13),
- "ptlbd" : (1 << 14),
+ "ptlbd" : (1 << 14), # deprecated
"echo" : (1 << 15),
"ldlm" : (1 << 16),
"lov" : (1 << 17),
- "ptlrouter" : (1 << 18),
+ "ptlrouter" : (1 << 18), # deprecated
"cobd" : (1 << 19),
"sm" : (1 << 20),
"asobd" : (1 << 21),
- "confobd" : (1 << 22),
+ "confobd" : (1 << 22), # deprecated
"lmv" : (1 << 23),
"cmobd" : (1 << 24),
"sec" : (1 << 25),
+ "sec" : (1 << 26),
+ "gss" : (1 << 27),
+ "gks" : (1 << 28),
+ "mgc" : (1 << 29),
+ "mgs" : (1 << 30),
}
if not first_cleanup_error:
first_cleanup_error = rc
-# ============================================================
+# ============================================================
# debugging and error funcs
def fixme(msg = "this feature"):
print string.strip(s)
def debug(*args):
+ # apparently, (non)execution of the following line affects mds device
+ # startup order (e.g. two mds's using loopback devices), so always do it.
+ msg = string.join(map(str,args))
if config.verbose:
- msg = string.join(map(str,args))
print msg
# ack, python's builtin int() does not support '0x123' syntax.
# eval can do it, although what a hack!
def my_int(s):
+ import types
+ if type(s) is types.IntType:
+ return s
try:
- if s[0:2] == '0x':
+ if (s[0:2] == '0x') or (s[0:1] == '0'):
return eval(s, {}, {})
else:
return int(s)
except SyntaxError, e:
raise ValueError("not a number")
+ except TypeError, e:
+ raise ValueError("not a number")
except NameError, e:
raise ValueError("not a number")
else:
print self.cmd_err
-
-# ============================================================
-# handle daemons, like the acceptor
-class DaemonHandler:
- """ Manage starting and stopping a daemon. Assumes daemon manages
- it's own pid file. """
-
- def __init__(self, cmd):
- self.command = cmd
- self.path =""
-
- def start(self):
- if self.running():
- log(self.command, "already running.")
- if not self.path:
- self.path = find_prog(self.command)
- if not self.path:
- panic(self.command, "not found.")
- ret, out = runcmd(self.path +' '+ self.command_line())
- if ret:
- raise CommandError(self.path, out, ret)
-
- def stop(self):
- if self.running():
- pid = self.read_pidfile()
- try:
- if pid != 1:
- log ("killing process", pid)
- os.kill(pid, 15)
- else:
- log("was unable to find pid of " + self.command)
- #time.sleep(1) # let daemon die
- except OSError, e:
- log("unable to kill", self.command, e)
- if self.running():
- log("unable to kill", self.command)
-
- def running(self):
- pid = self.read_pidfile()
- if pid:
- try:
- if pid != 1:
- os.kill(pid, 0)
- else:
- log("was unable to find pid of " + self.command)
- except OSError:
- self.clean_pidfile()
- else:
- return 1
- return 0
-
- def read_pidfile(self):
- try:
- fp = open(self.pidfile(), 'r')
- val = fp.read()
- if val == '':
- val = '1'
- pid = int(val)
- fp.close()
- return pid
- except IOError:
- return 0
-
- def clean_pidfile(self):
- """ Remove a stale pidfile """
- log("removing stale pidfile:", self.pidfile())
- try:
- os.unlink(self.pidfile())
- except OSError, e:
- log(self.pidfile(), e)
-
-class AcceptorHandler(DaemonHandler):
- def __init__(self, port, net_type):
- DaemonHandler.__init__(self, "acceptor")
- self.port = port
- self.flags = ''
-
- def pidfile(self):
- return "/var/run/%s-%d.pid" % (self.command, self.port)
-
- def command_line(self):
- return string.join(map(str,(self.flags, self.port)))
-
-acceptors = {}
-
-# start the acceptors
-def run_acceptors():
- if config.lctl_dump or config.record:
- return
- for port in acceptors.keys():
- daemon = acceptors[port]
- if not daemon.running():
- daemon.start()
-
-def run_one_acceptor(port):
- if config.lctl_dump or config.record:
- return
- if acceptors.has_key(port):
- daemon = acceptors[port]
- if not daemon.running():
- daemon.start()
- else:
- panic("run_one_acceptor: No acceptor defined for port:", port)
-
-def stop_acceptor(port):
- if acceptors.has_key(port):
- daemon = acceptors[port]
- if daemon.running():
- daemon.stop()
-
-
# ============================================================
# handle lctl interface
class LCTLInterface:
def use_save_file(self, file):
self.save_file = file
-
+
def record(self, dev_name, logname):
log("Recording log", logname, "on", dev_name)
self.record_device = dev_name
device $%s
record %s
%s""" % (self.record_device, self.record_log, cmds)
-
+
debug("+", cmd_line, cmds)
if config.noexec: return (0, [])
child = popen2.Popen3(cmd_line, 1) # Capture stdout and stderr from command
- child.tochild.write(cmds + "\n")
+ child.tochild.write(cmds + "\nq\n")
child.tochild.close()
-# print "LCTL:", cmds
# From "Python Cookbook" from O'Reilly
outfile = child.fromchild
raise CommandError(self.lctl, out, rc)
return rc, out
+ def unconfigure_network(self):
+ """get lnet to unreference itself"""
+ cmds = """
+ network unconfigure"""
+ self.run(cmds)
+
def clear_log(self, dev, log):
""" clear an existing log """
cmds = """
quit """ % (dev, log)
self.run(cmds)
- def root_squash(self, name, uid, nid):
- cmds = """
- device $%s
- root_squash %s %s
- quit""" % (name, uid, nid)
- self.run(cmds)
-
- def network(self, net, nid):
- """ set mynid """
- cmds = """
- network %s
- mynid %s
- quit """ % (net, nid)
- self.run(cmds)
-
- # add an interface
- def add_interface(self, net, ip, netmask = ""):
- """ add an interface """
- cmds = """
- network %s
- add_interface %s %s
- quit """ % (net, ip, netmask)
- self.run(cmds)
-
- # delete an interface
- def del_interface(self, net, ip):
- """ delete an interface """
- cmds = """
- network %s
- del_interface %s
- quit """ % (net, ip)
- self.run(cmds)
-
# create a new connection
def add_uuid(self, net_type, uuid, nid):
- cmds = "\n add_uuid %s %s %s" %(uuid, nid, net_type)
+ if net_type != 'lnet' and string.find(nid,'@') < 0:
+ nidstr = nid + "@" + net_type
+ else:
+ nidstr = nid
+ cmds = "\n add_uuid %s %s" %(uuid, nidstr)
self.run(cmds)
- def add_peer(self, net_type, nid, hostaddr, port):
- if net_type in ('tcp','openib','ra') and not config.lctl_dump:
- cmds = """
- network %s
- add_peer %s %s %d
- quit""" % (net_type,
- nid, hostaddr, port )
- self.run(cmds)
- elif net_type in ('iib',) and not config.lctl_dump:
- cmds = """
- network %s
- add_peer %s
- quit""" % (net_type,
- nid )
- self.run(cmds)
- elif net_type in ('vib',) and not config.lctl_dump:
- cmds = """
- network %s
- add_peer %s %s
- quit""" % (net_type,
- nid, hostaddr )
- self.run(cmds)
-
def connect(self, srv):
- self.add_uuid(srv.net_type, srv.nid_uuid, srv.nid)
- if srv.net_type in ('tcp','openib','iib','vib','ra') and not config.lctl_dump:
- if srv.hostaddr[0]:
- hostaddr = string.split(srv.hostaddr[0], '/')[0]
- self.add_peer(srv.net_type, srv.nid, hostaddr, srv.port)
+ if not srv.nid_uuid:
+ panic('nid_uuid not set for ', srv.net_type, srv.nid)
+ hostaddr = srv.db.get_hostaddr()
+ if len(hostaddr) > 1:
+ panic('multiple --hostaddr for ', srv.nid_uuid, ' not supported')
+ elif len(hostaddr) == 1 and hostaddr[0] != srv.nid:
+ panic('different --hostaddr and --nid for ', srv.nid_uuid, ' not supported')
+ else:
+ self.add_uuid(srv.net_type, srv.nid_uuid, srv.nid)
# Recover a device
def recover(self, dev_name, new_conn):
device $%s
recover %s""" %(dev_name, new_conn)
self.run(cmds)
-
- # add a route to a range
- def add_route(self, net, gw, lo, hi):
- cmds = """
- network %s
- add_route %s %s %s
- quit """ % (net,
- gw, lo, hi)
- try:
- self.run(cmds)
- except CommandError, e:
- log ("ignore: ")
- e.dump()
-
- def del_route(self, net, gw, lo, hi):
- cmds = """
- ignore_errors
- network %s
- del_route %s %s %s
- quit """ % (net, gw, lo, hi)
- self.run(cmds)
-
- # add a route to a host
- def add_route_host(self, net, uuid, gw, tgt):
- self.add_uuid(net, uuid, tgt)
- cmds = """
- network %s
- add_route %s %s
- quit """ % (net,
- gw, tgt)
- try:
- self.run(cmds)
- except CommandError, e:
- log ("ignore: ")
- e.dump()
-
- # add a route to a range
- def del_route_host(self, net, uuid, gw, tgt):
- self.del_uuid(uuid)
- cmds = """
- ignore_errors
- network %s
- del_route %s %s
- quit """ % (net, gw, tgt)
- self.run(cmds)
-
- def del_peer(self, net_type, nid, hostaddr):
- if net_type in ('tcp',) and not config.lctl_dump:
- cmds = """
- ignore_errors
- network %s
- del_peer %s %s single_share
- quit""" % (net_type,
- nid, hostaddr)
- self.run(cmds)
- elif net_type in ('openib','iib','vib','ra') and not config.lctl_dump:
- cmds = """
- ignore_errors
- network %s
- del_peer %s single_share
- quit""" % (net_type,
- nid)
- self.run(cmds)
-
# disconnect one connection
def disconnect(self, srv):
+ if not srv.nid_uuid:
+ panic('nid_uuid not set for ', srv.net_type, srv.nid)
self.del_uuid(srv.nid_uuid)
- if srv.net_type in ('tcp','openib','iib','vib','ra') and not config.lctl_dump:
- if srv.hostaddr[0]:
- hostaddr = string.split(srv.hostaddr[0], '/')[0]
- self.del_peer(srv.net_type, srv.nid, hostaddr)
def del_uuid(self, uuid):
cmds = """
quit""" % (uuid,)
self.run(cmds)
- # disconnect all
- def disconnectAll(self, net):
- cmds = """
- ignore_errors
- network %s
- disconnect
- quit""" % (net)
- self.run(cmds)
-
def attach(self, type, name, uuid):
cmds = """
attach %s %s %s
quit""" % (type, name, uuid)
self.run(cmds)
-
- def detach(self, name):
- cmds = """
- cfg_device %s
- detach
- quit""" % (name)
- self.run(cmds)
-
- def set_security(self, name, key, value):
- cmds = """
- cfg_device %s
- set_security %s %s
- quit""" % (name, key, value)
- self.run(cmds)
def setup(self, name, setup = ""):
cmds = """
setup %s
quit""" % (name, setup)
self.run(cmds)
-
+
+ def abort_recovery(self, name):
+ cmds = """
+ ignore_errors
+ device $%s
+ abort_recovery
+ quit""" % (name)
+ self.run(cmds)
+
def add_conn(self, name, conn_uuid):
cmds = """
cfg_device %s
quit""" % (name, conn_uuid)
self.run(cmds)
- def start(self, name, conf_name):
- cmds = """
- device $%s
- start %s
- quit""" % (name, conf_name)
- self.run(cmds)
-
# create a new device with lctl
def newdev(self, type, name, uuid, setup = ""):
self.attach(type, name, uuid);
except CommandError, e:
self.cleanup(name, uuid, 0)
raise e
-
+ if (config.abort_recovery):
+ if (type == 'obdfilter' or type == 'mds'):
+ self.abort_recovery(name)
+
# cleanup a device
def cleanup(self, name, uuid, force, failover = 0):
if failover: force = 1
self.run(cmds)
# create an lov
- def lov_setup(self, name, uuid, desc_uuid, stripe_cnt,
- stripe_sz, stripe_off, pattern, devlist = None):
+ def lov_setup(self, name, uuid, desc_uuid, mdsuuid, stripe_cnt,
+ stripe_sz, stripe_off, pattern):
cmds = """
attach lov %s %s
- lov_setup %s %d %d %d %s %s
- quit""" % (name, uuid, desc_uuid, stripe_cnt, stripe_sz, stripe_off,
- pattern, devlist)
+ lov_setup %s %d %d %d %s
+ quit""" % (name, uuid, desc_uuid, stripe_cnt, stripe_sz, stripe_off, pattern)
self.run(cmds)
# add an OBD to a LOV
def lov_add_obd(self, name, uuid, obd_uuid, index, gen):
cmds = """
+ cfg_device %s
lov_modify_tgts add %s %s %s %s
- quit""" % (name, obd_uuid, index, gen)
- self.run(cmds)
-
- # create an lmv
- def lmv_setup(self, name, uuid, desc_uuid, devlist):
- cmds = """
- attach lmv %s %s
- lmv_setup %s %s
- quit""" % (name, uuid, desc_uuid, devlist)
+ quit""" % (name, name, obd_uuid, index, gen)
self.run(cmds)
# delete an OBD from a LOV
def lov_del_obd(self, name, uuid, obd_uuid, index, gen):
cmds = """
+ cfg_device %s
lov_modify_tgts del %s %s %s %s
- quit""" % (name, obd_uuid, index, gen)
+ quit""" % (name, name, obd_uuid, index, gen)
self.run(cmds)
# deactivate an OBD
def deactivate(self, name):
cmds = """
- device $%s
+ cfg_device %s
deactivate
quit""" % (name)
self.run(cmds)
# get list of devices
def device_list(self):
- devices = '/proc/fs/lustre/devices'
ret = []
- if os.access(devices, os.R_OK):
- try:
- fp = open(devices, 'r')
- ret = fp.readlines()
- fp.close()
- except IOError, e:
- log(e)
+ if PLATFORM == 'LINUX':
+ devices = '/proc/fs/lustre/devices'
+ if os.access(devices, os.R_OK):
+ try:
+ fp = open(devices, 'r')
+ ret = fp.readlines()
+ fp.close()
+ except IOError, e:
+ log(e)
+ elif PLATFORM == 'DARWIN':
+ rc, out = self.run("device_list")
+ ret = out.split("\n")
+ if len(ret) == 0:
+ return ret
+ tail = ret[-1]
+ if not tail:
+ # remove the last empty line
+ ret = ret[:-1]
return ret
# get lustre version
quit""" % (timeout,)
self.run(cmds)
+ # set lustre upcall
def set_lustre_upcall(self, upcall):
cmds = """
set_lustre_upcall %s
if module:
return module
+def find_module(src_dir, dev_dir, modname):
+ modbase = src_dir +'/'+ dev_dir +'/'+ modname
+ for modext in '.ko', '.o':
+ module = modbase + modext
+ try:
+ if os.access(module, os.R_OK):
+ return module
+ except OSError:
+ pass
+ return None
+
# is the path a block device?
def is_block(path):
s = ()
return 0
return stat.S_ISBLK(s[stat.ST_MODE])
-# find the journal device from mkfs options
-def jdev(opts):
- if opts == None:
- return ''
- x=string.split(opts)
- i=0
- while i < len(x) - 1:
- if x[i] == '-J' and x[i+1].startswith('device='):
- str=x[i+1]
- return str[7:]
- i=i+1
- return ''
+def my_realpath(path):
+ try:
+ if os.path.islink(path):
+ # get the realpath of the mount point path
+ if 'realpath' in dir(os.path):
+ real_path = os.path.realpath(path)
+ else:
+ real_path = path
+ link_count = 0
+ while os.path.islink(real_path) and (link_count < 20):
+ link_count = link_count + 1
+ path_link = os.readlink(real_path)
+ if os.path.isabs(path_link):
+ real_path = path_link
+ else:
+ real_path = os.path.join(os.path.dirname(real_path), path_link)
+ if link_count > 19:
+ panic("Encountered too many symbolic links resolving path:", path)
+ else:
+ real_path = path
+
+ return real_path
+ except:
+ panic("Fatal error realpath()ing path:", path)
+
# build fs according to type
# fixme: dangerous
# devsize is in 1k, and fs block count is in 4k
block_cnt = devsize/4
- if fstype in ('ext3', 'extN', 'ldiskfs'):
+ if fstype in ('ext3', 'ldiskfs'):
# ext3 journal size is in megabytes
- # but don't set jsize if mkfsoptions indicates a separate journal device
- if jsize == 0 and jdev(mkfsoptions) == '':
+ if jsize == 0:
if devsize == 0:
if not is_block(dev):
ret, out = runcmd("ls -l %s" %dev)
# get the realpath of the device
# it may be the real device, such as /dev/hda7
# or the hardlink created via mknod for a device
- if 'realpath' in dir(os.path):
- real_dev = os.path.realpath(dev)
- else:
- real_dev = dev
- link_count = 0
- while os.path.islink(real_dev) and (link_count < 20):
- link_count = link_count + 1
- dev_link = os.readlink(real_dev)
- if os.path.isabs(dev_link):
- real_dev = dev_link
- else:
- real_dev = os.path.join(os.path.dirname(real_dev), dev_link)
- if link_count > 19:
- panic("Entountered too many symbolic links resolving block device:", dev)
+ real_dev = my_realpath(dev)
# get the major and minor number of the realpath via ls
- # it seems python(os.stat) does not return
+ # it seems python(os.stat) does not return
# the st_rdev member of the stat structure
ret, out = runcmd("ls -l %s" %real_dev)
major = string.split(string.split(out[0])[4], ",")[0]
if devsize > 1024 * 1024:
jsize = ((devsize / 102400) * 4)
if jsize > 400:
- jsize = 400
+ jsize = 400
if jsize: jopt = "-J size=%d" %(jsize,)
if isize: iopt = "-I %d" %(isize,)
mkfs = 'mkfs.ext2 -j -b 4096 '
if not isblock or config.force:
mkfs = mkfs + ' -F '
- if jdev(mkfsoptions) != '':
- jmkfs = 'mkfs.ext2 -b 4096 -O journal_dev '
- if config.force:
- jmkfs = jmkfs + '-F '
- jmkfs = jmkfs + jdev(mkfsoptions)
- (ret, out) = run (jmkfs)
- if ret:
- panic("Unable format journal device:", jdev(mkfsoptions), string.join(out))
elif fstype == 'reiserfs':
# reiserfs journal size is in blocks
if jsize: jopt = "--journal_size %d" %(jsize,)
if ret:
panic("Unable to build fs:", dev, string.join(out))
# enable hash tree indexing on fsswe
- if fstype in ('ext3', 'extN', 'ldiskfs'):
- htree = 'echo "feature FEATURE_C5" | debugfs -w'
+ if fstype in ('ext3', 'ldiskfs'):
+ htree = 'tune2fs -O dir_index'
(ret, out) = run (htree, dev)
if ret:
panic("Unable to enable htree:", dev)
if not os.access(loop + str(0), os.R_OK):
loop = loop + '/'
if not os.access(loop + str(0), os.R_OK):
- panic ("can't access loop devices")
+ loop='/dev/loop'
return loop
-
+
# find loop device assigned to the file
-def find_assigned_loop(file):
+def find_loop(file):
loop = loop_base()
for n in xrange(0, MAX_LOOP_DEVICES):
dev = loop + str(n)
m = re.search(r'\((.*)\)', out[0])
if m and file == m.group(1):
return dev
+ else:
+ break
return ''
-# find free loop device
-def find_free_loop(file):
+# create file if necessary and assign the first free loop device
+def init_loop(file, size, fstype, journal_size, inode_size, mkfsoptions, reformat):
+ dev = find_loop(file)
+ if dev:
+ print 'WARNING file:', file, 'already mapped to', dev
+ return dev
+ if reformat or not os.access(file, os.R_OK | os.W_OK):
+ if size < 8000:
+ panic("size of loopback file '%s' must be larger than 8MB, but is set to %s" % (file,size))
+ (ret, out) = run("dd if=/dev/zero bs=1k count=0 seek=%d of=%s" %(size,
+ file))
+ if ret:
+ panic("Unable to create backing store:", file)
+ mkfs(file, size, fstype, journal_size, inode_size, mkfsoptions, isblock=0)
+
loop = loop_base()
-
# find next free loop
for n in xrange(0, MAX_LOOP_DEVICES):
dev = loop + str(n)
if os.access(dev, os.R_OK):
(stat, out) = run('losetup', dev)
if stat:
+ (stat, out) = run('losetup', dev, file)
+ if stat:
+ panic("losetup failed: (%s) %s" % (stat, out[0].strip()))
return dev
+ else:
+ print "out of loop devices"
+ return ''
+ print "out of loop devices"
return ''
-# create file if necessary and assign the first free loop device
-def init_loop(file, size, fstype, journal_size, inode_size,
- mkfsoptions, reformat, autoformat, backfstype, backfile):
- if fstype == 'smfs':
- realfile = backfile
- realfstype = backfstype
- if is_block(backfile):
- if reformat or (need_format(realfstype, backfile) and autoformat == 'yes'):
- mkfs(realfile, size, realfstype, journal_size, inode_size, mkfsoptions, isblock=0)
- return realfile
- else:
- realfile = file
- realfstype = fstype
-
- dev = find_assigned_loop(realfile)
+# undo loop assignment
+def clean_loop(file):
+ dev = find_loop(file)
if dev:
- print 'WARNING: file', realfile, 'already mapped to', dev
- return dev
-
- if reformat or not os.access(realfile, os.R_OK | os.W_OK):
- (ret, out) = run("dd if=/dev/zero bs=1k count=0 seek=%d of=%s" %(size, realfile))
+ ret, out = run('losetup -d', dev)
if ret:
- panic("Unable to create backing store:", realfile)
- mkfs(realfile, size, realfstype, journal_size, inode_size,
- mkfsoptions, isblock=0)
-
- dev = find_free_loop(realfile)
- if dev:
- print "attach " + realfile + " <-> " + dev
- run('losetup', dev, realfile)
- return dev
-
- print "out of loop devices"
- return ''
+ log('unable to clean loop device:', dev, 'for file:', file)
+ logall(out)
-# undo loop assignment
-def clean_loop(dev, fstype, backfstype, backdev):
- if fstype == 'smfs':
- realfile = backdev
- else:
- realfile = dev
- if not is_block(realfile):
- dev = find_assigned_loop(realfile)
- if dev:
- print "detach " + dev + " <-> " + realfile
- ret, out = run('losetup -d', dev)
- if ret:
- log('unable to clean loop device', dev, 'for file', realfile)
- logall(out)
-
-# finilizes passed device
-def clean_dev(dev, fstype, backfstype, backdev):
- if fstype == 'smfs' or not is_block(dev):
- clean_loop(dev, fstype, backfstype, backdev)
-
# determine if dev is formatted as a <fstype> filesystem
def need_format(fstype, dev):
- # FIXME don't know how to implement this
+ # FIXME don't know how to implement this
return 0
# initialize a block device if needed
def block_dev(dev, size, fstype, reformat, autoformat, journal_size,
- inode_size, mkfsoptions, backfstype, backdev):
- if config.noexec:
- return dev
-
- if fstype == 'smfs' or not is_block(dev):
+ inode_size, mkfsoptions):
+ if config.noexec: return dev
+ if not is_block(dev):
dev = init_loop(dev, size, fstype, journal_size, inode_size,
- mkfsoptions, reformat, autoformat, backfstype, backdev)
+ mkfsoptions, reformat)
elif reformat or (need_format(fstype, dev) and autoformat == 'yes'):
mkfs(dev, size, fstype, journal_size, inode_size, mkfsoptions,
isblock=0)
# panic("device:", dev,
# "not prepared, and autoformat is not set.\n",
# "Rerun with --reformat option to format ALL filesystems")
-
return dev
def if2addr(iface):
ip = string.split(addr, ':')[1]
return ip
-def def_mount_options(fstype, target):
+def def_mount_options(fstype, target, blkdev):
"""returns deafult mount options for passed fstype and target (mds, ost)"""
if fstype == 'ext3' or fstype == 'ldiskfs':
mountfsoptions = "errors=remount-ro"
- if target == 'ost' and sys_get_branch() == '2.4':
- mountfsoptions = "%s,asyncdel" % (mountfsoptions)
- return mountfsoptions
- return ""
+ if target == 'ost':
+ if sys_get_branch() == '2.4':
+ mountfsoptions = "%s,asyncdel" % (mountfsoptions)
+ #else:
+ # mountfsoptions = "%s,extents,mballoc" % (mountfsoptions)
+ elif target == 'mds':
+ if config.user_xattr:
+ mountfsoptions = "%s,user_xattr" % (mountfsoptions)
+ if config.acl:
+ mountfsoptions = "%s,acl" % (mountfsoptions)
+
+ if blkdev:
+ # grab superblock info
+ dumpe2fs="dumpe2fs -f -h"
+ (ret, sb) = run(dumpe2fs, blkdev)
+ if ret:
+ panic("unable to get superblock for ", blkdev)
+
+ # extract journal UUID
+ journal_UUID=''
+ journal_DEV=''
+ for line in sb:
+ lst = string.split(line, ":")
+ if lst[0] == 'Journal UUID':
+ if len(lst[1]) < 3:
+ panic("cannot retrieve journal UUID for ", blkdev)
+ if string.split(lst[1])[0] != '<none>':
+ journal_UUID = string.split(lst[1])[0]
+ debug(blkdev, 'has journal UUID', journal_UUID)
+ if lst[0] == 'Journal device':
+ if len(lst[1]) < 3:
+ panic("cannot retrieve journal device for ", blkdev)
+ if string.split(lst[1])[0] != '0x0000':
+ journal_DEV = string.split(lst[1])[0]
+ debug(blkdev, 'has journal device', journal_DEV)
+ break
+
+ if len(journal_UUID) == 0 or len(journal_DEV) == 0:
+ debug('no external journal found for', blkdev)
+ # use internal journal
+ return mountfsoptions
-def sys_get_elan_position_file():
- procfiles = ["/proc/elan/device0/position",
- "/proc/qsnet/elan4/device0/position",
- "/proc/qsnet/elan3/device0/position"]
- for p in procfiles:
- if os.access(p, os.R_OK):
- return p
+ # run blkid, lookup highest-priority device with matching UUID
+ blkid = "blkid -o device -l -t UUID='%s'" % (journal_UUID)
+ (ret, devname) = run(blkid)
+ if ret or len(devname) == 0:
+ panic("cannot find external journal for ", blkdev)
+ debug('found', blkdev, 'journal UUID', journal_UUID, 'on',
+ string.replace(devname[0], '\n', ''))
+
+ try: # sigh, python 1.5 does not support os.stat().st_rdev
+ jdevpath = my_realpath(string.replace(devname[0], '\n', ''))
+ ret, out = runcmd("ls -l %s" %jdevpath)
+ debug('ls -l:', out)
+ major = int(string.split(string.split(out[0])[4], ',')[0])
+ minor = int(string.split(out[0])[5])
+ debug('major', major, 'minor', minor)
+ rdev = major << 8 | minor
+ except OSError:
+ panic("cannot stat ", devname[0])
+
+ debug('found', blkdev, 'journal UUID', journal_UUID, 'on',
+ jdevpath, 'rdev', rdev)
+
+ # add mount option
+ if string.atoi(journal_DEV, 0) != rdev:
+ mountfsoptions = "%s,journal_dev=%#x" % (mountfsoptions,rdev)
+
+ return mountfsoptions
return ""
-def sys_get_local_nid(net_type, wildcard, cluster_id):
- """Return the local nid."""
- local = ""
- if sys_get_elan_position_file():
- local = sys_get_local_address('elan', '*', cluster_id)
- else:
- local = sys_get_local_address(net_type, wildcard, cluster_id)
- return local
-
-def sys_get_local_address(net_type, wildcard, cluster_id):
- """Return the local address for the network type."""
- local = ""
- if net_type in ('tcp','openib','iib','vib','ra'):
- if ':' in wildcard:
- iface, star = string.split(wildcard, ':')
- local = if2addr(iface)
- if not local:
- panic ("unable to determine ip for:", wildcard)
- else:
- host = socket.gethostname()
- local = socket.gethostbyname(host)
- elif net_type == 'elan':
- # awk '/NodeId/ { print $2 }' 'sys_get_elan_position_file()'
- f = sys_get_elan_position_file()
- if not f:
- panic ("unable to determine local Elan ID")
+def sys_get_branch():
+ """Returns kernel release"""
+ return os.uname()[2][:3]
+
+def mod_loaded(modname):
+ """Check if a module is already loaded. Look in /proc/modules for it."""
+ if PLATFORM == 'LINUX':
try:
- fp = open(f, 'r')
+ fp = open('/proc/modules')
lines = fp.readlines()
fp.close()
- for l in lines:
- a = string.split(l)
- if a[0] == 'NodeId':
- elan_id = a[1]
- break
- try:
- nid = my_int(cluster_id) + my_int(elan_id)
- local = "%d" % (nid)
- except ValueError, e:
- local = elan_id
- except IOError, e:
- log(e)
- elif net_type == 'lo':
- fixme("automatic local address for loopback")
- elif net_type == 'gm':
- fixme("automatic local address for GM")
-
- return local
-
-def sys_get_branch():
- """Returns kernel release"""
- try:
- fp = open('/proc/sys/kernel/osrelease')
- lines = fp.readlines()
- fp.close()
-
- for l in lines:
- version = string.split(l)
- a = string.split(version[0], '.')
- return a[0] + '.' + a[1]
- except IOError, e:
- log(e)
- return ""
+ # please forgive my tired fingers for this one
+ ret = filter(lambda word, mod=modname: word == mod,
+ map(lambda line: string.split(line)[0], lines))
+ return ret
+ except Exception, e:
+ return 0
+ elif PLATFORM == 'DARWIN':
+ ret, out = run('/usr/sbin/kextstat | /usr/bin/grep', modname)
+ if ret == 0:
+ return 1
+ else:
+ return 0
+ else:
+ return 0
# XXX: instead of device_list, ask for $name and see what we get
def is_prepared(name):
e.dump()
return 0
-def net_is_prepared():
+def is_network_prepared():
"""If the any device exists, then assume that all networking
has been configured"""
out = lctl.device_list()
def fs_is_mounted(path):
"""Return true if path is a mounted lustre filesystem"""
try:
+ real_path = my_realpath(path)
+
fp = open('/proc/mounts')
lines = fp.readlines()
fp.close()
for l in lines:
a = string.split(l)
- if a[1] == path and a[2] == 'lustre_lite':
+ if a[1] == real_path and a[2] == 'lustre_lite':
return 1
except IOError, e:
log(e)
return 0
-def kmod_find(src_dir, dev_dir, modname):
- modbase = src_dir +'/'+ dev_dir +'/'+ modname
- for modext in '.ko', '.o':
- module = modbase + modext
- try:
- if os.access(module, os.R_OK):
- return module
- except OSError:
- pass
- return None
-
-def kmod_info(modname):
- """Returns reference count for passed module name."""
- try:
- fp = open('/proc/modules')
- lines = fp.readlines()
- fp.close()
-
- # please forgive my tired fingers for this one
- ret = filter(lambda word, mod = modname: word[0] == mod,
- map(lambda line: string.split(line), lines))
- if not ret:
- return ''
- return ret[0]
- except Exception, e:
- return 0
-
class kmod:
- """Presents kernel module"""
- def __init__(self, src_dir, dev_dir, name):
- self.src_dir = src_dir
- self.dev_dir = dev_dir
- self.name = name
-
- # FIXME we ignore the failure of loading gss module, because we might
- # don't need it at all.
- def load(self):
- """Load module"""
- log ('loading module:', self.name, 'srcdir',
- self.src_dir, 'devdir', self.dev_dir)
- if self.src_dir:
- module = kmod_find(self.src_dir, self.dev_dir,
- self.name)
- if not module and self.name != 'ptlrpcs_gss':
- panic('module not found:', self.name)
- (rc, out) = run('/sbin/insmod', module)
- if rc:
- if self.name == 'ptlrpcs_gss':
- print "Warning: not support gss security!"
- else:
- raise CommandError('insmod', out, rc)
- else:
- (rc, out) = run('/sbin/modprobe', self.name)
- if rc:
- if self.name == 'ptlrpcs_gss':
- print "Warning: not support gss security!"
- else:
- raise CommandError('modprobe', out, rc)
-
- def cleanup(self):
- """Unload module"""
- log('unloading module:', self.name)
- (rc, out) = run('/sbin/rmmod', self.name)
- if rc:
- log('unable to unload module:', self.name +
- "(" + self.refcount() + ")")
- logall(out)
-
- def info(self):
- """Returns module info if any."""
- return kmod_info(self.name)
-
- def loaded(self):
- """Returns 1 if module is loaded. Otherwise 0 is returned."""
- if self.info():
- return 1
- else:
- return 0
-
- def refcount(self):
- """Returns module refcount."""
- info = self.info()
- if not info:
- return ''
- return info[2]
-
- def used(self):
- """Returns 1 if module is used, otherwise 0 is returned."""
- info = self.info()
- if not info:
- return 0
- if len(info) > 3:
- users = info[3]
- if users and users != '(unused)' and users != '-':
- return 1
- else:
- return 0
- else:
- return 0
-
- def busy(self):
- """Returns 1 if module is busy, otherwise 0 is returned."""
- if self.loaded() and (self.used() or self.refcount() != '0'):
- return 1
- else:
- return 0
-
-class kmod_manager:
"""Manage kernel modules"""
def __init__(self, lustre_dir, portals_dir):
self.lustre_dir = lustre_dir
self.portals_dir = portals_dir
self.kmodule_list = []
- def find_module(self, modname):
- """Find module by module name"""
- for mod in self.kmodule_list:
- if mod.name == modname:
- return mod
- return ''
-
def add_portals_module(self, dev_dir, modname):
"""Append a module to list of modules to load."""
-
- mod = self.find_module(modname)
- if not mod:
- mod = kmod(self.portals_dir, dev_dir, modname)
- self.kmodule_list.append(mod)
+ self.kmodule_list.append((self.portals_dir, dev_dir, modname))
def add_lustre_module(self, dev_dir, modname):
"""Append a module to list of modules to load."""
+ self.kmodule_list.append((self.lustre_dir, dev_dir, modname))
- mod = self.find_module(modname)
- if not mod:
- mod = kmod(self.lustre_dir, dev_dir, modname)
- self.kmodule_list.append(mod)
-
- def load_modules(self):
+ def load_module(self):
"""Load all the modules in the list in the order they appear."""
- for mod in self.kmodule_list:
- if mod.loaded() and not config.noexec:
+ for src_dir, dev_dir, mod in self.kmodule_list:
+ if mod_loaded(mod) and not config.noexec:
continue
- mod.load()
-
- def cleanup_modules(self):
+ log ('loading module:', mod, 'srcdir', src_dir, 'devdir', dev_dir)
+ if PLATFORM == 'LINUX':
+ options = ''
+ if mod == 'lnet':
+ #For LNET we really need modprobe to load defined LNDs
+ run('/sbin/modprobe lnet')
+ #But if that fails, try insmod anyhow with dev option
+ #accept=all for dev liblustre testing
+ options = 'accept=all'
+ if src_dir:
+ module = find_module(src_dir, dev_dir, mod)
+ if not module:
+ panic('module not found:', mod)
+ (rc, out) = run('/sbin/insmod', module, options)
+ if rc and not mod_loaded(mod):
+ if rc == 1:
+ print("Bad module options? Check dmesg.")
+ raise CommandError('insmod', out, rc)
+ else:
+ (rc, out) = run('/sbin/modprobe', mod)
+ if rc and not mod_loaded(mod):
+ if rc == 1:
+ print("Bad module options? Check dmesg.")
+ raise CommandError('modprobe', out, rc)
+ elif PLATFORM == 'DARWIN':
+ run('/sbin/kextload', KEXTPATH + mod + '.kext');
+
+ def cleanup_module(self):
"""Unload the modules in the list in reverse order."""
- rev = self.kmodule_list
+
+ rev = self.kmodule_list[:] # make *copy* of list
rev.reverse()
- for mod in rev:
- if (not mod.loaded() or mod.busy()) and not config.noexec:
+ for src_dir, dev_dir, mod in rev:
+ if not mod_loaded(mod) and not config.noexec:
continue
- # debug hack
- if mod.name == 'portals' and config.dump:
- lctl.dump(config.dump)
- mod.cleanup()
-
+ if mod == 'ksocklnd' and not config.noexec:
+ # Ignore ksocklnd in module list (lnet will remove)
+ continue
+ log('unloading module:', mod)
+ if mod == 'lnet' and not config.noexec:
+ # remove any self-ref portals created
+ lctl.unconfigure_network()
+ if config.dump:
+ debug('dumping debug log to', config.dump)
+ # debug hack
+ lctl.dump(config.dump)
+ log('unloading the network')
+ lctl.unconfigure_network()
+ if mod_loaded("ksocklnd"):
+ if PLATFORM == 'LINUX':
+ run('/sbin/rmmod ksocklnd')
+ elif PLATFORM == 'DARWIN':
+ run('/sbin/kextunload', KEXTPATH+'ksocklnd.kext')
+ if mod_loaded("kqswlnd"):
+ run('/sbin/rmmod kqswlnd')
+ if mod_loaded("kgmlnd"):
+ run('/sbin/rmmod kgmlnd')
+ if mod_loaded("kopeniblnd"):
+ run('/sbin/rmmod kopeniblnd')
+ if mod_loaded("kiiblnd"):
+ run('/sbin/rmmod kiiblnd')
+ if mod_loaded("kviblnd"):
+ run('/sbin/rmmod kviblnd')
+ if mod_loaded("kciblnd"):
+ run('/sbin/rmmod kciblnd')
+ if mod_loaded("ko2iblnd"):
+ run('/sbin/rmmod ko2iblnd')
+ if mod_loaded("kralnd"):
+ run('/sbin/rmmod kralnd')
+ if mod_loaded("kptllnd"):
+ run('/sbin/rmmod kptllnd')
+ if PLATFORM == 'LINUX':
+ (rc, out) = run('/sbin/rmmod', mod)
+ elif PLATFORM == 'DARWIN':
+ (rc, out) = run('/sbin/kextunload', KEXTPATH+mod+'.kext');
+ if rc:
+ log('! unable to unload module:', mod)
+ logall(out)
+
+
# ============================================================
# Classes to prepare and cleanup the various objects
#
self.uuid = self.db.getUUID()
self._server = None
self._connected = 0
+ self.kmod = kmod(config.lustre, config.portals)
def info(self, *args):
msg = string.join(map(str,args))
- print self.module_name + ":", self.name, self.uuid, msg
+ log (self.module_name + ":", self.name, self.uuid, msg)
def cleanup(self):
""" default cleanup, used for most modules """
e.dump()
cleanup_error(e.rc)
- def add_module(self, manager):
- """Adds all needed modules in the order they appear."""
- return
+ def add_portals_module(self, dev_dir, modname):
+ """Append a module to list of modules to load."""
+ self.kmod.add_portals_module(dev_dir, modname)
+
+ def add_lustre_module(self, dev_dir, modname):
+ """Append a module to list of modules to load."""
+ self.kmod.add_lustre_module(dev_dir, modname)
+
+ def load_module(self):
+ """Load all the modules in the list in the order they appear."""
+ self.kmod.load_module()
+
+ def cleanup_module(self):
+ """Unload the modules in the list in reverse order."""
+ if self.safe_to_clean():
+ self.kmod.cleanup_module()
def safe_to_clean(self):
return 1
def safe_to_clean_modules(self):
return self.safe_to_clean()
-
+
class Network(Module):
- def __init__(self,db):
+ def __init__(self,db,nid_uuid=0):
Module.__init__(self, 'NETWORK', db)
self.net_type = self.db.get_val('nettype')
self.nid = self.db.get_val('nid', '*')
self.cluster_id = self.db.get_val('clusterid', "0")
self.port = self.db.get_val_int('port', 0)
-
- if '*' in self.nid:
- self.nid = sys_get_local_nid(self.net_type, self.nid, self.cluster_id)
- if not self.nid:
- panic("unable to set nid for", self.net_type, self.nid, cluster_id)
- self.generic_nid = 1
- debug("nid:", self.nid)
- else:
- self.generic_nid = 0
-
- self.nid_uuid = self.nid_to_uuid(self.nid)
- self.hostaddr = self.db.get_hostaddr()
- if len(self.hostaddr) == 0:
- self.hostaddr.append(self.nid)
- if '*' in self.hostaddr[0]:
- self.hostaddr[0] = sys_get_local_address(self.net_type, self.hostaddr[0], self.cluster_id)
- if not self.hostaddr[0]:
- panic("unable to set hostaddr for", self.net_type, self.hostaddr[0], self.cluster_id)
- debug("hostaddr:", self.hostaddr[0])
-
- def add_module(self, manager):
- manager.add_portals_module("libcfs", 'libcfs')
- manager.add_portals_module("portals", 'portals')
-
- if node_needs_router():
- manager.add_portals_module("router", 'kptlrouter')
- if self.net_type == 'tcp':
- manager.add_portals_module("knals/socknal", 'ksocknal')
- if self.net_type == 'elan':
- manager.add_portals_module("knals/qswnal", 'kqswnal')
- if self.net_type == 'gm':
- manager.add_portals_module("knals/gmnal", 'kgmnal')
- if self.net_type == 'openib':
- manager.add_portals_module("knals/openibnal", 'kopenibnal')
- if self.net_type == 'iib':
- manager.add_portals_module("knals/iibnal", 'kiibnal')
- if self.net_type == 'vib':
- self.add_portals_module("knals/vibnal", 'kvibnal')
- if self.net_type == 'lo':
- manager.add_portals_module("knals/lonal", 'klonal')
- if self.net_type == 'ra':
- manager.add_portals_module("knals/ranal", 'kranal')
-
- def nid_to_uuid(self, nid):
- return "NID_%s_UUID" %(nid,)
+ self.nid_uuid = nid_uuid
+ self.add_portals_module('libcfs', 'libcfs')
+ self.add_portals_module('lnet', 'lnet')
+ # Add the socklnd for developers without modprobe.conf (umls)
+ self.add_portals_module('klnds/socklnd', 'ksocklnd')
def prepare(self):
- if not config.record and net_is_prepared():
+ if is_network_prepared():
return
- self.info(self.net_type, self.nid, self.port)
- if not (config.record and self.generic_nid):
- lctl.network(self.net_type, self.nid)
+ self.info(self.net_type, self.nid)
if self.net_type == 'tcp':
sys_tweak_socknal()
- for hostaddr in self.db.get_hostaddr():
- ip = string.split(hostaddr, '/')[0]
- if len(string.split(hostaddr, '/')) == 2:
- netmask = string.split(hostaddr, '/')[1]
- else:
- netmask = ""
- lctl.add_interface(self.net_type, ip, netmask)
if self.net_type == 'elan':
sys_optimize_elan()
- if self.port and node_is_router():
- run_one_acceptor(self.port)
- self.connect_peer_gateways()
-
- def connect_peer_gateways(self):
- for router in self.db.lookup_class('node'):
- if router.get_val_int('router', 0):
- for netuuid in router.get_networks():
- net = self.db.lookup(netuuid)
- gw = Network(net)
- if (gw.cluster_id == self.cluster_id and
- gw.net_type == self.net_type):
- if gw.nid != self.nid:
- lctl.connect(gw)
-
- def disconnect_peer_gateways(self):
- for router in self.db.lookup_class('node'):
- if router.get_val_int('router', 0):
- for netuuid in router.get_networks():
- net = self.db.lookup(netuuid)
- gw = Network(net)
- if (gw.cluster_id == self.cluster_id and
- gw.net_type == self.net_type):
- if gw.nid != self.nid:
- try:
- lctl.disconnect(gw)
- except CommandError, e:
- print "disconnect failed: ", self.name
- e.dump()
- cleanup_error(e.rc)
-
- def safe_to_clean(self):
- return not net_is_prepared()
-
- def cleanup(self):
- self.info(self.net_type, self.nid, self.port)
- if self.port:
- stop_acceptor(self.port)
- if node_is_router():
- self.disconnect_peer_gateways()
- if self.net_type == 'tcp':
- for hostaddr in self.db.get_hostaddr():
- ip = string.split(hostaddr, '/')[0]
- lctl.del_interface(self.net_type, ip)
-
- def correct_level(self, level, op=None):
- return level
-
-class RouteTable(Module):
- def __init__(self,db):
- Module.__init__(self, 'ROUTES', db)
-
- def server_for_route(self, net_type, gw, gw_cluster_id, tgt_cluster_id,
- lo, hi):
- # only setup connections for tcp, openib, and iib NALs
- srvdb = None
- if not net_type in ('tcp','openib','iib','vib','ra'):
- return None
-
- # connect to target if route is to single node and this node is the gw
- if lo == hi and local_interface(net_type, gw_cluster_id, gw):
- if not local_cluster(net_type, tgt_cluster_id):
- panic("target", lo, " not on the local cluster")
- srvdb = self.db.nid2server(lo, net_type, gw_cluster_id)
- # connect to gateway if this node is not the gw
- elif (local_cluster(net_type, gw_cluster_id)
- and not local_interface(net_type, gw_cluster_id, gw)):
- srvdb = self.db.nid2server(gw, net_type, gw_cluster_id)
- else:
- return None
-
- if not srvdb:
- panic("no server for nid", lo)
- return None
-
- return Network(srvdb)
-
- def prepare(self):
- if not config.record and net_is_prepared():
- return
- self.info()
- for net_type, gw, gw_cluster_id, tgt_cluster_id, lo, hi in self.db.get_route_tbl():
- lctl.add_route(net_type, gw, lo, hi)
- srv = self.server_for_route(net_type, gw, gw_cluster_id, tgt_cluster_id, lo, hi)
- if srv:
- lctl.connect(srv)
-
- def safe_to_clean(self):
- return not net_is_prepared()
-
- def cleanup(self):
- if net_is_prepared():
- # the network is still being used, don't clean it up
- return
- for net_type, gw, gw_cluster_id, tgt_cluster_id, lo, hi in self.db.get_route_tbl():
- srv = self.server_for_route(net_type, gw, gw_cluster_id, tgt_cluster_id, lo, hi)
- if srv:
- try:
- lctl.disconnect(srv)
- except CommandError, e:
- print "disconnect failed: ", self.name
- e.dump()
- cleanup_error(e.rc)
-
- try:
- lctl.del_route(net_type, gw, lo, hi)
- except CommandError, e:
- print "del_route failed: ", self.name
- e.dump()
- cleanup_error(e.rc)
-
-class Management(Module):
- def __init__(self, db):
- Module.__init__(self, 'MGMT', db)
-
- def add_module(self, manager):
- manager.add_lustre_module('lvfs', 'lvfs')
- manager.add_lustre_module('obdclass', 'obdclass')
- manager.add_lustre_module('ptlrpc', 'ptlrpc')
- manager.add_lustre_module('mgmt', 'mgmt_svc')
-
- def prepare(self):
- if not config.record and is_prepared(self.name):
- return
- self.info()
- lctl.newdev("mgmt", self.name, self.uuid)
def safe_to_clean(self):
+ if PLATFORM == 'LINUX':
+ return not is_network_prepared()
+ elif PLATFORM == 'DARWIN':
+ # XXX always assume it's safe to clean
+ return 1
return 1
def cleanup(self):
- if is_prepared(self.name):
- Module.cleanup(self)
-
- def correct_level(self, level, op=None):
- return level
+ self.info(self.net_type, self.nid)
# This is only needed to load the modules; the LDLM device
# is now created automatically.
class LDLM(Module):
def __init__(self,db):
Module.__init__(self, 'LDLM', db)
-
- def add_module(self, manager):
- manager.add_lustre_module('lvfs', 'lvfs')
- manager.add_lustre_module('obdclass', 'obdclass')
- manager.add_lustre_module('sec', 'ptlrpcs')
- manager.add_lustre_module('ptlrpc', 'ptlrpc')
- manager.add_lustre_module('sec/gss', 'ptlrpcs_gss')
+ self.add_lustre_module('lvfs', 'lvfs')
+ self.add_lustre_module('obdclass', 'obdclass')
+ self.add_lustre_module('ptlrpc', 'ptlrpc')
+ self.add_lustre_module('ptlrpc/gss', 'ptlrpc_gss')
def prepare(self):
return
def cleanup(self):
return
- def correct_level(self, level, op=None):
- return level
-
class LOV(Module):
def __init__(self, db, uuid, fs_name, name_override = None, config_only = None):
Module.__init__(self, 'LOV', db)
if name_override != None:
self.name = "lov_%s" % name_override
+ self.add_lustre_module('lov', 'lov')
self.mds_uuid = self.db.get_first_ref('mds')
self.stripe_sz = self.db.get_val_int('stripesize', 1048576)
self.stripe_off = self.db.get_val_int('stripeoffset', 0)
self.pattern = self.db.get_val_int('stripepattern', 0)
- self.devlist = self.db.get_lov_tgts('lov_tgt')
- self.stripe_cnt = self.db.get_val_int('stripecount', len(self.devlist))
+ self.devlist = []
+ self.stripe_cnt = self.db.get_val_int('stripecount', 1)
self.osclist = []
- self.obdlist = []
self.desc_uuid = self.uuid
self.uuid = generate_client_uuid(self.name)
self.fs_name = fs_name
+ # settings below here won't be seen by the MDSDEV code!
if config_only:
self.config_only = 1
return
self.config_only = None
mds = self.db.lookup(self.mds_uuid)
self.mds_name = mds.getName()
+ self.devlist = self.db.get_lov_tgts('lov_tgt')
for (obd_uuid, index, gen, active) in self.devlist:
if obd_uuid == '':
continue
- self.obdlist.append(obd_uuid)
- obd = self.db.lookup(obd_uuid)
+ obd = self.db.lookup(obd_uuid)
osc = get_osc(obd, self.uuid, fs_name)
if osc:
- self.osclist.append((osc, index, gen, active))
+ self.osclist.append((osc, index, gen, active))
else:
panic('osc not found:', obd_uuid)
- def get_uuid(self):
- return self.uuid
- def get_name(self):
- return self.name
+ if self.osclist == []:
+ debug("get_lov_tgts failed, using get_refs");
+ index = 0
+ self.devlist = self.db.get_refs('obd')
+ for obd_uuid in self.devlist:
+ obd = self.db.lookup(obd_uuid)
+ osc = get_osc(obd, self.uuid, fs_name)
+ if osc:
+ self.osclist.append((osc, index, 1, 1))
+ else:
+ panic('osc not found:', obd_uuid)
+ index = index + 1
+ if self.osclist == []:
+ panic('No OSCs configured for LOV')
+ debug('dbg LOV __init__:', self.osclist, self.devlist, self.stripe_cnt)
+
def prepare(self):
- if not config.record and is_prepared(self.name):
+ debug('dbg LOV prepare')
+ if is_prepared(self.name):
return
+ debug('dbg LOV prepare:', self.osclist, self.devlist)
self.info(self.mds_uuid, self.stripe_cnt, self.stripe_sz,
self.stripe_off, self.pattern, self.devlist,
self.mds_name)
- lctl.lov_setup(self.name, self.uuid, self.desc_uuid, self.stripe_cnt,
- self.stripe_sz, self.stripe_off, self.pattern,
- string.join(self.obdlist))
+ lctl.lov_setup(self.name, self.uuid,
+ self.desc_uuid, self.mds_name, self.stripe_cnt,
+ self.stripe_sz, self.stripe_off, self.pattern)
+ if self.osclist == []:
+ panic('No OSCs configured for LOV?')
for (osc, index, gen, active) in self.osclist:
target_uuid = osc.target_uuid
try:
lctl.lov_add_obd(self.name, self.uuid, target_uuid, index, gen)
def cleanup(self):
- for (osc, index, gen, active) in self.osclist:
- target_uuid = osc.target_uuid
- osc.cleanup()
if is_prepared(self.name):
Module.cleanup(self)
+ for (osc, index, gen, active) in self.osclist:
+ osc.cleanup()
if self.config_only:
panic("Can't clean up config_only LOV ", self.name)
- def add_module(self, manager):
+ def load_module(self):
if self.config_only:
panic("Can't load modules for config_only LOV ", self.name)
for (osc, index, gen, active) in self.osclist:
- osc.add_module(manager)
+ osc.load_module()
break
- manager.add_lustre_module('lov', 'lov')
-
- def correct_level(self, level, op=None):
- return level
-
-class LMV(Module):
- def __init__(self, db, uuid, fs_name, name_override = None):
- Module.__init__(self, 'LMV', db)
- if name_override != None:
- self.name = "lmv_%s" % name_override
-
- self.devlist = self.db.get_lmv_tgts('lmv_tgt')
- if self.devlist == None:
- self.devlist = self.db.get_refs('mds')
-
- self.mdclist = []
- self.desc_uuid = self.uuid
- self.uuid = uuid
- self.fs_name = fs_name
- for mds_uuid in self.devlist:
- mds = self.db.lookup(mds_uuid)
- if not mds:
- panic("MDS not found!")
- mdc = MDC(mds, self.uuid, fs_name)
- if mdc:
- self.mdclist.append(mdc)
- else:
- panic('mdc not found:', mds_uuid)
+ Module.load_module(self)
- def prepare(self):
- if is_prepared(self.name):
- return
-
- self.info();
- for mdc in self.mdclist:
- try:
- # Only ignore connect failures with --force, which
- # isn't implemented here yet.
- mdc.prepare(ignore_connect_failure=0)
- except CommandError, e:
- print "Error preparing LMV %s\n" % mdc.uuid
- raise e
-
- lctl.lmv_setup(self.name, self.uuid, self.desc_uuid,
- string.join(self.devlist))
-
- def cleanup(self):
- for mdc in self.mdclist:
- mdc.cleanup()
- if is_prepared(self.name):
- Module.cleanup(self)
-
- def add_module(self, manager):
- for mdc in self.mdclist:
- mdc.add_module(manager)
+ def cleanup_module(self):
+ if self.config_only:
+ panic("Can't cleanup modules for config_only LOV ", self.name)
+ Module.cleanup_module(self)
+ for (osc, index, gen, active) in self.osclist:
+ if active:
+ osc.cleanup_module()
break
- manager.add_lustre_module('lmv', 'lmv')
-
- def correct_level(self, level, op=None):
- return level
-class CONFDEV(Module):
- def __init__(self, db, name, target_uuid, uuid):
- Module.__init__(self, 'CONFDEV', db)
+class MDSDEV(Module):
+ def __init__(self,db):
+ Module.__init__(self, 'MDSDEV', db)
self.devpath = self.db.get_val('devpath','')
- self.backdevpath = self.db.get_val('devpath','')
self.size = self.db.get_val_int('devsize', 0)
self.journal_size = self.db.get_val_int('journalsize', 0)
+
self.fstype = self.db.get_val('fstype', '')
- self.backfstype = self.db.get_val('backfstype', '')
- self.mkfsoptions = self.db.get_val('mkfsoptions', '')
+ if sys_get_branch() == '2.4' and self.fstype == 'ldiskfs':
+ self.fstype = 'ext3'
+ elif sys_get_branch() == '2.6' and self.fstype == 'ext3':
+ self.fstype = 'ldiskfs'
+
+ self.nspath = self.db.get_val('nspath', '')
+ self.mkfsoptions = '-i 4096 ' + self.db.get_val('mkfsoptions', '')
self.mountfsoptions = self.db.get_val('mountfsoptions', '')
- self.target = self.db.lookup(target_uuid)
- self.name = "conf_%s" % self.target.getName()
- self.client_uuids = self.target.get_refs('client')
- self.obdtype = self.db.get_val('obdtype', '')
-
- if self.obdtype == None:
- self.obdtype = 'dumb'
-
- self.conf_name = name
- self.conf_uuid = uuid
- self.realdev = self.devpath
-
- self.lmv = None
- self.master = None
-
- lmv_uuid = self.db.get_first_ref('lmv')
- if lmv_uuid != None:
- self.lmv = self.db.lookup(lmv_uuid)
- if self.lmv != None:
- self.client_uuids = self.lmv.get_refs('client')
-
- if self.target.get_class() == 'mds':
- if self.target.get_val('failover', 0):
- self.failover_mds = 'f'
- else:
- self.failover_mds = 'n'
- self.format = self.db.get_val('autoformat', "no")
+ if config.quota:
+ self.quota = config.quota
else:
- self.format = self.db.get_val('autoformat', "yes")
- self.osdtype = self.db.get_val('osdtype')
- ost = self.db.lookup(target_uuid)
- if ost.get_val('failover', 0):
- self.failover_ost = 'f'
- else:
- self.failover_ost = 'n'
-
- self.inode_size = self.get_inode_size()
-
- if self.lmv != None:
- client_uuid = self.name + "_lmv_UUID"
- self.master = LMV(self.lmv, client_uuid,
- self.conf_name, self.conf_name)
-
- def get_inode_size(self):
- inode_size = self.db.get_val_int('inodesize', 0)
- if inode_size == 0 and self.target.get_class() == 'mds':
-
- # default inode size for case when neither LOV either
- # LMV is accessible.
- self.inode_size = 256
-
+ self.quota = self.db.get_val('quota', '')
+ # overwrite the orignal MDSDEV name and uuid with the MDS name and uuid
+ target_uuid = self.db.get_first_ref('target')
+ mds = self.db.lookup(target_uuid)
+ self.name = mds.getName()
+ self.filesystem_uuids = mds.get_refs('filesystem')
+ # FIXME: if fstype not set, then determine based on kernel version
+ self.format = self.db.get_val('autoformat', "no")
+ if mds.get_val('failover', '1') != '0':
+ self.failover_mds = 'f'
+ else:
+ self.failover_mds = 'n'
+ active_uuid = get_active_target(mds)
+ if not active_uuid:
+ panic("No target device found:", target_uuid)
+ if active_uuid == self.uuid:
+ self.active = 1
+ else:
+ self.active = 0
+ if self.active and config.group and config.group != mds.get_val('group', mds.get_val('name')):
+ self.active = 0
+
+ self.inode_size = self.db.get_val_int('inodesize', 0)
+ debug('original inode_size ', self.inode_size)
+ if self.inode_size == 0:
# find the LOV for this MDS
- lovconfig_uuid = self.target.get_first_ref('lovconfig')
- if lovconfig_uuid or self.lmv != None:
- if self.lmv != None:
- lovconfig_uuid = self.lmv.get_first_ref('lovconfig')
- lovconfig = self.lmv.lookup(lovconfig_uuid)
- lov_uuid = lovconfig.get_first_ref('lov')
- if lov_uuid == None:
- panic(self.target.getName() + ": No LOV found for lovconfig ",
- lovconfig.name)
- else:
- lovconfig = self.target.lookup(lovconfig_uuid)
- lov_uuid = lovconfig.get_first_ref('lov')
- if lov_uuid == None:
- panic(self.target.getName() + ": No LOV found for lovconfig ",
- lovconfig.name)
- if self.lmv != None:
- lovconfig_uuid = self.lmv.get_first_ref('lovconfig')
- lovconfig = self.lmv.lookup(lovconfig_uuid)
- lov_uuid = lovconfig.get_first_ref('lov')
-
- lov = LOV(self.db.lookup(lov_uuid), lov_uuid, self.name,
- config_only = 1)
-
- # default stripe count controls default inode_size
- if lov.stripe_cnt > 0:
- stripe_count = lov.stripe_cnt
- else:
- stripe_count = len(lov.devlist)
- if stripe_count > 77:
- inode_size = 4096
- elif stripe_count > 35:
- inode_size = 2048
- elif stripe_count > 13:
- inode_size = 1024
- elif stripe_count > 3:
- inode_size = 512
- else:
- inode_size = 256
-
- return inode_size
-
- def get_mount_options(self, blkdev):
- options = def_mount_options(self.fstype,
- self.target.get_class())
-
- if config.mountfsoptions:
- if options:
- options = "%s,%s" %(options, config.mountfsoptions)
+ lovconfig_uuid = mds.get_first_ref('lovconfig')
+ if not lovconfig_uuid:
+ panic("No LOV config found for MDS ", mds.name)
+ lovconfig = mds.lookup(lovconfig_uuid)
+ lov_uuid = lovconfig.get_first_ref('lov')
+ if not lov_uuid:
+ panic("No LOV found for lovconfig ", lovconfig.name)
+ lov = LOV(self.db.lookup(lov_uuid), lov_uuid, 'FS_name', config_only = 1)
+
+ # default stripe count controls default inode_size
+ if (lov.stripe_cnt > 0):
+ stripe_count = lov.stripe_cnt
else:
- options = config.mountfsoptions
- if self.mountfsoptions:
- options = "%s,%s" %(options, self.mountfsoptions)
- else:
- if self.mountfsoptions:
- if options:
- options = "%s,%s" %(options, self.mountfsoptions)
- else:
- options = self.mountfsoptions
-
- if self.fstype == 'smfs':
- if options:
- options = "%s,type=%s,dev=%s" %(options, self.backfstype,
- blkdev)
+ stripe_count = 1
+ if stripe_count > 77:
+ self.inode_size = 512
+ elif stripe_count > 34:
+ self.inode_size = 2048
+ elif stripe_count > 13:
+ self.inode_size = 1024
+ #elif stripe_count < 3:
+ # self.inode_size = 256
else:
- options = "type=%s,dev=%s" %(self.backfstype,
- blkdev)
-
- if self.target.get_class() == 'mds':
- if options:
- options = "%s,acl,user_xattr,iopen_nopriv" %(options)
- else:
- options = "iopen_nopriv"
-
- return options
+ self.inode_size = 512
+ debug('stripe_count ', stripe_count,' inode_size ',self.inode_size)
+
+ self.target_dev_uuid = self.uuid
+ self.uuid = target_uuid
+
+ # loading modules
+ if self.quota:
+ self.add_lustre_module('quota', 'lquota')
+ self.add_lustre_module('mdc', 'mdc')
+ self.add_lustre_module('osc', 'osc')
+ self.add_lustre_module('lov', 'lov')
+ self.add_lustre_module('mds', 'mds')
+ if self.fstype == 'ldiskfs':
+ self.add_lustre_module('ldiskfs', 'ldiskfs')
+ if self.fstype:
+ self.add_lustre_module('lvfs', 'fsfilt_%s' % (self.fstype))
+
+ def load_module(self):
+ if self.active:
+ Module.load_module(self)
def prepare(self):
if is_prepared(self.name):
return
-
- blkdev = block_dev(self.devpath, self.size, self.fstype,
- config.reformat, self.format, self.journal_size,
- self.inode_size, self.mkfsoptions, self.backfstype,
- self.backdevpath)
-
- if self.fstype == 'smfs':
- realdev = blkdev
- else:
- realdev = blkdev
-
- mountfsoptions = self.get_mount_options(blkdev)
+ if not self.active:
+ debug(self.uuid, "not active")
+ return
+ if config.reformat:
+ # run write_conf automatically, if --reformat used
+ self.write_conf()
+ self.info(self.devpath, self.fstype, self.size, self.format)
+ # never reformat here
+ blkdev = block_dev(self.devpath, self.size, self.fstype, 0,
+ self.format, self.journal_size, self.inode_size,
+ self.mkfsoptions)
+ if not is_prepared('MDT'):
+ lctl.newdev("mdt", 'MDT', 'MDT_UUID', setup ="")
+ try:
+ mountfsoptions = def_mount_options(self.fstype, 'mds', blkdev)
- self.info(self.target.get_class(), realdev, mountfsoptions,
- self.fstype, self.size, self.format)
+ if config.mountfsoptions:
+ if mountfsoptions:
+ mountfsoptions = mountfsoptions + ',' + config.mountfsoptions
+ else:
+ mountfsoptions = config.mountfsoptions
+ if self.mountfsoptions:
+ mountfsoptions = mountfsoptions + ',' + self.mountfsoptions
+ else:
+ if self.mountfsoptions:
+ if mountfsoptions:
+ mountfsoptions = mountfsoptions + ',' + self.mountfsoptions
+ else:
+ mountfsoptions = self.mountfsoptions
- lctl.newdev("confobd", self.name, self.uuid,
- setup ="%s %s %s" %(realdev, self.fstype,
- mountfsoptions))
+ print 'MDS mount options: ' + mountfsoptions
- self.mountfsoptions = mountfsoptions
- self.realdev = realdev
+ lctl.newdev("mds", self.name, self.uuid,
+ setup ="%s %s %s %s %s" %(blkdev, self.fstype, self.name,
+ mountfsoptions, self.quota))
+ self.group_upcall = self.db.get_val('group_upcall','')
+ sys_set_group_upcall(self.name, self.group_upcall)
- def add_module(self, manager):
- manager.add_lustre_module('obdclass', 'confobd')
+ except CommandError, e:
+ if e.rc == 2:
+ panic("MDS failed to start. Check the syslog for details." +
+ " (May need to run lconf --write-conf)")
+ else:
+ raise e
def write_conf(self):
- if self.target.get_class() == 'ost':
- config.record = 1
- lctl.clear_log(self.name, self.target.getName() + '-conf')
- lctl.record(self.name, self.target.getName() + '-conf')
- lctl.newdev(self.osdtype, self.conf_name, self.conf_uuid,
- setup ="%s %s %s %s" %(self.realdev, self.fstype,
- self.failover_ost,
- self.mountfsoptions))
- lctl.end_record()
- lctl.clear_log(self.name, 'OSS-conf')
- lctl.record(self.name, 'OSS-conf')
- lctl.newdev("ost", 'OSS', 'OSS_UUID', setup ="")
- lctl.end_record()
- config.record = 0
+ if is_prepared(self.name):
return
-
- if self.target.get_class() == 'mds':
- if self.master != None:
- master_name = self.master.name
- else:
- master_name = 'dumb'
-
- config.record = 1
- lctl.clear_log(self.name, self.target.getName() + '-conf')
- lctl.record(self.name, self.target.getName() + '-conf')
- lctl.newdev("mds", self.conf_name, self.conf_uuid,
- setup ="%s %s %s %s %s %s" %(self.realdev, self.fstype,
- self.conf_name, self.mountfsoptions,
- master_name, self.obdtype))
- lctl.end_record()
- config.record = 0
-
- if not self.client_uuids:
- return 0
-
- for uuid in self.client_uuids:
- log("recording client:", uuid)
+ self.info(self.devpath, self.fstype, self.format)
+ blkdev = block_dev(self.devpath, self.size, self.fstype,
+ config.reformat, self.format, self.journal_size,
+ self.inode_size, self.mkfsoptions)
+ lctl.newdev("mds", self.name, self.uuid,
+ setup ="%s %s" %(blkdev, self.fstype))
+
+ # record logs for the MDS lov
+ for uuid in self.filesystem_uuids:
+ log("recording clients for filesystem:", uuid)
+ fs = self.db.lookup(uuid)
+ obd_uuid = fs.get_first_ref('obd')
client_uuid = generate_client_uuid(self.name)
- client = VOSC(self.db.lookup(uuid), client_uuid,
- self.target.getName(), self.name)
+ client = VOSC(self.db.lookup(obd_uuid), client_uuid, self.name,
+ self.name)
config.record = 1
- lctl.clear_log(self.name, self.target.getName())
- lctl.record(self.name, self.target.getName())
+ lctl.clear_log(self.name, self.name)
+ lctl.record(self.name, self.name)
client.prepare()
- lctl.mount_option(self.target.getName(), client.get_name(), "")
+ lctl.mount_option(self.name, client.get_name(), "")
lctl.end_record()
-
- config.cleanup = 1
- lctl.clear_log(self.name, self.target.getName() + '-clean')
- lctl.record(self.name, self.target.getName() + '-clean')
- client.cleanup()
- lctl.del_mount_option(self.target.getName())
- lctl.end_record()
- config.cleanup = 0
config.record = 0
- if config.record:
- return
-
# record logs for each client
if config.ldapurl:
config_options = "--ldapurl " + config.ldapurl + " --config " + config.config
client_name = node_db.getName()
for prof_uuid in node_db.get_refs('profile'):
prof_db = node_db.lookup(prof_uuid)
- # refactor this into a funtion to test "clientness"
- # of a node.
+ # refactor this into a funtion to test "clientness" of a node.
for ref_class, ref_uuid in prof_db.get_all_refs():
if ref_class in ('mountpoint','echoclient'):
- debug("recording", client_name)
+ thing = self.db.lookup(ref_uuid);
+ fs_uuid = thing.get_first_ref('filesystem')
+ if not fs_uuid in self.filesystem_uuids:
+ continue;
+
+ log("Recording log", client_name, "on", self.name)
old_noexec = config.noexec
config.noexec = 0
noexec_opt = ('', '-n')
ret, out = run (sys.argv[0],
noexec_opt[old_noexec == 1],
- " -v --record --nomod",
+ " -v --record --nomod --old_conf",
"--record_log", client_name,
"--record_device", self.name,
"--node", client_name,
config_options)
- if config.verbose:
- for s in out: log("record> ", string.strip(s))
- ret, out = run (sys.argv[0],
- noexec_opt[old_noexec == 1],
- "--cleanup -v --record --nomod",
- "--record_log", client_name + "-clean",
- "--record_device", self.name,
- "--node", client_name,
- config_options)
+ if ret:
+ lctl.clear_log(self.name, client_name)
+ print out
+ self.cleanup()
+ panic("Record client log %s on %s failed" %(
+ client_name, self.name))
if config.verbose:
for s in out: log("record> ", string.strip(s))
config.noexec = old_noexec
+ try:
+ lctl.cleanup(self.name, self.uuid, config.force, config.failover)
+ except CommandError, e:
+ log(self.module_name, "cleanup failed: ", self.name)
+ e.dump()
+ cleanup_error(e.rc)
+ Module.cleanup(self)
+ clean_loop(self.devpath)
- def start(self):
- try:
- lctl.start(self.name, self.conf_name)
- except CommandError, e:
- raise e
- if self.target.get_class() == 'ost':
- if not is_prepared('OSS'):
- try:
- lctl.start(self.name, 'OSS')
- except CommandError, e:
- raise e
-
- def cleanup(self):
- if is_prepared(self.name):
- try:
- lctl.cleanup(self.name, self.uuid, 0, 0)
- clean_dev(self.devpath, self.fstype,
- self.backfstype, self.backdevpath)
- except CommandError, e:
- log(self.module_name, "cleanup failed: ", self.name)
- e.dump()
- cleanup_error(e.rc)
- Module.cleanup(self)
-
-class MDSDEV(Module):
- def __init__(self,db):
- Module.__init__(self, 'MDSDEV', db)
- self.devpath = self.db.get_val('devpath','')
- self.backdevpath = self.db.get_val('devpath','')
- self.size = self.db.get_val_int('devsize', 0)
- self.journal_size = self.db.get_val_int('journalsize', 0)
- self.fstype = self.db.get_val('fstype', '')
- self.backfstype = self.db.get_val('backfstype', '')
- self.nspath = self.db.get_val('nspath', '')
- self.mkfsoptions = self.db.get_val('mkfsoptions', '')
- self.mountfsoptions = self.db.get_val('mountfsoptions', '')
- self.obdtype = self.db.get_val('obdtype', '')
- self.root_squash = self.db.get_val('root_squash', '')
- self.no_root_squash = self.db.get_val('no_root_squash', '')
-
- target_uuid = self.db.get_first_ref('target')
- self.target = self.db.lookup(target_uuid)
- self.name = self.target.getName()
- self.master = None
- self.lmv = None
-
- lmv_uuid = self.db.get_first_ref('lmv')
- if lmv_uuid != None:
- self.lmv = self.db.lookup(lmv_uuid)
-
- active_uuid = get_active_target(self.target)
- if not active_uuid:
- panic("No target device found:", target_uuid)
- if active_uuid == self.uuid:
- self.active = 1
- group = self.target.get_val('group')
- if config.group and config.group != group:
- self.active = 0
- else:
- self.active = 0
-
- self.uuid = target_uuid
-
- # setup LMV
- if self.lmv != None:
- client_uuid = self.name + "_lmv_UUID"
- self.master = LMV(self.lmv, client_uuid,
- self.name, self.name)
-
- self.confobd = CONFDEV(self.db, self.name,
- target_uuid, self.uuid)
-
- def add_module(self, manager):
- if self.active:
- manager.add_lustre_module('mdc', 'mdc')
- manager.add_lustre_module('osc', 'osc')
- manager.add_lustre_module('ost', 'ost')
- manager.add_lustre_module('lov', 'lov')
- manager.add_lustre_module('mds', 'mds')
-
- if self.fstype == 'smfs' or self.fstype == 'ldiskfs':
- manager.add_lustre_module(self.fstype, self.fstype)
-
- if self.fstype:
- manager.add_lustre_module('lvfs', 'fsfilt_%s' % (self.fstype))
-
- # if fstype is smfs, then we should also take care about backing
- # store fs.
- if self.fstype == 'smfs':
- manager.add_lustre_module(self.backfstype, self.backfstype)
- manager.add_lustre_module('lvfs', 'fsfilt_%s' % (self.backfstype))
-
- for option in string.split(self.mountfsoptions, ','):
- if option == 'snap':
- if not self.fstype == 'smfs':
- panic("mountoptions has 'snap', but fstype is not smfs.")
- manager.add_lustre_module('lvfs', 'fsfilt_snap_%s' % (self.fstype))
- manager.add_lustre_module('lvfs', 'fsfilt_snap_%s' % (self.backfstype))
-
- # add LMV modules
- if self.master != None:
- self.master.add_module(manager)
-
- # add CONFOBD modules
- if self.confobd != None:
- self.confobd.add_module(manager)
-
- def write_conf(self):
- if is_prepared(self.name):
- return
- if not self.active:
- debug(self.uuid, "not active")
- return
- run_acceptors()
- self.confobd.prepare()
- self.confobd.write_conf()
- self.confobd.cleanup()
-
- def prepare(self):
- if is_prepared(self.name):
- return
- if not self.active:
- debug(self.uuid, "not active")
- return
- run_acceptors()
-
- self.confobd.prepare()
- if config.reformat:
- self.confobd.write_conf()
-
- # prepare LMV
- if self.master != None:
- self.master.prepare()
-
- lctl.attach("mds", self.name, self.uuid)
- if config.mds_mds_sec:
- lctl.set_security(self.name, "mds_mds_sec", config.mds_mds_sec)
- if config.mds_ost_sec:
- lctl.set_security(self.name, "mds_ost_sec", config.mds_ost_sec)
- lctl.detach(self.name)
-
- if not config.record:
- self.confobd.start()
-
- if not is_prepared('MDT'):
- lctl.newdev("mdt", 'MDT', 'MDT_UUID', setup ="")
-
- if development_mode():
- procentry = "/proc/fs/lustre/mds/lsd_upcall"
- upcall = os.path.abspath(os.path.dirname(sys.argv[0]) + "/lsd_upcall")
- if not (os.access(procentry, os.R_OK) and os.access(upcall, os.R_OK)):
- print "MDS Warning: failed to set lsd cache upcall"
- else:
- run("echo ", upcall, " > ", procentry)
-
- if config.root_squash == None:
- config.root_squash = self.root_squash
- if config.no_root_squash == None:
- config.no_root_squash = self.no_root_squash
- if config.root_squash:
- if config.no_root_squash:
- nsnid = config.no_root_squash
+ #change the mtime of LLOG to match the XML creation time
+ if toplustreDB.get_mtime():
+ mtime = toplustreDB.get_mtime()
+ debug("changing mtime of LOGS to %s" %mtime)
+ ret, mktemp = runcmd("mktemp /tmp/lustre-cmd.XXXXXXXX")
+ if ret:
+ log(self.module_name, "create mtime LOGS cmdfile failed: ", self.name)
else:
- nsnid = "0"
- lctl.root_squash(self.name, config.root_squash, nsnid)
-
- def msd_remaining(self):
+ mtimecmdfile = string.split(mktemp[0])[0]
+ fd = os.open(mtimecmdfile, os.O_RDWR | os.O_CREAT)
+ os.write(fd, "\n\n\n\n\n%s\n\n" %mtime)
+ os.close(fd)
+ cmd = "debugfs -w -R \"mi /LOGS\" <%s %s" %(mtimecmdfile, self.devpath)
+ ret, outs = runcmd(cmd)
+ os.remove(mtimecmdfile)
+ if ret:
+ print "Can not change mtime of LOGS by debugfs."
+
+ def mds_remaining(self):
out = lctl.device_list()
for s in out:
if string.split(s)[2] in ('mds',):
+ if string.split(s)[1] in ('ST',):
+ return 0
return 1
def safe_to_clean(self):
return self.active
def safe_to_clean_modules(self):
- return not self.msd_remaining()
-
+ return not self.mds_remaining()
+
def cleanup(self):
if not self.active:
debug(self.uuid, "not active")
e.dump()
cleanup_error(e.rc)
Module.cleanup(self)
- # cleanup LMV
- if self.master != None:
- self.master.cleanup()
- if not self.msd_remaining() and is_prepared('MDT'):
+ if not self.mds_remaining() and is_prepared('MDT'):
try:
lctl.cleanup("MDT", "MDT_UUID", config.force,
config.failover)
print "cleanup failed: ", self.name
e.dump()
cleanup_error(e.rc)
-
- if self.confobd:
- self.confobd.cleanup()
-
- def correct_level(self, level, op=None):
- #if self.master != None:
- # level = level + 2
- return level
-
+ clean_loop(self.devpath)
+
class OSD(Module):
def __init__(self, db):
Module.__init__(self, 'OSD', db)
self.osdtype = self.db.get_val('osdtype')
self.devpath = self.db.get_val('devpath', '')
- self.backdevpath = self.db.get_val('devpath', '')
self.size = self.db.get_val_int('devsize', 0)
self.journal_size = self.db.get_val_int('journalsize', 0)
+
+ # now as we store fids in EA on OST we need to make inode bigger
self.inode_size = self.db.get_val_int('inodesize', 0)
+ if self.inode_size == 0:
+ self.inode_size = 256
self.mkfsoptions = self.db.get_val('mkfsoptions', '')
+ # Allocate fewer inodes on large OST devices. Most filesystems
+ # can be much more aggressive than this, but by default we can't.
+ if self.size > 1000000:
+ self.mkfsoptions = '-i 16384 ' + self.mkfsoptions
self.mountfsoptions = self.db.get_val('mountfsoptions', '')
+ if config.quota:
+ self.quota = config.quota
+ else:
+ self.quota = self.db.get_val('quota', '')
+
self.fstype = self.db.get_val('fstype', '')
- self.backfstype = self.db.get_val('backfstype', '')
+ if sys_get_branch() == '2.4' and self.fstype == 'ldiskfs':
+ self.fstype = 'ext3'
+ elif sys_get_branch() == '2.6' and self.fstype == 'ext3':
+ self.fstype = 'ldiskfs'
+
self.nspath = self.db.get_val('nspath', '')
target_uuid = self.db.get_first_ref('target')
ost = self.db.lookup(target_uuid)
self.name = ost.getName()
self.format = self.db.get_val('autoformat', 'yes')
- if ost.get_val('failover', 0):
+ if ost.get_val('failover', '1') != '0':
self.failover_ost = 'f'
else:
self.failover_ost = 'n'
panic("No target device found:", target_uuid)
if active_uuid == self.uuid:
self.active = 1
- group = ost.get_val('group')
- if config.group and config.group != group:
- self.active = 0
else:
self.active = 0
+ if self.active and config.group and config.group != ost.get_val('group', ost.get_val('name')):
+ self.active = 0
+ self.target_dev_uuid = self.uuid
self.uuid = target_uuid
- self.confobd = CONFDEV(self.db, self.name,
- target_uuid, self.uuid)
-
- def add_module(self, manager):
- if not self.active:
- return
- manager.add_lustre_module('ost', 'ost')
-
- if self.fstype == 'smfs' or self.fstype == 'ldiskfs':
- manager.add_lustre_module(self.fstype, self.fstype)
-
+ # modules
+ if self.quota:
+ self.add_lustre_module('quota', 'lquota')
+ self.add_lustre_module('ost', 'ost')
+ # FIXME: should we default to ext3 here?
+ if self.fstype == 'ldiskfs':
+ self.add_lustre_module('ldiskfs', 'ldiskfs')
if self.fstype:
- manager.add_lustre_module('lvfs' , 'fsfilt_%s' % (self.fstype))
-
- if self.fstype == 'smfs':
- manager.add_lustre_module(self.backfstype, self.backfstype)
- manager.add_lustre_module('lvfs' , 'fsfilt_%s' % (self.backfstype))
+ self.add_lustre_module('lvfs' , 'fsfilt_%s' % (self.fstype))
+ self.add_lustre_module(self.osdtype, self.osdtype)
- for option in self.mountfsoptions:
- if option == 'snap':
- if not self.fstype == 'smfs':
- panic("mountoptions with snap, but fstype is not smfs\n")
- manager.add_lustre_module('lvfs', 'fsfilt_snap_%s' % (self.fstype))
- manager.add_lustre_module('lvfs', 'fsfilt_snap_%s' % (self.backfstype))
-
- manager.add_lustre_module(self.osdtype, self.osdtype)
-
- # add CONFOBD modules
- if self.confobd != None:
- self.confobd.add_module(manager)
+ def load_module(self):
+ if self.active:
+ Module.load_module(self)
+ # need to check /proc/mounts and /etc/mtab before
+ # formatting anything.
+ # FIXME: check if device is already formatted.
def prepare(self):
if is_prepared(self.name):
return
if not self.active:
debug(self.uuid, "not active")
return
-
- run_acceptors()
+ self.info(self.osdtype, self.devpath, self.size, self.fstype,
+ self.format, self.journal_size, self.inode_size)
if self.osdtype == 'obdecho':
- self.info(self.osdtype)
- lctl.newdev("obdecho", self.name, self.uuid)
- if not is_prepared('OSS'):
- lctl.newdev("ost", 'OSS', 'OSS_UUID', setup="")
- else:
- self.confobd.prepare()
- if config.reformat:
- self.confobd.write_conf()
- if not config.record:
- self.confobd.start()
+ blkdev = ''
+ else:
+ blkdev = block_dev(self.devpath, self.size, self.fstype,
+ config.reformat, self.format, self.journal_size,
+ self.inode_size, self.mkfsoptions)
- def write_conf(self):
- if is_prepared(self.name):
- return
- if not self.active:
- debug(self.uuid, "not active")
- return
-
- run_acceptors()
- if self.osdtype != 'obdecho':
- self.confobd.prepare()
- self.confobd.write_conf()
- if not config.write_conf:
- self.confobd.start()
- self.confobd.cleanup()
+ mountfsoptions = def_mount_options(self.fstype, 'ost', blkdev)
+
+ if config.mountfsoptions:
+ if mountfsoptions:
+ mountfsoptions = mountfsoptions + ',' + config.mountfsoptions
+ else:
+ mountfsoptions = config.mountfsoptions
+ if self.mountfsoptions:
+ mountfsoptions = mountfsoptions + ',' + self.mountfsoptions
+ else:
+ if self.mountfsoptions:
+ if mountfsoptions:
+ mountfsoptions = mountfsoptions + ',' + self.mountfsoptions
+ else:
+ mountfsoptions = self.mountfsoptions
+
+ print 'OST mount options: ' + mountfsoptions
+
+ lctl.newdev(self.osdtype, self.name, self.uuid,
+ setup ="%s %s %s %s %s" %(blkdev, self.fstype,
+ self.failover_ost, mountfsoptions,
+ self.quota))
+ if not is_prepared('OSS'):
+ lctl.newdev("ost", 'OSS', 'OSS_UUID', setup ="")
def osd_remaining(self):
out = lctl.device_list()
if not self.active:
debug(self.uuid, "not active")
return
-
if is_prepared(self.name):
self.info()
try:
print "cleanup failed: ", self.name
e.dump()
cleanup_error(e.rc)
-
- if self.osdtype != 'obdecho':
- if self.confobd:
- self.confobd.cleanup()
-
- def correct_level(self, level, op=None):
- return level
+ if not self.osdtype == 'obdecho':
+ clean_loop(self.devpath)
# Generic client module, used by OSC and MDC
class Client(Module):
- def __init__(self, tgtdb, uuid, module, fs_name,
- self_name=None, module_dir=None):
+ def __init__(self, tgtdb, uuid, module, fs_name, self_name=None,
+ module_dir=None):
self.target_name = tgtdb.getName()
self.target_uuid = tgtdb.getUUID()
- self.module_dir = module_dir
- self.backup_targets = []
- self.module = module
self.db = tgtdb
+ self.backup_targets = []
self.tgt_dev_uuid = get_active_target(tgtdb)
if not self.tgt_dev_uuid:
- panic("No target device found for target(1):", self.target_name)
+ panic("No target device found for target:", self.target_name)
+ self.kmod = kmod(config.lustre, config.portals)
self._server = None
self._connected = 0
self.lookup_server(self.tgt_dev_uuid)
self.lookup_backup_targets()
self.fs_name = fs_name
- if not self.module_dir:
- self.module_dir = module
-
- def add_module(self, manager):
- manager.add_lustre_module(self.module_dir, self.module)
+ if not module_dir:
+ module_dir = module
+ self.add_lustre_module(module_dir, module)
def lookup_server(self, srv_uuid):
""" Lookup a server's network information """
self._server_nets = get_ost_net(self.db, srv_uuid)
if len(self._server_nets) == 0:
- panic ("Unable to find a server for:", srv_uuid)
-
- def get_name(self):
- return self.name
+ panic("Unable to find a server for:", srv_uuid)
def get_servers(self):
return self._server_nets
devdb = toplustreDB.lookup(ref_uuid)
uuid = devdb.get_first_ref('target')
if self.target_uuid == uuid and self.tgt_dev_uuid != ref_uuid:
+ debug("add backup target", ref_uuid)
self.backup_targets.append(ref_uuid)
def prepare(self, ignore_connect_failure = 0):
self.info(self.target_uuid)
- if not config.record and is_prepared(self.name):
+ if is_prepared(self.name):
self.cleanup()
try:
- srv = choose_local_server(self.get_servers())
- if srv:
+ srv_list = self.get_servers()
+ debug('dbg CLIENT __prepare__:', self.target_uuid, srv_list)
+ for srv in srv_list:
lctl.connect(srv)
- else:
- routes = find_route(self.get_servers())
- if len(routes) == 0:
- panic ("no route to", self.target_uuid)
- for (srv, r) in routes:
- lctl.add_route_host(r[0], srv.nid_uuid, r[1], r[3])
+ if len(srv_list) == 0:
+ panic("no servers for ", self.target_uuid)
except CommandError, e:
if not ignore_connect_failure:
raise e
- if srv:
+ if srv_list[0]:
+ srv = srv_list[0]
if self.target_uuid in config.inactive and self.permits_inactive():
debug("%s inactive" % self.target_uuid)
inactive_p = "inactive"
lctl.newdev(self.module, self.name, self.uuid,
setup ="%s %s %s" % (self.target_uuid, srv.nid_uuid,
inactive_p))
+ else:
+ panic("Unable to create OSC for ", self.target_uuid)
+
for tgt_dev_uuid in self.backup_targets:
this_nets = get_ost_net(toplustreDB, tgt_dev_uuid)
if len(this_nets) == 0:
- panic ("Unable to find a server for:", tgt_dev_uuid)
- srv = choose_local_server(this_nets)
- if srv:
- lctl.connect(srv)
+ panic ("Unable to find a backup server for:", tgt_dev_uuid)
else:
- routes = find_route(this_nets);
- if len(routes) == 0:
- panic("no route to", tgt_dev_uuid)
- for (srv, r) in routes:
- lctl.add_route_host(r[0]. srv.nid_uuid, r[1], r[3])
+ for srv in this_nets:
+ lctl.connect(srv)
if srv:
- lctl.add_conn(self.name, srv.nid_uuid);
-
- def cleanup(self):
- if is_prepared(self.name):
- Module.cleanup(self)
- try:
- srv = choose_local_server(self.get_servers())
- if srv:
- lctl.disconnect(srv)
- else:
- for (srv, r) in find_route(self.get_servers()):
- lctl.del_route_host(r[0], srv.nid_uuid, r[1], r[3])
- except CommandError, e:
- log(self.module_name, "cleanup failed: ", self.name)
- e.dump()
- cleanup_error(e.rc)
-
- for tgt_dev_uuid in self.backup_targets:
- this_net = get_ost_net(toplustreDB, tgt_dev_uuid)
- srv = choose_local_server(this_net)
- if srv:
- lctl.disconnect(srv)
- else:
- for (srv, r) in find_route(this_net):
- lctl.del_route_host(r[0]. srv.nid_uuid, r[1], r[3])
+ lctl.add_conn(self.name, srv.nid_uuid);
- def correct_level(self, level, op=None):
- return level
- def deactivate(self):
- try:
- lctl.deactivate(self.name)
- except CommandError, e:
- log(self.module_name, "deactivate failed: ", self.name)
- e.dump()
- cleanup_error(e.rc)
+ def cleanup(self):
+ if is_prepared(self.name):
+ Module.cleanup(self)
+ srv_list = self.get_servers()
+ for srv in srv_list:
+ lctl.disconnect(srv)
+ for tgt_dev_uuid in self.backup_targets:
+ this_nets = get_ost_net(toplustreDB, tgt_dev_uuid)
+ if len(this_nets) == 0:
+ panic ("Unable to find a backup server for:", tgt_dev_uuid)
+ else:
+ for srv in this_nets:
+ lctl.disconnect(srv)
class MDC(Client):
def __init__(self, db, uuid, fs_name):
def permits_inactive(self):
return 1
-class CMOBD(Module):
- def __init__(self, db):
- Module.__init__(self, 'CMOBD', db)
- self.name = self.db.getName();
- self.uuid = generate_client_uuid(self.name)
- self.master_uuid = self.db.get_first_ref('masterobd')
- self.cache_uuid = self.db.get_first_ref('cacheobd')
-
- master_obd = self.db.lookup(self.master_uuid)
- if not master_obd:
- panic('master obd not found:', self.master_uuid)
-
- cache_obd = self.db.lookup(self.cache_uuid)
- if not cache_obd:
- panic('cache obd not found:', self.cache_uuid)
-
- self.master = None
- self.cache = None
-
- master_class = master_obd.get_class()
- cache_class = cache_obd.get_class()
-
- if master_class == 'ost' or master_class == 'lov':
- client_uuid = "%s_lov_master_UUID" % (self.name)
- self.master = LOV(master_obd, client_uuid, self.name);
- elif master_class == 'mds':
- self.master = get_mdc(db, self.name, self.master_uuid)
- elif master_class == 'lmv':
- client_uuid = "%s_lmv_master_UUID" % (self.name)
- self.master = LMV(master_obd, client_uuid, self.name);
- else:
- panic("unknown master obd class '%s'" %(master_class))
-
- if cache_class == 'ost' or cache_class == 'lov':
- client_uuid = "%s_lov_cache_UUID" % (self.name)
- self.cache = LOV(cache_obd, client_uuid, self.name);
- elif cache_class == 'mds':
- self.cache = get_mdc(db, self.name, self.cache_uuid)
- elif cache_class == 'lmv':
- client_uuid = "%s_lmv_cache_UUID" % (self.name)
- self.cache = LMV(cache_obd, client_uuid, self.name);
- else:
- panic("unknown cache obd class '%s'" %(cache_class))
-
- def prepare(self):
- self.master.prepare()
- if not config.record and is_prepared(self.name):
- return
- self.info(self.master_uuid, self.cache_uuid)
- lctl.newdev("cmobd", self.name, self.uuid,
- setup ="%s %s" %(self.master.uuid,
- self.cache.uuid))
-
- def get_uuid(self):
- return self.uuid
-
- def get_name(self):
- return self.name
-
- def get_master_name(self):
- return self.master.name
-
- def get_cache_name(self):
- return self.cache.name
-
- def cleanup(self):
- if is_prepared(self.name):
- Module.cleanup(self)
- if self.master:
- self.master.cleanup()
-
- def add_module(self, manager):
- manager.add_lustre_module('cmobd', 'cmobd')
- self.master.add_module(manager)
-
- def correct_level(self, level, op=None):
- return level
-
class COBD(Module):
- def __init__(self, db, uuid, name):
+ def __init__(self, db):
Module.__init__(self, 'COBD', db)
- self.name = self.db.getName();
- self.uuid = generate_client_uuid(self.name)
- self.master_uuid = self.db.get_first_ref('masterobd')
+ self.real_uuid = self.db.get_first_ref('realobd')
self.cache_uuid = self.db.get_first_ref('cacheobd')
+ self.add_lustre_module('cobd' , 'cobd')
- master_obd = self.db.lookup(self.master_uuid)
- if not master_obd:
- panic('master obd not found:', self.master_uuid)
-
- cache_obd = self.db.lookup(self.cache_uuid)
- if not cache_obd:
- panic('cache obd not found:', self.cache_uuid)
-
- self.master = None
- self.cache = None
-
- master_class = master_obd.get_class()
- cache_class = cache_obd.get_class()
-
- if master_class == 'ost' or master_class == 'lov':
- client_uuid = "%s_lov_master_UUID" % (self.name)
- self.master = LOV(master_obd, client_uuid, name);
- elif master_class == 'mds':
- self.master = get_mdc(db, name, self.master_uuid)
- elif master_class == 'lmv':
- client_uuid = "%s_lmv_master_UUID" % (self.name)
- self.master = LMV(master_obd, client_uuid, self.name);
- else:
- panic("unknown master obd class '%s'" %(master_class))
-
- if cache_class == 'ost' or cache_class == 'lov':
- client_uuid = "%s_lov_cache_UUID" % (self.name)
- self.cache = LOV(cache_obd, client_uuid, name);
- elif cache_class == 'mds':
- self.cache = get_mdc(db, name, self.cache_uuid)
- elif cache_class == 'lmv':
- client_uuid = "%s_lmv_cache_UUID" % (self.name)
- self.cache = LMV(cache_obd, client_uuid, self.name);
- else:
- panic("unknown cache obd class '%s'" %(cache_class))
-
- def get_uuid(self):
- return self.uuid
-
- def get_name(self):
- return self.name
-
- def get_master_name(self):
- return self.master.name
-
- def get_cache_name(self):
- return self.cache.name
-
+ # need to check /proc/mounts and /etc/mtab before
+ # formatting anything.
+ # FIXME: check if device is already formatted.
def prepare(self):
- self.master.prepare()
- self.cache.prepare()
- if not config.record and is_prepared(self.name):
+ if is_prepared(self.name):
return
- self.info(self.master_uuid, self.cache_uuid)
+ self.info(self.real_uuid, self.cache_uuid)
lctl.newdev("cobd", self.name, self.uuid,
- setup ="%s %s" %(self.master.name,
- self.cache.name))
-
- def cleanup(self):
- if is_prepared(self.name):
- Module.cleanup(self)
- self.master.cleanup()
- self.cache.cleanup()
+ setup ="%s %s" %(self.real_uuid, self.cache_uuid))
- def add_module(self, manager):
- manager.add_lustre_module('cobd', 'cobd')
- self.master.add_module(manager)
# virtual interface for OSC and LOV
class VOSC(Module):
- def __init__(self, db, client_uuid, name, name_override = None):
+ def __init__(self, db, uuid, fs_name, name_override = None, quota = None):
Module.__init__(self, 'VOSC', db)
+ if quota:
+ self.add_lustre_module('quota', 'lquota')
if db.get_class() == 'lov':
- self.osc = LOV(db, client_uuid, name, name_override)
- self.type = 'lov'
- elif db.get_class() == 'cobd':
- self.osc = COBD(db, client_uuid, name)
- self.type = 'cobd'
+ self.osc = LOV(db, uuid, fs_name, name_override)
else:
- self.osc = OSC(db, client_uuid, name)
- self.type = 'osc'
-
+ self.osc = get_osc(db, uuid, fs_name)
def get_uuid(self):
- return self.osc.get_uuid()
-
+ return self.osc.uuid
def get_name(self):
- return self.osc.get_name()
-
+ return self.osc.name
def prepare(self):
self.osc.prepare()
-
def cleanup(self):
self.osc.cleanup()
-
- def add_module(self, manager):
- self.osc.add_module(manager)
-
- def correct_level(self, level, op=None):
- return self.osc.correct_level(level, op)
-
-# virtual interface for MDC and LMV
-class VMDC(Module):
- def __init__(self, db, client_uuid, name, name_override = None):
- Module.__init__(self, 'VMDC', db)
- if db.get_class() == 'lmv':
- self.mdc = LMV(db, client_uuid, name, name_override)
- elif db.get_class() == 'cobd':
- self.mdc = COBD(db, client_uuid, name)
- else:
- self.mdc = MDC(db, client_uuid, name)
-
- def get_uuid(self):
- return self.mdc.uuid
-
- def get_name(self):
- return self.mdc.name
+ def load_module(self):
+ Module.load_module(self)
+ self.osc.load_module()
+ def cleanup_module(self):
+ self.osc.cleanup_module()
+ Module.cleanup_module(self)
- def prepare(self):
- self.mdc.prepare()
-
- def cleanup(self):
- self.mdc.cleanup()
-
- def add_module(self, manager):
- self.mdc.add_module(manager)
-
- def correct_level(self, level, op=None):
- return self.mdc.correct_level(level, op)
class ECHO_CLIENT(Module):
def __init__(self,db):
Module.__init__(self, 'ECHO_CLIENT', db)
+ self.add_lustre_module('obdecho', 'obdecho')
self.obd_uuid = self.db.get_first_ref('obd')
obd = self.db.lookup(self.obd_uuid)
self.uuid = generate_client_uuid(self.name)
self.osc = VOSC(obd, self.uuid, self.name)
def prepare(self):
- if not config.record and is_prepared(self.name):
+ if is_prepared(self.name):
return
- run_acceptors()
self.osc.prepare() # XXX This is so cheating. -p
self.info(self.obd_uuid)
Module.cleanup(self)
self.osc.cleanup()
- def add_module(self, manager):
- self.osc.add_module(manager)
- manager.add_lustre_module('obdecho', 'obdecho')
+ def load_module(self):
+ self.osc.load_module()
+ Module.load_module(self)
+
+ def cleanup_module(self):
+ Module.cleanup_module(self)
+ self.osc.cleanup_module()
- def correct_level(self, level, op=None):
- return level
def generate_client_uuid(name):
client_uuid = '%05x_%.19s_%05x%05x' % (int(random.random() * 1048576),
int(random.random() * 1048576))
return client_uuid[:36]
+
+def my_rstrip(s, chars):
+ """my_rstrip(s, chars) -> strips any instances of the characters
+ found in chars from the right side of string s"""
+ # XXX required because python versions pre 2.2.3 don't allow
+ #string.rstrip() to take alternate char lists
+ import string
+ ns=s
+ try:
+ ns = string.rstrip(s, '/')
+ except TypeError, e:
+ for i in range(len(s) - 1, 0, -1):
+ if s[i] in chars:
+ continue
+ else:
+ ns = s[0:i+1]
+ break
+ return ns
+
+
class Mountpoint(Module):
def __init__(self,db):
Module.__init__(self, 'MTPT', db)
- self.path = self.db.get_val('path')
+ self.path = my_rstrip(self.db.get_val('path'), '/')
self.clientoptions = self.db.get_val('clientoptions', '')
self.fs_uuid = self.db.get_first_ref('filesystem')
fs = self.db.lookup(self.fs_uuid)
- self.mds_uuid = fs.get_first_ref('lmv')
- if not self.mds_uuid:
- self.mds_uuid = fs.get_first_ref('mds')
+ self.mds_uuid = fs.get_first_ref('mds')
+ mds_db = self.db.lookup(self.mds_uuid)
+ if config.quota:
+ quota = config.quota
+ else:
+ quota = mds_db.get_val('quota', config.quota)
self.obd_uuid = fs.get_first_ref('obd')
+ obd = self.db.lookup(self.obd_uuid)
client_uuid = generate_client_uuid(self.name)
+ self.vosc = VOSC(obd, client_uuid, self.name, quota=quota)
+ self.mdc = get_mdc(db, client_uuid, self.name, self.mds_uuid)
+
+ self.add_lustre_module('mdc', 'mdc')
+ self.add_lustre_module('llite', 'llite')
- ost = self.db.lookup(self.obd_uuid)
- if not ost:
- panic("no ost: ", self.obd_uuid)
-
- mds = self.db.lookup(self.mds_uuid)
- if not mds:
- panic("no mds: ", self.mds_uuid)
-
- self.vosc = VOSC(ost, client_uuid, self.name, self.name)
- self.vmdc = VMDC(mds, client_uuid, self.name, self.name)
-
def prepare(self):
- if not config.record and fs_is_mounted(self.path):
+ if fs_is_mounted(self.path):
log(self.path, "already mounted.")
return
- run_acceptors()
-
- self.vosc.prepare()
- self.vmdc.prepare()
+ self.vosc.prepare()
+ self.mdc.prepare()
+ mdc_name = self.mdc.name
self.info(self.path, self.mds_uuid, self.obd_uuid)
if config.record or config.lctl_dump:
- lctl.mount_option(local_node_name, self.vosc.get_name(),
- self.vmdc.get_name())
+ lctl.mount_option(local_node_name, self.vosc.get_name(), mdc_name)
return
if config.clientoptions:
# so replace it with Lustre async
self.clientoptions = string.replace(self.clientoptions, "async", "lasync")
- if not config.sec:
- config.sec = "null"
- cmd = "mount -t lustre_lite -o osc=%s,mdc=%s,sec=%s%s %s %s" % \
- (self.vosc.get_name(), self.vmdc.get_name(), config.sec,
- self.clientoptions, config.config, self.path)
+ cmd = "mount -t lustre_lite -o osc=%s,mdc=%s%s %s %s" % \
+ (self.vosc.get_name(), mdc_name, self.clientoptions, config.config, self.path)
run("mkdir", self.path)
ret, val = run(cmd)
if ret:
- self.vmdc.cleanup()
+ self.mdc.cleanup()
self.vosc.cleanup()
panic("mount failed:", self.path, ":", string.join(val))
if fs_is_mounted(self.path):
panic("fs is still mounted:", self.path)
- self.vmdc.cleanup()
+ self.mdc.cleanup()
self.vosc.cleanup()
- def add_module(self, manager):
- self.vosc.add_module(manager)
- self.vmdc.add_module(manager)
- manager.add_lustre_module('llite', 'llite')
+ def load_module(self):
+ self.vosc.load_module()
+ Module.load_module(self)
+
+ def cleanup_module(self):
+ Module.cleanup_module(self)
+ self.vosc.cleanup_module()
- def correct_level(self, level, op=None):
- return level
# ============================================================
# misc query functions
node = self.lookup(node_uuid)
if not node:
panic("unable to find node for osd_uuid:", osd_uuid,
- " node_ref:", node_uuid_)
+ " node_ref:", node_uuid)
for net_uuid in node.get_networks():
db = node.lookup(net_uuid)
- srv_list.append(Network(db))
+ net = Network(db, node_uuid)
+ srv_list.append(net)
return srv_list
-# the order of iniitailization is based on level.
+
+# the order of iniitailization is based on level.
def getServiceLevel(self):
type = self.get_class()
ret=0;
if type in ('network',):
ret = 5
- elif type in ('routetbl',):
- ret = 6
elif type in ('ldlm',):
ret = 20
elif type in ('osd', 'cobd'):
ret = 30
elif type in ('mdsdev',):
ret = 40
- elif type in ('lmv',):
- ret = 45
- elif type in ('cmobd',):
- ret = 50
elif type in ('mountpoint', 'echoclient'):
ret = 70
else:
osc = OSC(ost_db, uuid, fs_name)
return osc
-def get_mdc(db, fs_name, mds_uuid):
+def get_mdc(db, uuid, fs_name, mds_uuid):
mds_db = db.lookup(mds_uuid);
if not mds_db:
- error("no mds:", mds_uuid)
- mdc = MDC(mds_db, mds_uuid, fs_name)
+ panic("no mds:", mds_uuid)
+ mdc = MDC(mds_db, uuid, fs_name)
return mdc
-############################################################
-# routing ("rooting")
-
-# list of (nettype, cluster_id, nid)
-local_clusters = []
-
-def find_local_clusters(node_db):
- global local_clusters
- for netuuid in node_db.get_networks():
- net = node_db.lookup(netuuid)
- srv = Network(net)
- debug("add_local", netuuid)
- local_clusters.append((srv.net_type, srv.cluster_id, srv.nid))
- if srv.port > 0:
- if not acceptors.has_key(srv.port):
- acceptors[srv.port] = AcceptorHandler(srv.port, srv.net_type)
-
-# This node is a gateway.
-is_router = 0
-def node_is_router():
- return is_router
-
-# If there are any routers found in the config, then this will be true
-# and all nodes will load kptlrouter.
-needs_router = 0
-def node_needs_router():
- return needs_router or is_router
-
-# list of (nettype, gw, tgt_cluster_id, lo, hi)
-# Currently, these local routes are only added to kptlrouter route
-# table if they are needed to connect to a specific server. This
-# should be changed so all available routes are loaded, and the
-# ptlrouter can make all the decisions.
-local_routes = []
-
-def find_local_routes(lustre):
- """ Scan the lustre config looking for routers . Build list of
- routes. """
- global local_routes, needs_router
- local_routes = []
- list = lustre.lookup_class('node')
- for router in list:
- if router.get_val_int('router', 0):
- needs_router = 1
- for (local_type, local_cluster_id, local_nid) in local_clusters:
- gw = None
- for netuuid in router.get_networks():
- db = router.lookup(netuuid)
- if (local_type == db.get_val('nettype') and
- local_cluster_id == db.get_val('clusterid')):
- gw = db.get_val('nid')
- break
- if gw:
- debug("find_local_routes: gw is", gw)
- for route in router.get_local_routes(local_type, gw):
- local_routes.append(route)
- debug("find_local_routes:", local_routes)
-
-
-def choose_local_server(srv_list):
- for srv in srv_list:
- if local_cluster(srv.net_type, srv.cluster_id):
- return srv
-
-def local_cluster(net_type, cluster_id):
- for cluster in local_clusters:
- if net_type == cluster[0] and cluster_id == cluster[1]:
- return 1
- return 0
-
-def local_interface(net_type, cluster_id, nid):
- for cluster in local_clusters:
- if (net_type == cluster[0] and cluster_id == cluster[1]
- and nid == cluster[2]):
- return 1
- return 0
-
-def find_route(srv_list):
- result = []
- frm_type = local_clusters[0][0]
- for srv in srv_list:
- debug("find_route: srv:", srv.nid, "type: ", srv.net_type)
- to_type = srv.net_type
- to = srv.nid
- cluster_id = srv.cluster_id
- debug ('looking for route to', to_type, to)
- for r in local_routes:
- debug("find_route: ", r)
- if (r[3] <= to and to <= r[4]) and cluster_id == r[2]:
- result.append((srv, r))
- return result
-
def get_active_target(db):
target_uuid = db.getUUID()
target_name = db.getName()
net = Network(n)
if net.nid_uuid == nid_uuid:
return net
-
+
############################################################
# lconf level logic
n = LOV(db, "YOU_SHOULD_NEVER_SEE_THIS_UUID")
elif type == 'network':
n = Network(db)
- elif type == 'routetbl':
- n = RouteTable(db)
elif type == 'osd':
n = OSD(db)
elif type == 'cobd':
- n = COBD(db, "YOU_SHOULD_NEVER_SEE_THIS_UUID")
- elif type == 'cmobd':
- n = CMOBD(db)
+ n = COBD(db)
elif type == 'mdsdev':
n = MDSDEV(db)
elif type == 'mountpoint':
n = Mountpoint(db)
elif type == 'echoclient':
n = ECHO_CLIENT(db)
- elif type == 'lmv':
- n = LMV(db)
else:
- panic ("unknown service type:", type)
+ panic("unknown service type:", type)
return n
#
# Prepare the system to run lustre using a particular profile
-# in a the configuration.
+# in a the configuration.
# * load & the modules
# * setup networking for the current node
# * make sure partitions are in place and prepared
services = getServices(prof_db)
operation(services)
-def magic_get_osc(db, rec, lov):
- if lov:
- lov_uuid = lov.get_uuid()
- lov_name = lov.osc.fs_name
- else:
- lov_uuid = rec.getAttribute('lov_uuidref')
- # FIXME: better way to find the mountpoint?
- filesystems = db.root_node.getElementsByTagName('filesystem')
- fsuuid = None
- for fs in filesystems:
- ref = fs.getElementsByTagName('obd_ref')
- if ref[0].getAttribute('uuidref') == lov_uuid:
- fsuuid = fs.getAttribute('uuid')
- break
-
- if not fsuuid:
- panic("malformed xml: lov uuid '" + lov_uuid + "' referenced in 'add' record is not used by any filesystems.")
-
- mtpts = db.root_node.getElementsByTagName('mountpoint')
- lov_name = None
- for fs in mtpts:
- ref = fs.getElementsByTagName('filesystem_ref')
- if ref[0].getAttribute('uuidref') == fsuuid:
- lov_name = fs.getAttribute('name')
- break
-
- if not lov_name:
- panic("malformed xml: 'add' record references lov uuid '" + lov_uuid + "', which references filesystem uuid '" + fsuuid + "', which does not reference a mountpoint.")
-
- print "lov_uuid: " + lov_uuid + "; lov_name: " + lov_name
-
- ost_uuid = rec.getAttribute('ost_uuidref')
- obd = db.lookup(ost_uuid)
-
- if not obd:
- panic("malformed xml: 'add' record references ost uuid '" + ost_uuid + "' which cannot be found.")
-
- osc = get_osc(obd, lov_uuid, lov_name)
- if not osc:
- panic('osc not found:', obd_uuid)
- return osc
-
-# write logs for update records. sadly, logs of all types -- and updates in
-# particular -- are something of an afterthought. lconf needs rewritten with
-# these as core concepts. so this is a pretty big hack.
-def process_update_record(db, update, lov):
- for rec in update.childNodes:
- if rec.nodeType != rec.ELEMENT_NODE:
- continue
-
- log("found "+rec.nodeName+" record in update version " +
- str(update.getAttribute('version')))
-
- lov_uuid = rec.getAttribute('lov_uuidref')
- ost_uuid = rec.getAttribute('ost_uuidref')
- index = rec.getAttribute('index')
- gen = rec.getAttribute('generation')
-
- if not lov_uuid or not ost_uuid or not index or not gen:
- panic("malformed xml: 'update' record requires lov_uuid, ost_uuid, index, and generation.")
-
- if not lov:
- tmplov = db.lookup(lov_uuid)
- if not tmplov:
- panic("malformed xml: 'delete' record contains lov UUID '" + lov_uuid + "', which cannot be located.")
- lov_name = tmplov.getName()
- else:
- lov_name = lov.osc.name
-
- # ------------------------------------------------------------- add
- if rec.nodeName == 'add':
- if config.cleanup:
- lctl.lov_del_obd(lov_name, lov_uuid, ost_uuid, index, gen)
- continue
-
- osc = magic_get_osc(db, rec, lov)
-
- try:
- # Only ignore connect failures with --force, which
- # isn't implemented here yet.
- osc.prepare(ignore_connect_failure=0)
- except CommandError, e:
- print "Error preparing OSC %s\n" % osc.uuid
- raise e
-
- lctl.lov_add_obd(lov_name, lov_uuid, ost_uuid, index, gen)
-
- # ------------------------------------------------------ deactivate
- elif rec.nodeName == 'deactivate':
- if config.cleanup:
- continue
-
- osc = magic_get_osc(db, rec, lov)
-
- try:
- osc.deactivate()
- except CommandError, e:
- print "Error deactivating OSC %s\n" % osc.uuid
- raise e
-
- # ---------------------------------------------------------- delete
- elif rec.nodeName == 'delete':
- if config.cleanup:
- continue
-
- osc = magic_get_osc(db, rec, lov)
-
- try:
- config.cleanup = 1
- osc.cleanup()
- config.cleanup = 0
- except CommandError, e:
- print "Error cleaning up OSC %s\n" % osc.uuid
- raise e
-
- lctl.lov_del_obd(lov_name, lov_uuid, ost_uuid, index, gen)
-
-def process_updates(db, log_device, log_name, lov = None):
- updates = db.root_node.getElementsByTagName('update')
- for u in updates:
- if not u.childNodes:
- log("ignoring empty update record (version " +
- str(u.getAttribute('version')) + ")")
- continue
-
- version = u.getAttribute('version')
- real_name = "%s-%s" % (log_name, version)
- lctl.clear_log(log_device, real_name)
- lctl.record(log_device, real_name)
-
- process_update_record(db, u, lov)
-
- lctl.end_record()
-
def doWriteconf(services):
- #if config.nosetup:
- # return
+ if config.nosetup:
+ return
+ have_mds = 0
for s in services:
- if s[1].get_class() == 'mdsdev' or s[1].get_class() == 'osd':
+ if s[1].get_class() == 'mdsdev':
n = newService(s[1])
n.write_conf()
- n.cleanup()
+ have_mds = 1
+ if have_mds == 0:
+ panic("Cannot find mds device, please run --write_conf on the mds node.")
+
def doSetup(services):
if config.nosetup:
return
- slist = []
for s in services:
n = newService(s[1])
- n.level = s[0]
- slist.append((n.level, n))
- nlist = []
- for n in slist:
- nl = n[1].correct_level(n[0])
- nlist.append((nl, n[1]))
- nlist.sort()
- for n in nlist:
- n[1].prepare()
-
-def doLoadModules(services):
+ n.prepare()
+
+def doModules(services):
if config.nomod:
return
-
- # adding all needed modules from all services
for s in services:
n = newService(s[1])
- n.add_module(mod_manager)
-
- # loading all registered modules
- mod_manager.load_modules()
+ n.load_module()
+
+def doCleanup(services):
+ if config.nosetup:
+ return
+ services.reverse()
+ for s in services:
+ n = newService(s[1])
+ if n.safe_to_clean():
+ n.cleanup()
def doUnloadModules(services):
if config.nomod:
return
-
- # adding all needed modules from all services
+ services.reverse()
for s in services:
n = newService(s[1])
if n.safe_to_clean_modules():
- n.add_module(mod_manager)
-
- # unloading all registered modules
- mod_manager.cleanup_modules()
+ n.cleanup_module()
-def doCleanup(services):
+def doMakeServiceScript(services):
if config.nosetup:
return
- slist = []
-
+ try:
+ os.makedirs(config.service_scripts)
+ except OSError, e:
+ if e[0] != errno.EEXIST:
+ panic("Couldn't create scripts dir " + config.service_scripts + ": " + e[1])
+
for s in services:
- n = newService(s[1])
- n.level = s[0]
- slist.append((n.level, n))
- nlist = []
- for n in slist:
- nl = n[1].correct_level(n[0])
- nlist.append((nl, n[1]))
- nlist.sort()
- nlist.reverse()
-
- for n in nlist:
- if n[1].safe_to_clean():
- n[1].cleanup()
+ if s[1].get_class() != 'osd' and s[1].get_class() != 'mdsdev':
+ continue
+
+ target_uuid = s[1].get_first_ref('target')
+ target = toplustreDB.lookup(target_uuid)
+ target_symlink = config.service_scripts + "/" + target.getName()
+ if config.force:
+ try:
+ try:
+ os.unlink(target_symlink)
+ if config.verbose:
+ print "Removed " + target_symlink
+ except OSError, e:
+ if e[0] != errno.EISDIR:
+ raise e
+ os.rmdir(target_symlink)
+ if config.verbose:
+ print "Removed " + target_symlink
+ except OSError, e:
+ if e[0] != errno.ENOENT:
+ panic("Error removing " + target_symlink + ": " + e[1])
+
+ try:
+ os.symlink("/etc/init.d/lustre", target_symlink)
+ if config.verbose:
+ print "Created service link " + target_symlink + " to /etc/init.d/lustre"
+
+ except OSError, e:
+ if e[0] == errno.EEXIST:
+ extra_error = " (use --force option to remove existing files)"
+ else:
+ extra_error = ""
+ panic("Error creating " + target_symlink + ": " + e[1] + extra_error)
+
+# Check mtime of config logs
+def doCheckMtime(lustreDB, hosts):
+ for h in hosts:
+ node_db = lustreDB.lookup_name(h, 'node')
+ if node_db:
+ break
+ if not node_db:
+ return
+
+ mdsdb = 0
+ prof_list = node_db.get_refs('profile')
+ for prof_uuid in prof_list:
+ prof_db = node_db.lookup(prof_uuid)
+ if prof_db:
+ services = getServices(prof_db)
+ for s in services:
+ if s[1].get_class() == 'mdsdev':
+ mdsdb = s[1]
+ break
+
+ if mdsdb and lustreDB.get_mtime():
+ debug("Checking XML modification time")
+ devpath = mdsdb.get_val('devpath','')
+ xmtime = string.atol(lustreDB.get_mtime())
+ cmd = "debugfs -c -R 'stat /LOGS' %s 2>&1 | grep mtime" %devpath
+ ret, kmtimes = runcmd(cmd)
+ if ret:
+ log("Can not get mtime info of MDS LOGS directory")
+ else:
+ kmtime = string.atoi(string.split(kmtimes[0])[1], 0)
+ if xmtime > kmtime:
+ debug('xmtime ', xmtime, '> kmtime', kmtime)
+ if config.old_conf:
+ log("Warning: MDS startup logs are older than config %s."
+ " Please run --write_conf on stopped MDS to update."
+ %CONFIG_FILE)
+ else:
+ panic("Error: MDS startup logs are older than config %s."
+ " Please run --write_conf on stopped MDS to update."
+ " Use '--old_conf' to start anyways." %CONFIG_FILE)
+ return
#
-# Load profile for
+# Load profile for
def doHost(lustreDB, hosts):
- global is_router, local_node_name
+ global local_node_name, tgt_select
node_db = None
for h in hosts:
node_db = lustreDB.lookup_name(h, 'node')
if node_db:
+ if config.service:
+ tgt_select[config.service] = h
+ config.group = config.service
break
if not node_db:
panic('No host entry found.')
local_node_name = node_db.get_val('name', 0)
- is_router = node_db.get_val_int('router', 0)
lustre_upcall = node_db.get_val('lustreUpcall', '')
portals_upcall = node_db.get_val('portalsUpcall', '')
timeout = node_db.get_val_int('timeout', 0)
ptldebug = node_db.get_val('ptldebug', '')
subsystem = node_db.get_val('subsystem', '')
-
- find_local_clusters(node_db)
- if not is_router:
- find_local_routes(lustreDB)
# Two step process: (1) load modules, (2) setup lustre
# if not cleaning, load modules first.
prof_list = node_db.get_refs('profile')
- if config.write_conf:
- for_each_profile(node_db, prof_list, doLoadModules)
- sys_make_devices()
+ if config.make_service_scripts:
+ for_each_profile(node_db, prof_list, doMakeServiceScript)
+ return
+
+ elif config.write_conf:
+ for_each_profile(node_db, prof_list, doModules)
for_each_profile(node_db, prof_list, doWriteconf)
for_each_profile(node_db, prof_list, doUnloadModules)
lustreDB.close()
doRecovery(lustreDB, lctl, config.tgt_uuid, config.client_uuid,
config.conn_uuid)
elif config.cleanup:
- if config.force:
- # the command line can override this value
- timeout = 5
+ if not mod_loaded('lnet'):
+ return
+
# ugly hack, only need to run lctl commands for --dump
if config.lctl_dump or config.record:
for_each_profile(node_db, prof_list, doCleanup)
return
- sys_set_timeout(timeout)
sys_set_ptldebug(ptldebug)
sys_set_subsystem(subsystem)
sys_set_lustre_upcall(lustre_upcall)
for_each_profile(node_db, prof_list, doSetup)
return
- sys_make_devices()
- sys_set_netmem_max('/proc/sys/net/core/rmem_max', MAXTCPBUF)
- sys_set_netmem_max('/proc/sys/net/core/wmem_max', MAXTCPBUF)
+ if PLATFORM == 'LINUX':
+ sys_set_netmem_max('/proc/sys/net/core/rmem_max', MAXTCPBUF)
+ sys_set_netmem_max('/proc/sys/net/core/wmem_max', MAXTCPBUF)
- for_each_profile(node_db, prof_list, doLoadModules)
+ for_each_profile(node_db, prof_list, doModules)
- sys_set_debug_path()
- sys_set_ptldebug(ptldebug)
- sys_set_subsystem(subsystem)
+ if PLATFORM == 'LINUX':
+ # XXX need to be fixed for Darwin
+ sys_set_debug_path()
+ sys_set_ptldebug(ptldebug)
+ sys_set_subsystem(subsystem)
script = config.gdb_script
run(lctl.lctl, ' modules >', script)
if config.gdb:
for_each_profile(node_db, prof_list, doSetup)
lustreDB.close()
+def add_clumanager_node(node_db, nodes, services):
+ new_services = []
+ node_name = node_db.getUUID()
+ nodes[node_name] = []
+
+ for prof_uuid in node_db.get_refs('profile'):
+ prof_db = toplustreDB.lookup(prof_uuid)
+ for ref_class, ref_uuid in prof_db.get_all_refs():
+ if ref_class not in ('osd', 'mdsdev'):
+ continue
+ devdb = toplustreDB.lookup(ref_uuid)
+ tgt_uuid = devdb.get_first_ref('target')
+
+ nodes[node_name].append(ref_uuid)
+
+ if not services.has_key(tgt_uuid):
+ if config.verbose:
+ print "New service: " + tgt_uuid + " (originally found on " + node_name + ")"
+ new_services.append(tgt_uuid)
+ services[tgt_uuid] = []
+ services[tgt_uuid].append(ref_uuid)
+
+ return new_services
+
+def add_clumanager_services(new_services, nodes, dev_list):
+ new_nodes = []
+ for devdb in dev_list:
+ tgt_uuid = devdb.get_first_ref('target')
+ if tgt_uuid in new_services:
+ node_uuid = devdb.get_first_ref('node')
+
+ if not (nodes.has_key(node_uuid) or node_uuid in new_nodes):
+ if config.verbose:
+ print "New node: " + node_uuid + " for service " + tgt_uuid
+ new_nodes.append(node_uuid)
+
+ return new_nodes
+
+def doClumanager(lustreDB, hosts):
+ nodes = {}
+ services = {}
+
+ dev_list = []
+
+ for dev_uuid in toplustreDB.get_refs('osd') + toplustreDB.get_refs('mdsdev'):
+ dev_list.append(lustreDB.lookup(dev_uuid))
+
+ node_db = None
+ for h in hosts:
+ node_db = lustreDB.lookup_name(h, 'node')
+ if node_db:
+ our_host = h
+ new_services = add_clumanager_node(node_db, nodes, services)
+ break
+
+ if not node_db:
+ panic('No host entry found.')
+
+ while 1:
+ if len(new_services) == 0:
+ break
+
+ new_nodes = add_clumanager_services(new_services, nodes, dev_list)
+ if len(new_nodes) == 0:
+ break
+
+ if len(new_nodes) + len(nodes.keys()) > 8:
+ panic("CluManager only supports 8 nodes per failover \"cluster.\"")
+
+ new_services = []
+ for node_uuid in new_nodes:
+ node_db = lustreDB.lookup(node_uuid)
+ if not node_db:
+ panic("No node entry for " + node_uuid + " was found.")
+
+ new_services.append(add_clumanager_node(node_db, nodes, services))
+
+ nodenames = []
+ for node in nodes.keys():
+ nodedb = lustreDB.lookup(node)
+ nodenames.append(nodedb.getName())
+ nodenames.sort()
+
+ print """<?xml version="1.0"?>
+<cluconfig version="3.0">
+ <clumembd broadcast="no" interval="750000" loglevel="5" multicast="yes" multicast_ipaddress="225.0.0.11" thread="yes" tko_count="20"/>
+ <cluquorumd loglevel="5" pinginterval="2"/>
+ <clurmtabd loglevel="5" pollinterval="4"/>
+ <clusvcmgrd loglevel="5"/>
+ <clulockd loglevel="5"/>
+ <cluster config_viewnumber="1" name="%s"/>
+ <sharedstate driver="libsharedraw.so" rawprimary="%s" rawshadow="%s" type="raw"/>
+ <members> """ % (string.join(nodenames), config.rawprimary, config.rawsecondary)
+
+
+ i = 0
+ for node in nodenames:
+ print " <member id=\"%d\" name=\"%s\" watchdog=\"yes\"/>" % (i, node)
+ i = i + 1
+
+ print " </members>\n <failoverdomains>"
+
+ servicekeys = services.keys()
+ servicekeys.sort()
+
+ i = 0
+ for service in servicekeys:
+ svcdb = lustreDB.lookup(service)
+ print " <failoverdomain id=\"%d\" name=\"%s\" ordered=\"yes\" restricted=\"yes\">" % (i, svcdb.getName())
+ i = i + 1
+
+ j = 0
+ active_uuid = get_active_target(svcdb)
+ for svc_uuid in [active_uuid] + services[service]:
+ if svc_uuid == active_uuid and j > 0:
+ continue
+ svcdb = lustreDB.lookup(svc_uuid)
+
+ svc_node_uuid = svcdb.get_first_ref('node')
+ svc_nodedb = lustreDB.lookup(svc_node_uuid)
+
+ print " <failoverdomainnode id=\"%d\" name=\"%s\"/>" % (j, svc_nodedb.getName())
+ j = j + 1
+
+ print " </failoverdomain>"
+
+ print " </failoverdomains>\n <services>"
+
+ i = 0
+ for service in servicekeys:
+ svcdb = lustreDB.lookup(service)
+ active_uuid = get_active_target(svcdb)
+ activedb = lustreDB.lookup(active_uuid)
+
+ svc_node_uuid = activedb.get_first_ref('node')
+ svc_nodedb = lustreDB.lookup(svc_node_uuid)
+
+ print " <service checkinterval=\"30\" failoverdomain=\"%s\" id=\"%d\" name=\"%s\" userscript=\"%s/%s\">" \
+ % ( svcdb.getName(), i, svcdb.getName(), config.service_scripts, svcdb.getName())
+ print " <service_ipaddresses/>\n </service>"
+ i = i + 1
+
+ print " </services>\n</cluconfig>"
+
def doRecovery(lustreDB, lctl, tgt_uuid, client_uuid, nid_uuid):
tgt = lustreDB.lookup(tgt_uuid)
if not tgt:
if not new_uuid:
raise Lustre.LconfError("doRecovery: no active target found for: " +
tgt_uuid)
- net = choose_local_server(get_ost_net(lustreDB, new_uuid))
- if not net:
+ srv_list = find_local_servers(get_ost_net(lustreDB, new_uuid))
+ if not srv_list[0]:
raise Lustre.LconfError("Unable to find a connection to:" + new_uuid)
- log("Reconnecting", tgt_uuid, " to ", net.nid_uuid);
- try:
- oldnet = get_server_by_nid_uuid(lustreDB, nid_uuid)
- lustreDB.close()
- if oldnet:
- lctl.disconnect(oldnet)
- except CommandError, e:
- log("recover: disconnect", nid_uuid, "failed: ")
- e.dump()
+ oldsrv = get_server_by_nid_uuid(lustreDB, nid_uuid)
+ lustreDB.close()
- try:
- lctl.connect(net)
- except CommandError, e:
- log("recover: connect failed")
- e.dump()
+ for srv in srv_list:
+ if oldsrv.net_type != srv.net_type:
+ continue
+
+ log("Reconnecting", tgt_uuid, "to", srv.nid_uuid)
- lctl.recover(client_uuid, net.nid_uuid)
+ lctl.recover(client_uuid, srv.nid_uuid)
def setupModulePath(cmd, portals_dir = PORTALS_DIR):
base = os.path.dirname(cmd)
if development_mode():
if not config.lustre:
- debug('using objdir module paths')
+ debug('using objdir module paths')
config.lustre = (os.path.join(base, ".."))
# normalize the portals dir, using command line arg if set
if config.portals:
debug('config.portals', config.portals)
elif config.lustre and config.portals:
# production mode
- # if --lustre and --portals, normalize portals
+ # if --lustre and --portals, normalize portals
# can ignore POTRALS_DIR here, since it is probly useless here
config.portals = os.path.join(config.lustre, config.portals)
debug('config.portals B', config.portals)
def sys_set_debug_path():
- sysctl('portals/debug_path', config.debug_path)
+ sysctl('lnet/debug_path', config.debug_path)
+
+def validate_upcall(upcall):
+ import os
+ if upcall in ('DEFAULT','NONE'):
+ pass
+ elif os.path.exists(upcall):
+ if not os.access(upcall, os.X_OK):
+ print "WARNING upcall script not executable: %s" % upcall
+ else:
+ print "WARNING invalid upcall script specified: %s" % upcall
def sys_set_lustre_upcall(upcall):
- # the command overrides the value in the node config
+ # the command line overrides the value in the node config
if config.lustre_upcall:
upcall = config.lustre_upcall
elif config.upcall:
upcall = config.upcall
if upcall:
+ validate_upcall(upcall)
lctl.set_lustre_upcall(upcall)
def sys_set_portals_upcall(upcall):
- # the command overrides the value in the node config
+ # the command line overrides the value in the node config
if config.portals_upcall:
upcall = config.portals_upcall
elif config.upcall:
upcall = config.upcall
if upcall:
- sysctl('portals/upcall', upcall)
+ validate_upcall(upcall)
+ sysctl('lnet/upcall', upcall)
+
+def sys_set_group_upcall(mds, upcall):
+ if config.noexec:
+ return
+ # the command line overrides the value in the MDS config
+ if config.group_upcall:
+ upcall = config.group_upcall
+ if upcall:
+ validate_upcall(upcall)
+ debug("setting MDS", mds, "upcall to:", upcall)
+ path = "/proc/fs/lustre/mds/" + mds + "/group_upcall"
+ fp = open(path, 'w')
+ fp.write(upcall)
+ fp.close()
def sys_set_timeout(timeout):
# the command overrides the value in the node config
lctl.set_timeout(timeout)
def sys_tweak_socknal ():
- # reserve at least 8MB, or we run out of RAM in skb_alloc under read
- if sys_get_branch() == '2.6':
- fp = open('/proc/meminfo')
- lines = fp.readlines()
- fp.close()
- memtotal = 131072
- for l in lines:
- a = string.split(l)
- if a[0] == 'MemTotal:':
- memtotal = a[1]
- debug("memtotal" + memtotal)
- if int(memtotal) < 262144:
- minfree = int(memtotal) / 16
- else:
- minfree = 32768
- debug("+ minfree ", minfree)
- sysctl("vm/min_free_kbytes", minfree)
if config.single_socket:
sysctl("socknal/typed", 0)
if ptldebug:
try:
val = eval(ptldebug, ptldebug_names)
- val = "0x%x" % (val & 0xffffffffL)
- sysctl('portals/debug', val)
+ val = "0x%x" % (val)
+ sysctl('lnet/debug', val)
except NameError, e:
panic(str(e))
if subsystem:
try:
val = eval(subsystem, subsystem_names)
- val = "0x%x" % (val & 0xffffffffL)
- sysctl('portals/subsystem_debug', val)
+ val = "0x%x" % (val)
+ sysctl('lnet/subsystem_debug', val)
except NameError, e:
panic(str(e))
fp = open(path, 'w')
fp.write('%d\n' %(max))
fp.close()
-
-def sys_make_devices():
- if not os.access('/dev/portals', os.R_OK):
- run('mknod /dev/portals c 10 240')
- if not os.access('/dev/obd', os.R_OK):
- run('mknod /dev/obd c 10 241')
+
# Add dir to the global PATH, if not already there.
def add_to_path(new_dir):
if new_dir in syspath:
return
os.environ['PATH'] = os.environ['PATH'] + ':' + new_dir
-
+
def default_debug_path():
path = '/tmp/lustre-log'
if os.path.isdir('/r'):
tgt_select = {}
def init_select(args):
# args = [service=nodeA,service2=nodeB service3=nodeC]
+ # --service <service> is analagous to:
+ # --group <service> --select <service>=<node>
+ # this is handled in doHost()
global tgt_select
for arg in args:
list = string.split(arg, ',')
('ldapurl',"LDAP server URL, eg. ldap://localhost", PARAM),
('config', "Cluster config name used for LDAP query", PARAM),
('select', "service=nodeA,service2=nodeB ", PARAMLIST),
+ ('service', "shorthand for --group <service> --select <service>=<node>", PARAM),
('node', "Load config for <nodename>", PARAM),
- ('sec', "security flavor <null|krb5i|krb5p> of client", PARAM),
- ('mds_mds_sec', "security flavor <null|krb5i|krb5p> of inter mds's", PARAM),
- ('mds_ost_sec', "security flavor <null|krb5i|krb5p> of mds's-ost's", PARAM),
('cleanup,d', "Cleans up config. (Shutdown)"),
('force,f', "Forced unmounting and/or obd detach during cleanup",
FLAG, 0),
another node for failover purposes. This will not
be a clean shutdown.""",
FLAG, 0),
+ ('abort_recovery',"""Used to start a service when you know recovery
+ will not succeed. This will skip the recovery
+ timeout period."""),
('gdb', """Prints message after creating gdb module script
and sleeps for 5 seconds."""),
('noexec,n', """Prints the commands and steps that will be run for a
('dump', "Dump the kernel debug log to file before portals is unloaded",
PARAM),
('write_conf', "Save all the client config information on mds."),
+ ('old_conf', "Start up service even though config logs appear outdated."),
('record', "Write config information on mds."),
('record_log', "Name of config record log.", PARAM),
('record_device', "MDS device name that will record the config commands",
PARAM),
- ('root_squash', "MDS squash root to appointed uid",
- PARAM),
- ('no_root_squash', "Don't squash root for appointed nid",
- PARAM),
('minlevel', "Minimum level of services to configure/cleanup",
INTPARAM, 0),
- ('maxlevel', """Maximum level of services to configure/cleanup
+ ('maxlevel', """Maximum level of services to configure/cleanup
Levels are aproximatly like:
- 10 - netwrk
+ 10 - network
20 - device, ldlm
30 - osd, mdd
40 - mds, ost
('upcall', "Set both portals and lustre upcall script", PARAM),
('lustre_upcall', "Set lustre upcall script", PARAM),
('portals_upcall', "Set portals upcall script", PARAM),
+ ('group_upcall', "Set supplementary group upcall program", PARAM),
('lctl_dump', "Save lctl ioctls to the dumpfile argument", PARAM),
('ptldebug', "Set the portals debug level", PARAM),
('subsystem', "Set the portals debug subsystem", PARAM),
('gdb_script', "Fullname of gdb debug script", PARAM, default_gdb_script()),
('debug_path', "Path to save debug dumps", PARAM, default_debug_path()),
+ ('allow_unprivileged_port', "Allow connections from unprivileged ports"),
+ ('clumanager', "Generate CluManager config file for this node's cluster"),
+ ('rawprimary', "For clumanager, device of the primary quorum", PARAM, "/dev/raw/raw1"),
+ ('rawsecondary', "For clumanager, device of the secondary quorum", PARAM, "/dev/raw/raw2"),
+ ('service_scripts', "For clumanager, directory containing per-service scripts", PARAM, "/etc/lustre/services"),
+ ('make_service_scripts', "Create per-service symlinks for use with clumanager"),
# Client recovery options
('recover', "Recover a device"),
- ('group', "The group of devices to configure or cleanup", PARAM),
+ ('group,g', "The group of devices to configure or cleanup", PARAM),
('tgt_uuid', "The failed target (required for recovery)", PARAM),
('client_uuid', "The failed client (required for recovery)", PARAM),
('conn_uuid', "The failed connection (required for recovery)", PARAM),
('inactive', """The name of an inactive service, to be ignored during
mounting (currently OST-only). Can be repeated.""",
PARAMLIST),
- ]
+ ('user_xattr', """Enable user_xattr support on MDS""", FLAG, 0),
+ ('acl', """Enable ACL support on MDS""", FLAG, 0),
+ ('quota', "Enable quota support for client file system", PARAM),
+ ]
def main():
- global lctl, config, toplustreDB, CONFIG_FILE, mod_manager
+ global lctl, config, toplustreDB, CONFIG_FILE
# in the upcall this is set to SIG_IGN
signal.signal(signal.SIGCHLD, signal.SIG_DFL)
-
+
cl = Lustre.Options("lconf", "config.xml", lconf_options)
try:
config, args = cl.parse(sys.argv[1:])
random.seed(seed)
sanitise_path()
-
+
init_select(config.select)
if len(args) > 0:
print 'see lconf --help for command summary'
sys.exit(1)
+ if config.reformat and config.cleanup:
+ panic("Options \"reformat\" and \"cleanup\" are incompatible. "+
+ "Please specify only one.")
+
toplustreDB = lustreDB
ver = lustreDB.get_version()
else:
if len(host) > 0:
node_list.append(host)
- node_list.append('localhost')
+# node_list.append('localhost')
debug("configuring for host: ", node_list)
if config.lctl_dump:
lctl.use_save_file(config.lctl_dump)
+ if not (config.reformat or config.write_conf or config.cleanup):
+ doCheckMtime(lustreDB, node_list)
+
if config.record:
if not (config.record_device and config.record_log):
panic("When recording, both --record_log and --record_device must be specified.")
lctl.clear_log(config.record_device, config.record_log)
lctl.record(config.record_device, config.record_log)
- # init module manager
- mod_manager = kmod_manager(config.lustre, config.portals)
-
- doHost(lustreDB, node_list)
-
- if not config.record:
- return
-
- lctl.end_record()
+ if config.clumanager:
+ doClumanager(lustreDB, node_list)
+ else:
+ doHost(lustreDB, node_list)
- process_updates(lustreDB, config.record_device, config.record_log)
+ if config.record:
+ lctl.end_record()
if __name__ == "__main__":
try:
sys.exit(1)
except CommandError, e:
e.dump()
- sys.exit(e.rc)
+ rc = e.rc
+ if rc == 0:
+ rc = 1
+ sys.exit(rc)
if first_cleanup_error:
sys.exit(first_cleanup_error)
git://git.whamcloud.com / fs / lustre-release.git / blobdiff