+++ /dev/null
-#!/usr/bin/env python
-#
-# GPL HEADER START
-#
-# DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
-#
-# This program is free software; you can redistribute it and/or modify
-# it under the terms of the GNU General Public License version 2 only,
-# as published by the Free Software Foundation.
-#
-# This program is distributed in the hope that it will be useful, but
-# WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
-# General Public License version 2 for more details (a copy is included
-# in the LICENSE file that accompanied this code).
-#
-# You should have received a copy of the GNU General Public License
-# version 2 along with this program; If not, see
-# http://www.sun.com/software/products/lustre/docs/GPLv2.pdf
-# copy of GPLv2].
-#
-# Please contact Sun Microsystems, Inc., 4150 Network Circle, Santa Clara,
-# CA 95054 USA or visit www.sun.com if you need additional information or
-# have any questions.
-#
-# GPL HEADER END
-#
-
-#
-# Copyright (c) 2002, 2010, Oracle and/or its affiliates. All rights reserved.
-# Use is subject to license terms.
-#
-
-#
-# This file is part of Lustre, http://www.lustre.org/
-# Lustre is a trademark of Sun Microsystems, Inc.
-#
-# Author: Robert Read <rread@clusterfs.com>
-# Author: Mike Shaver <shaver@clusterfs.com>
-#
-# lconf - lustre configuration tool
-#
-# lconf is the main driver script for starting and stopping
-# lustre filesystem services.
-#
-# Based in part on the XML obdctl modifications done by Brian Behlendorf
-
-import sys, getopt, types, errno
-import string, os, stat, popen2, socket, time, random, fcntl, select
-import re, exceptions, signal, traceback
-import xml.dom.minidom
-
-if sys.version[0] == '1':
- from FCNTL import F_GETFL, F_SETFL
-else:
- from fcntl import F_GETFL, F_SETFL
-
-PYMOD_DIR = ["/usr/lib64/lustre/python", "/usr/lib/lustre/python"]
-PLATFORM = ''
-KEXTPATH = ''
-if string.find(sys.platform, 'linux') != -1:
- PLATFORM='LINUX'
-elif string.find(sys.platform, 'darwin') != -1:
- PLATFORM='DARWIN'
- KEXTPATH='/System/Library/Extensions/'
-else:
- PLATFORM='Unsupported'
-
-def development_mode():
- base = os.path.dirname(sys.argv[0])
- if os.access(base+"/Makefile", os.R_OK):
- return 1
- return 0
-
-if development_mode():
- sys.path.append('../utils')
-else:
- sys.path.extend(PYMOD_DIR)
-
-import Lustre
-
-# Global parameters
-MAXTCPBUF = 16777216
-#
-# Maximum number of devices to search for.
-# (the /dev/loop* nodes need to be created beforehand)
-MAX_LOOP_DEVICES = 256
-PORTALS_DIR = '../lnet'
-
-# Needed to call lconf --record
-CONFIG_FILE = ""
-
-# Please keep these in sync with the values in lnet/include/libcfs/libcfs.h
-ptldebug_names = {
- "trace" : (1 << 0),
- "inode" : (1 << 1),
- "super" : (1 << 2),
- "ext2" : (1 << 3),
- "malloc" : (1 << 4),
- "cache" : (1 << 5),
- "info" : (1 << 6),
- "ioctl" : (1 << 7),
- "blocks" : (1 << 8),
- "net" : (1 << 9),
- "warning" : (1 << 10),
- "buffs" : (1 << 11),
- "other" : (1 << 12),
- "dentry" : (1 << 13),
- "portals" : (1 << 14), # deprecated
- "lnet" : (1 << 14),
- "page" : (1 << 15),
- "dlmtrace" : (1 << 16),
- "error" : (1 << 17),
- "emerg" : (1 << 18),
- "ha" : (1 << 19),
- "rpctrace" : (1 << 20),
- "vfstrace" : (1 << 21),
- "reada" : (1 << 22),
- "mmap" : (1 << 23),
- "config" : (1 << 24),
- "console" : (1 << 25),
- "quota" : (1 << 26),
- "sec" : (1 << 27),
- }
-
-subsystem_names = {
- "undefined" : (1 << 0),
- "mdc" : (1 << 1),
- "mds" : (1 << 2),
- "osc" : (1 << 3),
- "ost" : (1 << 4),
- "class" : (1 << 5),
- "log" : (1 << 6),
- "llite" : (1 << 7),
- "rpc" : (1 << 8),
- "lnet" : (1 << 10),
- "portals" : (1 << 10), # deprecated
- "lnd" : (1 << 11),
- "nal" : (1 << 11), # deprecated
- "pinger" : (1 << 12),
- "filter" : (1 << 13),
- "ptlbd" : (1 << 14), # deprecated
- "echo" : (1 << 15),
- "ldlm" : (1 << 16),
- "lov" : (1 << 17),
- "ptlrouter" : (1 << 18), # deprecated
- "cobd" : (1 << 19),
- "sm" : (1 << 20),
- "asobd" : (1 << 21),
- "confobd" : (1 << 22), # deprecated
- "lmv" : (1 << 23),
- "cmobd" : (1 << 24),
- "sec" : (1 << 25),
- "sec" : (1 << 26),
- "gss" : (1 << 27),
- "gks" : (1 << 28),
- "mgc" : (1 << 29),
- "mgs" : (1 << 30),
- }
-
-
-first_cleanup_error = 0
-def cleanup_error(rc):
- global first_cleanup_error
- if not first_cleanup_error:
- first_cleanup_error = rc
-
-# ============================================================
-# debugging and error funcs
-
-def fixme(msg = "this feature"):
- raise Lustre.LconfError, msg + ' not implemented yet.'
-
-def panic(*args):
- msg = string.join(map(str,args))
- if not config.noexec:
- raise Lustre.LconfError(msg)
- else:
- print "! " + msg
-
-def log(*args):
- msg = string.join(map(str,args))
- print msg
-
-def logall(msgs):
- for s in msgs:
- print string.strip(s)
-
-def debug(*args):
- # apparently, (non)execution of the following line affects mds device
- # startup order (e.g. two mds's using loopback devices), so always do it.
- msg = string.join(map(str,args))
- if config.verbose:
- print msg
-
-# ack, python's builtin int() does not support '0x123' syntax.
-# eval can do it, although what a hack!
-def my_int(s):
- import types
- if type(s) is types.IntType:
- return s
- try:
- if (s[0:2] == '0x') or (s[0:1] == '0'):
- return eval(s, {}, {})
- else:
- return int(s)
- except SyntaxError, e:
- raise ValueError("not a number")
- except TypeError, e:
- raise ValueError("not a number")
- except NameError, e:
- raise ValueError("not a number")
-
-# ============================================================
-# locally defined exceptions
-class CommandError (exceptions.Exception):
- def __init__(self, cmd_name, cmd_err, rc=None):
- self.cmd_name = cmd_name
- self.cmd_err = cmd_err
- self.rc = rc
-
- def dump(self):
- import types
- if type(self.cmd_err) == types.StringType:
- if self.rc:
- print "! %s (%d): %s" % (self.cmd_name, self.rc, self.cmd_err)
- else:
- print "! %s: %s" % (self.cmd_name, self.cmd_err)
- elif type(self.cmd_err) == types.ListType:
- if self.rc:
- print "! %s (error %d):" % (self.cmd_name, self.rc)
- else:
- print "! %s:" % (self.cmd_name)
- for s in self.cmd_err:
- print "> %s" %(string.strip(s))
- else:
- print self.cmd_err
-
-# ============================================================
-# handle lctl interface
-class LCTLInterface:
- """
- Manage communication with lctl
- """
-
- def __init__(self, cmd):
- """
- Initialize close by finding the lctl binary.
- """
- self.lctl = find_prog(cmd)
- self.save_file = ''
- self.record_device = ''
- if not self.lctl:
- if config.noexec:
- debug('! lctl not found')
- self.lctl = 'lctl'
- else:
- raise CommandError('lctl', "unable to find lctl binary.")
-
- def use_save_file(self, file):
- self.save_file = file
-
- def record(self, dev_name, logname):
- log("Recording log", logname, "on", dev_name)
- self.record_device = dev_name
- self.record_log = logname
-
- def end_record(self):
- log("End recording log", self.record_log, "on", self.record_device)
- self.record_device = None
- self.record_log = None
-
- def set_nonblock(self, fd):
- fl = fcntl.fcntl(fd, F_GETFL)
- fcntl.fcntl(fd, F_SETFL, fl | os.O_NDELAY)
-
- def run(self, cmds):
- """
- run lctl
- the cmds are written to stdin of lctl
- lctl doesn't return errors when run in script mode, so
- stderr is checked
- should modify command line to accept multiple commands, or
- create complex command line options
- """
- cmd_line = self.lctl
- if self.save_file:
- cmds = '\n dump ' + self.save_file + '\n' + cmds
- elif self.record_device:
- cmds = """
- device $%s
- record %s
- %s""" % (self.record_device, self.record_log, cmds)
-
- debug("+", cmd_line, cmds)
- if config.noexec: return (0, [])
-
- child = popen2.Popen3(cmd_line, 1) # Capture stdout and stderr from command
- child.tochild.write(cmds + "\nq\n")
- child.tochild.close()
-
- # From "Python Cookbook" from O'Reilly
- outfile = child.fromchild
- outfd = outfile.fileno()
- self.set_nonblock(outfd)
- errfile = child.childerr
- errfd = errfile.fileno()
- self.set_nonblock(errfd)
-
- outdata = errdata = ''
- outeof = erreof = 0
- while 1:
- ready = select.select([outfd,errfd],[],[]) # Wait for input
- if outfd in ready[0]:
- outchunk = outfile.read()
- if outchunk == '': outeof = 1
- outdata = outdata + outchunk
- if errfd in ready[0]:
- errchunk = errfile.read()
- if errchunk == '': erreof = 1
- errdata = errdata + errchunk
- if outeof and erreof: break
- # end of "borrowed" code
-
- ret = child.wait()
- if os.WIFEXITED(ret):
- rc = os.WEXITSTATUS(ret)
- else:
- rc = 0
- if rc or len(errdata):
- raise CommandError(self.lctl, errdata, rc)
- return rc, outdata
-
- def runcmd(self, *args):
- """
- run lctl using the command line
- """
- cmd = string.join(map(str,args))
- debug("+", self.lctl, cmd)
- rc, out = run(self.lctl, cmd)
- if rc:
- raise CommandError(self.lctl, out, rc)
- return rc, out
-
- def unconfigure_network(self):
- """get lnet to unreference itself"""
- cmds = """
- network unconfigure"""
- self.run(cmds)
-
- def clear_log(self, dev, log):
- """ clear an existing log """
- cmds = """
- device $%s
- probe
- clear_log %s
- quit """ % (dev, log)
- self.run(cmds)
-
- # create a new connection
- def add_uuid(self, net_type, uuid, nid):
- if net_type != 'lnet' and string.find(nid,'@') < 0:
- nidstr = nid + "@" + net_type
- else:
- nidstr = nid
- cmds = "\n add_uuid %s %s" %(uuid, nidstr)
- self.run(cmds)
-
- def connect(self, srv):
- if not srv.nid_uuid:
- panic('nid_uuid not set for ', srv.net_type, srv.nid)
- hostaddr = srv.db.get_hostaddr()
- if len(hostaddr) > 1:
- panic('multiple --hostaddr for ', srv.nid_uuid, ' not supported')
- elif len(hostaddr) == 1 and hostaddr[0] != srv.nid:
- panic('different --hostaddr and --nid for ', srv.nid_uuid, ' not supported')
- else:
- self.add_uuid(srv.net_type, srv.nid_uuid, srv.nid)
-
- # Recover a device
- def recover(self, dev_name, new_conn):
- cmds = """
- device $%s
- recover %s""" %(dev_name, new_conn)
- self.run(cmds)
-
- # disconnect one connection
- def disconnect(self, srv):
- if not srv.nid_uuid:
- panic('nid_uuid not set for ', srv.net_type, srv.nid)
- self.del_uuid(srv.nid_uuid)
-
- def del_uuid(self, uuid):
- cmds = """
- ignore_errors
- del_uuid %s
- quit""" % (uuid,)
- self.run(cmds)
-
- def attach(self, type, name, uuid):
- cmds = """
- attach %s %s %s
- quit""" % (type, name, uuid)
- self.run(cmds)
-
- def setup(self, name, setup = ""):
- cmds = """
- cfg_device %s
- setup %s
- quit""" % (name, setup)
- self.run(cmds)
-
- def abort_recovery(self, name):
- cmds = """
- ignore_errors
- device $%s
- abort_recovery
- quit""" % (name)
- self.run(cmds)
-
- def add_conn(self, name, conn_uuid):
- cmds = """
- cfg_device %s
- add_conn %s
- quit""" % (name, conn_uuid)
- self.run(cmds)
-
- # create a new device with lctl
- def newdev(self, type, name, uuid, setup = ""):
- self.attach(type, name, uuid);
- try:
- self.setup(name, setup)
- except CommandError, e:
- self.cleanup(name, uuid, 0)
- raise e
- if (config.abort_recovery):
- if (type == 'obdfilter' or type == 'mds'):
- self.abort_recovery(name)
-
- # cleanup a device
- def cleanup(self, name, uuid, force, failover = 0):
- if failover: force = 1
- cmds = """
- ignore_errors
- cfg_device $%s
- cleanup %s %s
- detach
- quit""" % (name, ('', 'force')[force],
- ('', 'failover')[failover])
- self.run(cmds)
-
- # create an lov
- def lov_setup(self, name, uuid, desc_uuid, mdsuuid, stripe_cnt,
- stripe_sz, stripe_off, pattern):
- cmds = """
- attach lov %s %s
- lov_setup %s %d %d %d %s
- quit""" % (name, uuid, desc_uuid, stripe_cnt, stripe_sz, stripe_off, pattern)
- self.run(cmds)
-
- # add an OBD to a LOV
- def lov_add_obd(self, name, uuid, obd_uuid, index, gen):
- cmds = """
- cfg_device %s
- lov_modify_tgts add %s %s %s %s
- quit""" % (name, name, obd_uuid, index, gen)
- self.run(cmds)
-
- # delete an OBD from a LOV
- def lov_del_obd(self, name, uuid, obd_uuid, index, gen):
- cmds = """
- cfg_device %s
- lov_modify_tgts del %s %s %s %s
- quit""" % (name, name, obd_uuid, index, gen)
- self.run(cmds)
-
- # deactivate an OBD
- def deactivate(self, name):
- cmds = """
- cfg_device %s
- deactivate
- quit""" % (name)
- self.run(cmds)
-
- # dump the log file
- def dump(self, dump_file):
- cmds = """
- debug_kernel %s 1
- quit""" % (dump_file)
- self.run(cmds)
-
- # get list of devices
- def device_list(self):
- ret = []
- if PLATFORM == 'LINUX':
- devices = '/proc/fs/lustre/devices'
- if os.access(devices, os.R_OK):
- try:
- fp = open(devices, 'r')
- ret = fp.readlines()
- fp.close()
- except IOError, e:
- log(e)
- elif PLATFORM == 'DARWIN':
- rc, out = self.run("device_list")
- ret = out.split("\n")
- if len(ret) == 0:
- return ret
- tail = ret[-1]
- if not tail:
- # remove the last empty line
- ret = ret[:-1]
- return ret
-
- # get lustre version
- def lustre_version(self):
- rc, out = self.runcmd('version')
- return out
-
- # dump mount options
- def mount_option(self, profile, osc, mdc):
- cmds = """
- mount_option %s %s %s
- quit""" % (profile, osc, mdc)
- self.run(cmds)
-
- # delete mount options
- def del_mount_option(self, profile):
- cmds = """
- del_mount_option %s
- quit""" % (profile,)
- self.run(cmds)
-
- def set_timeout(self, timeout):
- cmds = """
- set_timeout %s
- quit""" % (timeout,)
- self.run(cmds)
-
- # set lustre upcall
- def set_lustre_upcall(self, upcall):
- cmds = """
- set_lustre_upcall %s
- quit""" % (upcall,)
- self.run(cmds)
-# ============================================================
-# Various system-level functions
-# (ideally moved to their own module)
-
-# Run a command and return the output and status.
-# stderr is sent to /dev/null, could use popen3 to
-# save it if necessary
-def runcmd(cmd):
- debug ("+", cmd)
- if config.noexec: return (0, [])
- f = os.popen(cmd + ' 2>&1')
- out = f.readlines()
- ret = f.close()
- if ret:
- ret = ret >> 8
- else:
- ret = 0
- return (ret, out)
-
-def run(*args):
- cmd = string.join(map(str,args))
- return runcmd(cmd)
-
-# Run a command in the background.
-def run_daemon(*args):
- cmd = string.join(map(str,args))
- debug ("+", cmd)
- if config.noexec: return 0
- f = os.popen(cmd + ' 2>&1')
- ret = f.close()
- if ret:
- ret = ret >> 8
- else:
- ret = 0
- return ret
-
-# Determine full path to use for an external command
-# searches dirname(argv[0]) first, then PATH
-def find_prog(cmd):
- syspath = string.split(os.environ['PATH'], ':')
- cmdpath = os.path.dirname(sys.argv[0])
- syspath.insert(0, cmdpath);
- if config.portals:
- syspath.insert(0, os.path.join(config.portals, 'utils/'))
- for d in syspath:
- prog = os.path.join(d,cmd)
- if os.access(prog, os.X_OK):
- return prog
- return ''
-
-# Recursively look for file starting at base dir
-def do_find_file(base, mod):
- fullname = os.path.join(base, mod)
- if os.access(fullname, os.R_OK):
- return fullname
- for d in os.listdir(base):
- dir = os.path.join(base,d)
- if os.path.isdir(dir):
- module = do_find_file(dir, mod)
- if module:
- return module
-
-def find_module(src_dir, dev_dir, modname):
- modbase = src_dir +'/'+ dev_dir +'/'+ modname
- for modext in '.ko', '.o':
- module = modbase + modext
- try:
- if os.access(module, os.R_OK):
- return module
- except OSError:
- pass
- return None
-
-# is the path a block device?
-def is_block(path):
- s = ()
- try:
- s = os.stat(path)
- except OSError:
- return 0
- return stat.S_ISBLK(s[stat.ST_MODE])
-
-def my_realpath(path):
- try:
- if os.path.islink(path):
- # get the realpath of the mount point path
- if 'realpath' in dir(os.path):
- real_path = os.path.realpath(path)
- else:
- real_path = path
- link_count = 0
- while os.path.islink(real_path) and (link_count < 20):
- link_count = link_count + 1
- path_link = os.readlink(real_path)
- if os.path.isabs(path_link):
- real_path = path_link
- else:
- real_path = os.path.join(os.path.dirname(real_path), path_link)
- if link_count > 19:
- panic("Encountered too many symbolic links resolving path:", path)
- else:
- real_path = path
-
- return real_path
- except:
- panic("Fatal error realpath()ing path:", path)
-
-
-# build fs according to type
-# fixme: dangerous
-def mkfs(dev, devsize, fstype, jsize, isize, mkfsoptions, isblock=1):
- block_cnt = ''
- jopt = ''
- iopt = ''
- if devsize:
- if devsize < 8000:
- panic("size of filesystem on '%s' must be larger than 8MB, but is set to %s"%
- (dev, devsize))
- # devsize is in 1k, and fs block count is in 4k
- block_cnt = devsize/4
-
- if fstype in ('ext3', 'ldiskfs'):
- # ext3 journal size is in megabytes
- if jsize == 0:
- if devsize == 0:
- if not is_block(dev):
- ret, out = runcmd("ls -l %s" %dev)
- devsize = int(string.split(out[0])[4]) / 1024
- else:
- # sfdisk works for symlink, hardlink, and realdev
- ret, out = runcmd("sfdisk -s %s" %dev)
- if not ret:
- devsize = int(out[0])
- else:
- # sfdisk -s will fail for too large block device,
- # then, read the size of partition from /proc/partitions
-
- # get the realpath of the device
- # it may be the real device, such as /dev/hda7
- # or the hardlink created via mknod for a device
- real_dev = my_realpath(dev)
-
- # get the major and minor number of the realpath via ls
- # it seems python(os.stat) does not return
- # the st_rdev member of the stat structure
- ret, out = runcmd("ls -l %s" %real_dev)
- major = string.split(string.split(out[0])[4], ",")[0]
- minor = string.split(out[0])[5]
-
- # get the devsize from /proc/partitions with the major and minor number
- ret, out = runcmd("cat /proc/partitions")
- for line in out:
- if len(line) > 1:
- if string.split(line)[0] == major and string.split(line)[1] == minor:
- devsize = int(string.split(line)[2])
- break
-
- if devsize > 1024 * 1024:
- jsize = ((devsize / 102400) * 4)
- if jsize > 400:
- jsize = 400
- if jsize: jopt = "-J size=%d" %(jsize,)
- if isize: iopt = "-I %d" %(isize,)
- mkfs = 'mkfs.ext2 -j -b 4096 '
- if not isblock or config.force:
- mkfs = mkfs + ' -F '
- elif fstype == 'reiserfs':
- # reiserfs journal size is in blocks
- if jsize: jopt = "--journal_size %d" %(jsize,)
- mkfs = 'mkreiserfs -ff'
- else:
- panic('unsupported fs type: ', fstype)
-
- if config.mkfsoptions != None:
- mkfs = mkfs + ' ' + config.mkfsoptions
- if mkfsoptions != None:
- mkfs = mkfs + ' ' + mkfsoptions
- (ret, out) = run (mkfs, jopt, iopt, dev, block_cnt)
- if ret:
- panic("Unable to build fs:", dev, string.join(out))
- # enable hash tree indexing on fsswe
- if fstype in ('ext3', 'ldiskfs'):
- htree = 'tune2fs -O dir_index'
- (ret, out) = run (htree, dev)
- if ret:
- panic("Unable to enable htree:", dev)
-
-# some systems use /dev/loopN, some /dev/loop/N
-def loop_base():
- import re
- loop = '/dev/loop'
- if not os.access(loop + str(0), os.R_OK):
- loop = loop + '/'
- if not os.access(loop + str(0), os.R_OK):
- loop='/dev/loop'
- return loop
-
-# find loop device assigned to the file
-def find_loop(file):
- loop = loop_base()
- for n in xrange(0, MAX_LOOP_DEVICES):
- dev = loop + str(n)
- if os.access(dev, os.R_OK):
- (stat, out) = run('losetup', dev)
- if out and stat == 0:
- m = re.search(r'\((.*)\)', out[0])
- if m and file == m.group(1):
- return dev
- else:
- break
- return ''
-
-# create file if necessary and assign the first free loop device
-def init_loop(file, size, fstype, journal_size, inode_size, mkfsoptions, reformat):
- dev = find_loop(file)
- if dev:
- print 'WARNING file:', file, 'already mapped to', dev
- return dev
- if reformat or not os.access(file, os.R_OK | os.W_OK):
- if size < 8000:
- panic("size of loopback file '%s' must be larger than 8MB, but is set to %s" % (file,size))
- (ret, out) = run("dd if=/dev/zero bs=1k count=0 seek=%d of=%s" %(size,
- file))
- if ret:
- panic("Unable to create backing store:", file)
- mkfs(file, size, fstype, journal_size, inode_size, mkfsoptions, isblock=0)
-
- loop = loop_base()
- # find next free loop
- for n in xrange(0, MAX_LOOP_DEVICES):
- dev = loop + str(n)
- if os.access(dev, os.R_OK):
- (stat, out) = run('losetup', dev)
- if stat:
- (stat, out) = run('losetup', dev, file)
- if stat:
- panic("losetup failed: (%s) %s" % (stat, out[0].strip()))
- return dev
- else:
- print "out of loop devices"
- return ''
- print "out of loop devices"
- return ''
-
-# undo loop assignment
-def clean_loop(file):
- dev = find_loop(file)
- if dev:
- ret, out = run('losetup -d', dev)
- if ret:
- log('unable to clean loop device:', dev, 'for file:', file)
- logall(out)
-
-# determine if dev is formatted as a <fstype> filesystem
-def need_format(fstype, dev):
- # FIXME don't know how to implement this
- return 0
-
-# initialize a block device if needed
-def block_dev(dev, size, fstype, reformat, autoformat, journal_size,
- inode_size, mkfsoptions):
- if config.noexec: return dev
- if not is_block(dev):
- dev = init_loop(dev, size, fstype, journal_size, inode_size,
- mkfsoptions, reformat)
- elif reformat or (need_format(fstype, dev) and autoformat == 'yes'):
- mkfs(dev, size, fstype, journal_size, inode_size, mkfsoptions,
- isblock=0)
-# else:
-# panic("device:", dev,
-# "not prepared, and autoformat is not set.\n",
-# "Rerun with --reformat option to format ALL filesystems")
- return dev
-
-def if2addr(iface):
- """lookup IP address for an interface"""
- rc, out = run("/sbin/ifconfig", iface)
- if rc or not out:
- return None
- addr = string.split(out[1])[1]
- ip = string.split(addr, ':')[1]
- return ip
-
-def def_mount_options(fstype, target, blkdev):
- """returns deafult mount options for passed fstype and target (mds, ost)"""
- if fstype == 'ext3' or fstype == 'ldiskfs':
- mountfsoptions = "errors=remount-ro"
- if target == 'ost':
- if sys_get_branch() == '2.4':
- mountfsoptions = "%s,asyncdel" % (mountfsoptions)
- #else:
- # mountfsoptions = "%s,extents,mballoc" % (mountfsoptions)
- elif target == 'mds':
- if config.user_xattr:
- mountfsoptions = "%s,user_xattr" % (mountfsoptions)
- if config.acl:
- mountfsoptions = "%s,acl" % (mountfsoptions)
-
- if blkdev:
- # grab superblock info
- dumpe2fs="dumpe2fs -f -h"
- (ret, sb) = run(dumpe2fs, blkdev)
- if ret:
- panic("unable to get superblock for ", blkdev)
-
- # extract journal UUID
- journal_UUID=''
- journal_DEV=''
- for line in sb:
- lst = string.split(line, ":")
- if lst[0] == 'Journal UUID':
- if len(lst[1]) < 3:
- panic("cannot retrieve journal UUID for ", blkdev)
- if string.split(lst[1])[0] != '<none>':
- journal_UUID = string.split(lst[1])[0]
- debug(blkdev, 'has journal UUID', journal_UUID)
- if lst[0] == 'Journal device':
- if len(lst[1]) < 3:
- panic("cannot retrieve journal device for ", blkdev)
- if string.split(lst[1])[0] != '0x0000':
- journal_DEV = string.split(lst[1])[0]
- debug(blkdev, 'has journal device', journal_DEV)
- break
-
- if len(journal_UUID) == 0 or len(journal_DEV) == 0:
- debug('no external journal found for', blkdev)
- # use internal journal
- return mountfsoptions
-
- # run blkid, lookup highest-priority device with matching UUID
- blkid = "blkid -o device -l -t UUID='%s'" % (journal_UUID)
- (ret, devname) = run(blkid)
- if ret or len(devname) == 0:
- panic("cannot find external journal for ", blkdev)
- debug('found', blkdev, 'journal UUID', journal_UUID, 'on',
- string.replace(devname[0], '\n', ''))
-
- try: # sigh, python 1.5 does not support os.stat().st_rdev
- jdevpath = my_realpath(string.replace(devname[0], '\n', ''))
- ret, out = runcmd("ls -l %s" %jdevpath)
- debug('ls -l:', out)
- major = int(string.split(string.split(out[0])[4], ',')[0])
- minor = int(string.split(out[0])[5])
- debug('major', major, 'minor', minor)
- rdev = major << 8 | minor
- except OSError:
- panic("cannot stat ", devname[0])
-
- debug('found', blkdev, 'journal UUID', journal_UUID, 'on',
- jdevpath, 'rdev', rdev)
-
- # add mount option
- if string.atoi(journal_DEV, 0) != rdev:
- mountfsoptions = "%s,journal_dev=%#x" % (mountfsoptions,rdev)
-
- return mountfsoptions
- return ""
-
-def sys_get_branch():
- """Returns kernel release"""
- return os.uname()[2][:3]
-
-def mod_loaded(modname):
- """Check if a module is already loaded. Look in /proc/modules for it."""
- if PLATFORM == 'LINUX':
- try:
- fp = open('/proc/modules')
- lines = fp.readlines()
- fp.close()
- # please forgive my tired fingers for this one
- ret = filter(lambda word, mod=modname: word == mod,
- map(lambda line: string.split(line)[0], lines))
- return ret
- except Exception, e:
- return 0
- elif PLATFORM == 'DARWIN':
- ret, out = run('/usr/sbin/kextstat | /usr/bin/grep', modname)
- if ret == 0:
- return 1
- else:
- return 0
- else:
- return 0
-
-# XXX: instead of device_list, ask for $name and see what we get
-def is_prepared(name):
- """Return true if a device exists for the name"""
- if config.lctl_dump:
- return 0
- if (config.noexec or config.record) and config.cleanup:
- return 1
- try:
- # expect this format:
- # 1 UP ldlm ldlm ldlm_UUID 2
- out = lctl.device_list()
- for s in out:
- if name == string.split(s)[3]:
- return 1
- except CommandError, e:
- e.dump()
- return 0
-
-def is_network_prepared():
- """If the any device exists, then assume that all networking
- has been configured"""
- out = lctl.device_list()
- return len(out) > 0
-
-def fs_is_mounted(path):
- """Return true if path is a mounted lustre filesystem"""
- try:
- real_path = my_realpath(path)
-
- fp = open('/proc/mounts')
- lines = fp.readlines()
- fp.close()
- for l in lines:
- a = string.split(l)
- if a[1] == real_path and a[2] == 'lustre_lite':
- return 1
- except IOError, e:
- log(e)
- return 0
-
-class kmod:
- """Manage kernel modules"""
- def __init__(self, lustre_dir, portals_dir):
- self.lustre_dir = lustre_dir
- self.portals_dir = portals_dir
- self.kmodule_list = []
-
- def add_portals_module(self, dev_dir, modname):
- """Append a module to list of modules to load."""
- self.kmodule_list.append((self.portals_dir, dev_dir, modname))
-
- def add_lustre_module(self, dev_dir, modname):
- """Append a module to list of modules to load."""
- self.kmodule_list.append((self.lustre_dir, dev_dir, modname))
-
- def load_module(self):
- """Load all the modules in the list in the order they appear."""
- for src_dir, dev_dir, mod in self.kmodule_list:
- if mod_loaded(mod) and not config.noexec:
- continue
- log ('loading module:', mod, 'srcdir', src_dir, 'devdir', dev_dir)
- if PLATFORM == 'LINUX':
- options = ''
- if mod == 'lnet':
- #For LNET we really need modprobe to load defined LNDs
- run('/sbin/modprobe lnet')
- #But if that fails, try insmod anyhow with dev option
- #accept=all for dev liblustre testing
- options = 'accept=all'
- if src_dir:
- module = find_module(src_dir, dev_dir, mod)
- if not module:
- panic('module not found:', mod)
- (rc, out) = run('/sbin/insmod', module, options)
- if rc and not mod_loaded(mod):
- if rc == 1:
- print("Bad module options? Check dmesg.")
- raise CommandError('insmod', out, rc)
- else:
- (rc, out) = run('/sbin/modprobe', mod)
- if rc and not mod_loaded(mod):
- if rc == 1:
- print("Bad module options? Check dmesg.")
- raise CommandError('modprobe', out, rc)
- elif PLATFORM == 'DARWIN':
- run('/sbin/kextload', KEXTPATH + mod + '.kext');
-
- def cleanup_module(self):
- """Unload the modules in the list in reverse order."""
-
- rev = self.kmodule_list[:] # make *copy* of list
- rev.reverse()
- for src_dir, dev_dir, mod in rev:
- if not mod_loaded(mod) and not config.noexec:
- continue
- if mod == 'ksocklnd' and not config.noexec:
- # Ignore ksocklnd in module list (lnet will remove)
- continue
- log('unloading module:', mod)
- if mod == 'lnet' and not config.noexec:
- # remove any self-ref portals created
- lctl.unconfigure_network()
- if config.dump:
- debug('dumping debug log to', config.dump)
- # debug hack
- lctl.dump(config.dump)
- log('unloading the network')
- lctl.unconfigure_network()
- if mod_loaded("ksocklnd"):
- if PLATFORM == 'LINUX':
- run('/sbin/rmmod ksocklnd')
- elif PLATFORM == 'DARWIN':
- run('/sbin/kextunload', KEXTPATH+'ksocklnd.kext')
- if mod_loaded("kqswlnd"):
- run('/sbin/rmmod kqswlnd')
- if mod_loaded("kgmlnd"):
- run('/sbin/rmmod kgmlnd')
- if mod_loaded("kopeniblnd"):
- run('/sbin/rmmod kopeniblnd')
- if mod_loaded("kiiblnd"):
- run('/sbin/rmmod kiiblnd')
- if mod_loaded("kviblnd"):
- run('/sbin/rmmod kviblnd')
- if mod_loaded("kciblnd"):
- run('/sbin/rmmod kciblnd')
- if mod_loaded("ko2iblnd"):
- run('/sbin/rmmod ko2iblnd')
- if mod_loaded("kralnd"):
- run('/sbin/rmmod kralnd')
- if mod_loaded("kptllnd"):
- run('/sbin/rmmod kptllnd')
- if PLATFORM == 'LINUX':
- (rc, out) = run('/sbin/rmmod', mod)
- elif PLATFORM == 'DARWIN':
- (rc, out) = run('/sbin/kextunload', KEXTPATH+mod+'.kext');
- if rc:
- log('! unable to unload module:', mod)
- logall(out)
-
-
-# ============================================================
-# Classes to prepare and cleanup the various objects
-#
-class Module:
- """ Base class for the rest of the modules. The default cleanup method is
- defined here, as well as some utilitiy funcs.
- """
- def __init__(self, module_name, db):
- self.db = db
- self.module_name = module_name
- self.name = self.db.getName()
- self.uuid = self.db.getUUID()
- self._server = None
- self._connected = 0
- self.kmod = kmod(config.lustre, config.portals)
-
- def info(self, *args):
- msg = string.join(map(str,args))
- log (self.module_name + ":", self.name, self.uuid, msg)
-
- def cleanup(self):
- """ default cleanup, used for most modules """
- self.info()
- try:
- lctl.cleanup(self.name, self.uuid, config.force)
- except CommandError, e:
- log(self.module_name, "cleanup failed: ", self.name)
- e.dump()
- cleanup_error(e.rc)
-
- def add_portals_module(self, dev_dir, modname):
- """Append a module to list of modules to load."""
- self.kmod.add_portals_module(dev_dir, modname)
-
- def add_lustre_module(self, dev_dir, modname):
- """Append a module to list of modules to load."""
- self.kmod.add_lustre_module(dev_dir, modname)
-
- def load_module(self):
- """Load all the modules in the list in the order they appear."""
- self.kmod.load_module()
-
- def cleanup_module(self):
- """Unload the modules in the list in reverse order."""
- if self.safe_to_clean():
- self.kmod.cleanup_module()
-
- def safe_to_clean(self):
- return 1
-
- def safe_to_clean_modules(self):
- return self.safe_to_clean()
-
-class Network(Module):
- def __init__(self,db,nid_uuid=0):
- Module.__init__(self, 'NETWORK', db)
- self.net_type = self.db.get_val('nettype')
- self.nid = self.db.get_val('nid', '*')
- self.cluster_id = self.db.get_val('clusterid', "0")
- self.port = self.db.get_val_int('port', 0)
- self.nid_uuid = nid_uuid
- self.add_portals_module('libcfs', 'libcfs')
- self.add_portals_module('lnet', 'lnet')
- # Add the socklnd for developers without modprobe.conf (umls)
- self.add_portals_module('klnds/socklnd', 'ksocklnd')
-
- def prepare(self):
- if is_network_prepared():
- return
- self.info(self.net_type, self.nid)
- if self.net_type == 'tcp':
- sys_tweak_socknal()
- if self.net_type == 'elan':
- sys_optimize_elan()
-
- def safe_to_clean(self):
- if PLATFORM == 'LINUX':
- return not is_network_prepared()
- elif PLATFORM == 'DARWIN':
- # XXX always assume it's safe to clean
- return 1
- return 1
-
- def cleanup(self):
- self.info(self.net_type, self.nid)
-
-# This is only needed to load the modules; the LDLM device
-# is now created automatically.
-class LDLM(Module):
- def __init__(self,db):
- Module.__init__(self, 'LDLM', db)
- self.add_lustre_module('lvfs', 'lvfs')
- self.add_lustre_module('obdclass', 'obdclass')
- self.add_lustre_module('ptlrpc', 'ptlrpc')
- self.add_lustre_module('ptlrpc/gss', 'ptlrpc_gss')
-
- def prepare(self):
- return
-
- def cleanup(self):
- return
-
-class LOV(Module):
- def __init__(self, db, uuid, fs_name, name_override = None, config_only = None):
- Module.__init__(self, 'LOV', db)
- if name_override != None:
- self.name = "lov_%s" % name_override
- self.add_lustre_module('lov', 'lov')
- self.mds_uuid = self.db.get_first_ref('mds')
- self.stripe_sz = self.db.get_val_int('stripesize', 1048576)
- self.stripe_off = self.db.get_val_int('stripeoffset', 0)
- self.pattern = self.db.get_val_int('stripepattern', 0)
- self.devlist = []
- self.stripe_cnt = self.db.get_val_int('stripecount', 1)
- self.osclist = []
- self.desc_uuid = self.uuid
- self.uuid = generate_client_uuid(self.name)
- self.fs_name = fs_name
- # settings below here won't be seen by the MDSDEV code!
- if config_only:
- self.config_only = 1
- return
- self.config_only = None
- mds = self.db.lookup(self.mds_uuid)
- self.mds_name = mds.getName()
- self.devlist = self.db.get_lov_tgts('lov_tgt')
- for (obd_uuid, index, gen, active) in self.devlist:
- if obd_uuid == '':
- continue
- obd = self.db.lookup(obd_uuid)
- osc = get_osc(obd, self.uuid, fs_name)
- if osc:
- self.osclist.append((osc, index, gen, active))
- else:
- panic('osc not found:', obd_uuid)
- if self.osclist == []:
- debug("get_lov_tgts failed, using get_refs");
- index = 0
- self.devlist = self.db.get_refs('obd')
- for obd_uuid in self.devlist:
- obd = self.db.lookup(obd_uuid)
- osc = get_osc(obd, self.uuid, fs_name)
- if osc:
- self.osclist.append((osc, index, 1, 1))
- else:
- panic('osc not found:', obd_uuid)
- index = index + 1
- if self.osclist == []:
- panic('No OSCs configured for LOV')
- debug('dbg LOV __init__:', self.osclist, self.devlist, self.stripe_cnt)
-
- def prepare(self):
- debug('dbg LOV prepare')
- if is_prepared(self.name):
- return
- debug('dbg LOV prepare:', self.osclist, self.devlist)
- self.info(self.mds_uuid, self.stripe_cnt, self.stripe_sz,
- self.stripe_off, self.pattern, self.devlist,
- self.mds_name)
- lctl.lov_setup(self.name, self.uuid,
- self.desc_uuid, self.mds_name, self.stripe_cnt,
- self.stripe_sz, self.stripe_off, self.pattern)
- if self.osclist == []:
- panic('No OSCs configured for LOV?')
- for (osc, index, gen, active) in self.osclist:
- target_uuid = osc.target_uuid
- try:
- # Only ignore connect failures with --force, which
- # isn't implemented here yet.
- osc.active = active
- osc.prepare(ignore_connect_failure=0)
- except CommandError, e:
- print "Error preparing OSC %s\n" % osc.uuid
- raise e
- lctl.lov_add_obd(self.name, self.uuid, target_uuid, index, gen)
-
- def cleanup(self):
- if is_prepared(self.name):
- Module.cleanup(self)
- for (osc, index, gen, active) in self.osclist:
- osc.cleanup()
- if self.config_only:
- panic("Can't clean up config_only LOV ", self.name)
-
- def load_module(self):
- if self.config_only:
- panic("Can't load modules for config_only LOV ", self.name)
- for (osc, index, gen, active) in self.osclist:
- osc.load_module()
- break
- Module.load_module(self)
-
- def cleanup_module(self):
- if self.config_only:
- panic("Can't cleanup modules for config_only LOV ", self.name)
- Module.cleanup_module(self)
- for (osc, index, gen, active) in self.osclist:
- if active:
- osc.cleanup_module()
- break
-
-class MDSDEV(Module):
- def __init__(self,db):
- Module.__init__(self, 'MDSDEV', db)
- self.devpath = self.db.get_val('devpath','')
- self.size = self.db.get_val_int('devsize', 0)
- self.journal_size = self.db.get_val_int('journalsize', 0)
-
- self.fstype = self.db.get_val('fstype', '')
- if sys_get_branch() == '2.4' and self.fstype == 'ldiskfs':
- self.fstype = 'ext3'
- elif sys_get_branch() == '2.6' and self.fstype == 'ext3':
- self.fstype = 'ldiskfs'
-
- self.nspath = self.db.get_val('nspath', '')
- self.mkfsoptions = '-i 4096 ' + self.db.get_val('mkfsoptions', '')
- self.mountfsoptions = self.db.get_val('mountfsoptions', '')
- if config.quota:
- self.quota = config.quota
- else:
- self.quota = self.db.get_val('quota', '')
- # overwrite the orignal MDSDEV name and uuid with the MDS name and uuid
- target_uuid = self.db.get_first_ref('target')
- mds = self.db.lookup(target_uuid)
- self.name = mds.getName()
- self.filesystem_uuids = mds.get_refs('filesystem')
- # FIXME: if fstype not set, then determine based on kernel version
- self.format = self.db.get_val('autoformat', "no")
- if mds.get_val('failover', '1') != '0':
- self.failover_mds = 'f'
- else:
- self.failover_mds = 'n'
- active_uuid = get_active_target(mds)
- if not active_uuid:
- panic("No target device found:", target_uuid)
- if active_uuid == self.uuid:
- self.active = 1
- else:
- self.active = 0
- if self.active and config.group and config.group != mds.get_val('group', mds.get_val('name')):
- self.active = 0
-
- self.inode_size = self.db.get_val_int('inodesize', 0)
- debug('original inode_size ', self.inode_size)
- if self.inode_size == 0:
- # find the LOV for this MDS
- lovconfig_uuid = mds.get_first_ref('lovconfig')
- if not lovconfig_uuid:
- panic("No LOV config found for MDS ", mds.name)
- lovconfig = mds.lookup(lovconfig_uuid)
- lov_uuid = lovconfig.get_first_ref('lov')
- if not lov_uuid:
- panic("No LOV found for lovconfig ", lovconfig.name)
- lov = LOV(self.db.lookup(lov_uuid), lov_uuid, 'FS_name', config_only = 1)
-
- # default stripe count controls default inode_size
- if (lov.stripe_cnt > 0):
- stripe_count = lov.stripe_cnt
- else:
- stripe_count = 1
- if stripe_count > 77:
- self.inode_size = 512
- elif stripe_count > 34:
- self.inode_size = 2048
- elif stripe_count > 13:
- self.inode_size = 1024
- #elif stripe_count < 3:
- # self.inode_size = 256
- else:
- self.inode_size = 512
- debug('stripe_count ', stripe_count,' inode_size ',self.inode_size)
-
- self.target_dev_uuid = self.uuid
- self.uuid = target_uuid
-
- # loading modules
- if self.quota:
- self.add_lustre_module('quota', 'lquota')
- self.add_lustre_module('mdc', 'mdc')
- self.add_lustre_module('osc', 'osc')
- self.add_lustre_module('lov', 'lov')
- self.add_lustre_module('mds', 'mds')
- if self.fstype == 'ldiskfs':
- self.add_lustre_module('ldiskfs', 'ldiskfs')
- if self.fstype:
- self.add_lustre_module('lvfs', 'fsfilt_%s' % (self.fstype))
-
- def load_module(self):
- if self.active:
- Module.load_module(self)
-
- def prepare(self):
- if is_prepared(self.name):
- return
- if not self.active:
- debug(self.uuid, "not active")
- return
- if config.reformat:
- # run write_conf automatically, if --reformat used
- self.write_conf()
- self.info(self.devpath, self.fstype, self.size, self.format)
- # never reformat here
- blkdev = block_dev(self.devpath, self.size, self.fstype, 0,
- self.format, self.journal_size, self.inode_size,
- self.mkfsoptions)
- if not is_prepared('MDT'):
- lctl.newdev("mdt", 'MDT', 'MDT_UUID', setup ="")
- try:
- mountfsoptions = def_mount_options(self.fstype, 'mds', blkdev)
-
- if config.mountfsoptions:
- if mountfsoptions:
- mountfsoptions = mountfsoptions + ',' + config.mountfsoptions
- else:
- mountfsoptions = config.mountfsoptions
- if self.mountfsoptions:
- mountfsoptions = mountfsoptions + ',' + self.mountfsoptions
- else:
- if self.mountfsoptions:
- if mountfsoptions:
- mountfsoptions = mountfsoptions + ',' + self.mountfsoptions
- else:
- mountfsoptions = self.mountfsoptions
-
- print 'MDS mount options: ' + mountfsoptions
-
- lctl.newdev("mds", self.name, self.uuid,
- setup ="%s %s %s %s %s" %(blkdev, self.fstype, self.name,
- mountfsoptions, self.quota))
- self.group_upcall = self.db.get_val('group_upcall','')
- sys_set_group_upcall(self.name, self.group_upcall)
-
- except CommandError, e:
- if e.rc == 2:
- panic("MDS failed to start. Check the syslog for details." +
- " (May need to run lconf --write-conf)")
- else:
- raise e
-
- def write_conf(self):
- if is_prepared(self.name):
- return
- self.info(self.devpath, self.fstype, self.format)
- blkdev = block_dev(self.devpath, self.size, self.fstype,
- config.reformat, self.format, self.journal_size,
- self.inode_size, self.mkfsoptions)
- lctl.newdev("mds", self.name, self.uuid,
- setup ="%s %s" %(blkdev, self.fstype))
-
- # record logs for the MDS lov
- for uuid in self.filesystem_uuids:
- log("recording clients for filesystem:", uuid)
- fs = self.db.lookup(uuid)
- obd_uuid = fs.get_first_ref('obd')
- client_uuid = generate_client_uuid(self.name)
- client = VOSC(self.db.lookup(obd_uuid), client_uuid, self.name,
- self.name)
- config.record = 1
- lctl.clear_log(self.name, self.name)
- lctl.record(self.name, self.name)
- client.prepare()
- lctl.mount_option(self.name, client.get_name(), "")
- lctl.end_record()
- config.record = 0
-
- # record logs for each client
- if config.ldapurl:
- config_options = "--ldapurl " + config.ldapurl + " --config " + config.config
- else:
- config_options = CONFIG_FILE
-
- for node_db in self.db.lookup_class('node'):
- client_name = node_db.getName()
- for prof_uuid in node_db.get_refs('profile'):
- prof_db = node_db.lookup(prof_uuid)
- # refactor this into a funtion to test "clientness" of a node.
- for ref_class, ref_uuid in prof_db.get_all_refs():
- if ref_class in ('mountpoint','echoclient'):
- thing = self.db.lookup(ref_uuid);
- fs_uuid = thing.get_first_ref('filesystem')
- if not fs_uuid in self.filesystem_uuids:
- continue;
-
- log("Recording log", client_name, "on", self.name)
- old_noexec = config.noexec
- config.noexec = 0
- noexec_opt = ('', '-n')
- ret, out = run (sys.argv[0],
- noexec_opt[old_noexec == 1],
- " -v --record --nomod --old_conf",
- "--record_log", client_name,
- "--record_device", self.name,
- "--node", client_name,
- config_options)
- if ret:
- lctl.clear_log(self.name, client_name)
- print out
- self.cleanup()
- panic("Record client log %s on %s failed" %(
- client_name, self.name))
- if config.verbose:
- for s in out: log("record> ", string.strip(s))
- config.noexec = old_noexec
- try:
- lctl.cleanup(self.name, self.uuid, config.force, config.failover)
- except CommandError, e:
- log(self.module_name, "cleanup failed: ", self.name)
- e.dump()
- cleanup_error(e.rc)
- Module.cleanup(self)
- clean_loop(self.devpath)
-
- #change the mtime of LLOG to match the XML creation time
- if toplustreDB.get_mtime():
- mtime = toplustreDB.get_mtime()
- debug("changing mtime of LOGS to %s" %mtime)
- ret, mktemp = runcmd("mktemp /tmp/lustre-cmd.XXXXXXXX")
- if ret:
- log(self.module_name, "create mtime LOGS cmdfile failed: ", self.name)
- else:
- mtimecmdfile = string.split(mktemp[0])[0]
- fd = os.open(mtimecmdfile, os.O_RDWR | os.O_CREAT)
- os.write(fd, "\n\n\n\n\n%s\n\n" %mtime)
- os.close(fd)
- cmd = "debugfs -w -R \"mi /LOGS\" <%s %s" %(mtimecmdfile, self.devpath)
- ret, outs = runcmd(cmd)
- os.remove(mtimecmdfile)
- if ret:
- print "Can not change mtime of LOGS by debugfs."
-
- def mds_remaining(self):
- out = lctl.device_list()
- for s in out:
- if string.split(s)[2] in ('mds',):
- if string.split(s)[1] in ('ST',):
- return 0
- return 1
-
- def safe_to_clean(self):
- return self.active
-
- def safe_to_clean_modules(self):
- return not self.mds_remaining()
-
- def cleanup(self):
- if not self.active:
- debug(self.uuid, "not active")
- return
- self.info()
- if is_prepared(self.name):
- try:
- lctl.cleanup(self.name, self.uuid, config.force,
- config.failover)
- except CommandError, e:
- log(self.module_name, "cleanup failed: ", self.name)
- e.dump()
- cleanup_error(e.rc)
- Module.cleanup(self)
- if not self.mds_remaining() and is_prepared('MDT'):
- try:
- lctl.cleanup("MDT", "MDT_UUID", config.force,
- config.failover)
- except CommandError, e:
- print "cleanup failed: ", self.name
- e.dump()
- cleanup_error(e.rc)
- clean_loop(self.devpath)
-
-class OSD(Module):
- def __init__(self, db):
- Module.__init__(self, 'OSD', db)
- self.osdtype = self.db.get_val('osdtype')
- self.devpath = self.db.get_val('devpath', '')
- self.size = self.db.get_val_int('devsize', 0)
- self.journal_size = self.db.get_val_int('journalsize', 0)
-
- # now as we store fids in EA on OST we need to make inode bigger
- self.inode_size = self.db.get_val_int('inodesize', 0)
- if self.inode_size == 0:
- self.inode_size = 256
- self.mkfsoptions = self.db.get_val('mkfsoptions', '')
- # Allocate fewer inodes on large OST devices. Most filesystems
- # can be much more aggressive than this, but by default we can't.
- if self.size > 1000000:
- self.mkfsoptions = '-i 16384 ' + self.mkfsoptions
- self.mountfsoptions = self.db.get_val('mountfsoptions', '')
- if config.quota:
- self.quota = config.quota
- else:
- self.quota = self.db.get_val('quota', '')
-
- self.fstype = self.db.get_val('fstype', '')
- if sys_get_branch() == '2.4' and self.fstype == 'ldiskfs':
- self.fstype = 'ext3'
- elif sys_get_branch() == '2.6' and self.fstype == 'ext3':
- self.fstype = 'ldiskfs'
-
- self.nspath = self.db.get_val('nspath', '')
- target_uuid = self.db.get_first_ref('target')
- ost = self.db.lookup(target_uuid)
- self.name = ost.getName()
- self.format = self.db.get_val('autoformat', 'yes')
- if ost.get_val('failover', '1') != '0':
- self.failover_ost = 'f'
- else:
- self.failover_ost = 'n'
-
- active_uuid = get_active_target(ost)
- if not active_uuid:
- panic("No target device found:", target_uuid)
- if active_uuid == self.uuid:
- self.active = 1
- else:
- self.active = 0
- if self.active and config.group and config.group != ost.get_val('group', ost.get_val('name')):
- self.active = 0
-
- self.target_dev_uuid = self.uuid
- self.uuid = target_uuid
- # modules
- if self.quota:
- self.add_lustre_module('quota', 'lquota')
- self.add_lustre_module('ost', 'ost')
- # FIXME: should we default to ext3 here?
- if self.fstype == 'ldiskfs':
- self.add_lustre_module('ldiskfs', 'ldiskfs')
- if self.fstype:
- self.add_lustre_module('lvfs' , 'fsfilt_%s' % (self.fstype))
- self.add_lustre_module(self.osdtype, self.osdtype)
-
- def load_module(self):
- if self.active:
- Module.load_module(self)
-
- # need to check /proc/mounts and /etc/mtab before
- # formatting anything.
- # FIXME: check if device is already formatted.
- def prepare(self):
- if is_prepared(self.name):
- return
- if not self.active:
- debug(self.uuid, "not active")
- return
- self.info(self.osdtype, self.devpath, self.size, self.fstype,
- self.format, self.journal_size, self.inode_size)
- if self.osdtype == 'obdecho':
- blkdev = ''
- else:
- blkdev = block_dev(self.devpath, self.size, self.fstype,
- config.reformat, self.format, self.journal_size,
- self.inode_size, self.mkfsoptions)
-
- mountfsoptions = def_mount_options(self.fstype, 'ost', blkdev)
-
- if config.mountfsoptions:
- if mountfsoptions:
- mountfsoptions = mountfsoptions + ',' + config.mountfsoptions
- else:
- mountfsoptions = config.mountfsoptions
- if self.mountfsoptions:
- mountfsoptions = mountfsoptions + ',' + self.mountfsoptions
- else:
- if self.mountfsoptions:
- if mountfsoptions:
- mountfsoptions = mountfsoptions + ',' + self.mountfsoptions
- else:
- mountfsoptions = self.mountfsoptions
-
- print 'OST mount options: ' + mountfsoptions
-
- lctl.newdev(self.osdtype, self.name, self.uuid,
- setup ="%s %s %s %s %s" %(blkdev, self.fstype,
- self.failover_ost, mountfsoptions,
- self.quota))
- if not is_prepared('OSS'):
- lctl.newdev("ost", 'OSS', 'OSS_UUID', setup ="")
-
- def osd_remaining(self):
- out = lctl.device_list()
- for s in out:
- if string.split(s)[2] in ('obdfilter', 'obdecho'):
- return 1
-
- def safe_to_clean(self):
- return self.active
-
- def safe_to_clean_modules(self):
- return not self.osd_remaining()
-
- def cleanup(self):
- if not self.active:
- debug(self.uuid, "not active")
- return
- if is_prepared(self.name):
- self.info()
- try:
- lctl.cleanup(self.name, self.uuid, config.force,
- config.failover)
- except CommandError, e:
- log(self.module_name, "cleanup failed: ", self.name)
- e.dump()
- cleanup_error(e.rc)
- if not self.osd_remaining() and is_prepared('OSS'):
- try:
- lctl.cleanup("OSS", "OSS_UUID", config.force,
- config.failover)
- except CommandError, e:
- print "cleanup failed: ", self.name
- e.dump()
- cleanup_error(e.rc)
- if not self.osdtype == 'obdecho':
- clean_loop(self.devpath)
-
-# Generic client module, used by OSC and MDC
-class Client(Module):
- def __init__(self, tgtdb, uuid, module, fs_name, self_name=None,
- module_dir=None):
- self.target_name = tgtdb.getName()
- self.target_uuid = tgtdb.getUUID()
- self.db = tgtdb
- self.backup_targets = []
-
- self.tgt_dev_uuid = get_active_target(tgtdb)
- if not self.tgt_dev_uuid:
- panic("No target device found for target:", self.target_name)
-
- self.kmod = kmod(config.lustre, config.portals)
- self._server = None
- self._connected = 0
-
- self.module = module
- self.module_name = string.upper(module)
- if not self_name:
- self.name = '%s_%s_%s_%s' % (self.module_name, socket.gethostname(),
- self.target_name, fs_name)
- else:
- self.name = self_name
- self.uuid = uuid
- self.lookup_server(self.tgt_dev_uuid)
- self.lookup_backup_targets()
- self.fs_name = fs_name
- if not module_dir:
- module_dir = module
- self.add_lustre_module(module_dir, module)
-
- def lookup_server(self, srv_uuid):
- """ Lookup a server's network information """
- self._server_nets = get_ost_net(self.db, srv_uuid)
- if len(self._server_nets) == 0:
- panic("Unable to find a server for:", srv_uuid)
-
- def get_servers(self):
- return self._server_nets
-
- def lookup_backup_targets(self):
- """ Lookup alternative network information """
- prof_list = toplustreDB.get_refs('profile')
- for prof_uuid in prof_list:
- prof_db = toplustreDB.lookup(prof_uuid)
- if not prof_db:
- panic("profile:", prof_uuid, "not found.")
- for ref_class, ref_uuid in prof_db.get_all_refs():
- if ref_class in ('osd', 'mdsdev'):
- devdb = toplustreDB.lookup(ref_uuid)
- uuid = devdb.get_first_ref('target')
- if self.target_uuid == uuid and self.tgt_dev_uuid != ref_uuid:
- debug("add backup target", ref_uuid)
- self.backup_targets.append(ref_uuid)
-
- def prepare(self, ignore_connect_failure = 0):
- self.info(self.target_uuid)
- if is_prepared(self.name):
- self.cleanup()
- try:
- srv_list = self.get_servers()
- debug('dbg CLIENT __prepare__:', self.target_uuid, srv_list)
- for srv in srv_list:
- lctl.connect(srv)
- if len(srv_list) == 0:
- panic("no servers for ", self.target_uuid)
- except CommandError, e:
- if not ignore_connect_failure:
- raise e
-
- if srv_list[0]:
- srv = srv_list[0]
- if self.target_uuid in config.inactive and self.permits_inactive():
- debug("%s inactive" % self.target_uuid)
- inactive_p = "inactive"
- else:
- debug("%s active" % self.target_uuid)
- inactive_p = ""
- lctl.newdev(self.module, self.name, self.uuid,
- setup ="%s %s %s" % (self.target_uuid, srv.nid_uuid,
- inactive_p))
- else:
- panic("Unable to create OSC for ", self.target_uuid)
-
- for tgt_dev_uuid in self.backup_targets:
- this_nets = get_ost_net(toplustreDB, tgt_dev_uuid)
- if len(this_nets) == 0:
- panic ("Unable to find a backup server for:", tgt_dev_uuid)
- else:
- for srv in this_nets:
- lctl.connect(srv)
- if srv:
- lctl.add_conn(self.name, srv.nid_uuid);
-
-
- def cleanup(self):
- if is_prepared(self.name):
- Module.cleanup(self)
- srv_list = self.get_servers()
- for srv in srv_list:
- lctl.disconnect(srv)
- for tgt_dev_uuid in self.backup_targets:
- this_nets = get_ost_net(toplustreDB, tgt_dev_uuid)
- if len(this_nets) == 0:
- panic ("Unable to find a backup server for:", tgt_dev_uuid)
- else:
- for srv in this_nets:
- lctl.disconnect(srv)
-
-class MDC(Client):
- def __init__(self, db, uuid, fs_name):
- Client.__init__(self, db, uuid, 'mdc', fs_name)
-
- def permits_inactive(self):
- return 0
-
-class OSC(Client):
- def __init__(self, db, uuid, fs_name):
- Client.__init__(self, db, uuid, 'osc', fs_name)
-
- def permits_inactive(self):
- return 1
-
-class COBD(Module):
- def __init__(self, db):
- Module.__init__(self, 'COBD', db)
- self.real_uuid = self.db.get_first_ref('realobd')
- self.cache_uuid = self.db.get_first_ref('cacheobd')
- self.add_lustre_module('cobd' , 'cobd')
-
- # need to check /proc/mounts and /etc/mtab before
- # formatting anything.
- # FIXME: check if device is already formatted.
- def prepare(self):
- if is_prepared(self.name):
- return
- self.info(self.real_uuid, self.cache_uuid)
- lctl.newdev("cobd", self.name, self.uuid,
- setup ="%s %s" %(self.real_uuid, self.cache_uuid))
-
-
-# virtual interface for OSC and LOV
-class VOSC(Module):
- def __init__(self, db, uuid, fs_name, name_override = None, quota = None):
- Module.__init__(self, 'VOSC', db)
- if quota:
- self.add_lustre_module('quota', 'lquota')
- if db.get_class() == 'lov':
- self.osc = LOV(db, uuid, fs_name, name_override)
- else:
- self.osc = get_osc(db, uuid, fs_name)
- def get_uuid(self):
- return self.osc.uuid
- def get_name(self):
- return self.osc.name
- def prepare(self):
- self.osc.prepare()
- def cleanup(self):
- self.osc.cleanup()
- def load_module(self):
- Module.load_module(self)
- self.osc.load_module()
- def cleanup_module(self):
- self.osc.cleanup_module()
- Module.cleanup_module(self)
-
-
-class ECHO_CLIENT(Module):
- def __init__(self,db):
- Module.__init__(self, 'ECHO_CLIENT', db)
- self.add_lustre_module('obdecho', 'obdecho')
- self.obd_uuid = self.db.get_first_ref('obd')
- obd = self.db.lookup(self.obd_uuid)
- self.uuid = generate_client_uuid(self.name)
- self.osc = VOSC(obd, self.uuid, self.name)
-
- def prepare(self):
- if is_prepared(self.name):
- return
- self.osc.prepare() # XXX This is so cheating. -p
- self.info(self.obd_uuid)
-
- lctl.newdev("echo_client", self.name, self.uuid,
- setup = self.osc.get_name())
-
- def cleanup(self):
- if is_prepared(self.name):
- Module.cleanup(self)
- self.osc.cleanup()
-
- def load_module(self):
- self.osc.load_module()
- Module.load_module(self)
-
- def cleanup_module(self):
- Module.cleanup_module(self)
- self.osc.cleanup_module()
-
-
-def generate_client_uuid(name):
- client_uuid = '%05x_%.19s_%05x%05x' % (int(random.random() * 1048576),
- name,
- int(random.random() * 1048576),
- int(random.random() * 1048576))
- return client_uuid[:36]
-
-
-def my_rstrip(s, chars):
- """my_rstrip(s, chars) -> strips any instances of the characters
- found in chars from the right side of string s"""
- # XXX required because python versions pre 2.2.3 don't allow
- #string.rstrip() to take alternate char lists
- import string
- ns=s
- try:
- ns = string.rstrip(s, '/')
- except TypeError, e:
- for i in range(len(s) - 1, 0, -1):
- if s[i] in chars:
- continue
- else:
- ns = s[0:i+1]
- break
- return ns
-
-
-class Mountpoint(Module):
- def __init__(self,db):
- Module.__init__(self, 'MTPT', db)
- self.path = my_rstrip(self.db.get_val('path'), '/')
- self.clientoptions = self.db.get_val('clientoptions', '')
- self.fs_uuid = self.db.get_first_ref('filesystem')
- fs = self.db.lookup(self.fs_uuid)
- self.mds_uuid = fs.get_first_ref('mds')
- mds_db = self.db.lookup(self.mds_uuid)
- if config.quota:
- quota = config.quota
- else:
- quota = mds_db.get_val('quota', config.quota)
- self.obd_uuid = fs.get_first_ref('obd')
- obd = self.db.lookup(self.obd_uuid)
- client_uuid = generate_client_uuid(self.name)
- self.vosc = VOSC(obd, client_uuid, self.name, quota=quota)
- self.mdc = get_mdc(db, client_uuid, self.name, self.mds_uuid)
-
- self.add_lustre_module('mdc', 'mdc')
- self.add_lustre_module('llite', 'llite')
-
- def prepare(self):
- if fs_is_mounted(self.path):
- log(self.path, "already mounted.")
- return
- self.vosc.prepare()
- self.mdc.prepare()
- mdc_name = self.mdc.name
-
- self.info(self.path, self.mds_uuid, self.obd_uuid)
- if config.record or config.lctl_dump:
- lctl.mount_option(local_node_name, self.vosc.get_name(), mdc_name)
- return
-
- if config.clientoptions:
- if self.clientoptions:
- self.clientoptions = self.clientoptions + ',' + config.clientoptions
- else:
- self.clientoptions = config.clientoptions
- if self.clientoptions:
- self.clientoptions = ',' + self.clientoptions
- # Linux kernel will deal with async and not pass it to ll_fill_super,
- # so replace it with Lustre async
- self.clientoptions = string.replace(self.clientoptions, "async", "lasync")
-
- cmd = "mount -t lustre_lite -o osc=%s,mdc=%s%s %s %s" % \
- (self.vosc.get_name(), mdc_name, self.clientoptions, config.config, self.path)
- run("mkdir", self.path)
- ret, val = run(cmd)
- if ret:
- self.mdc.cleanup()
- self.vosc.cleanup()
- panic("mount failed:", self.path, ":", string.join(val))
-
- def cleanup(self):
- self.info(self.path, self.mds_uuid,self.obd_uuid)
-
- if config.record or config.lctl_dump:
- lctl.del_mount_option(local_node_name)
- else:
- if fs_is_mounted(self.path):
- if config.force:
- (rc, out) = run("umount", "-f", self.path)
- else:
- (rc, out) = run("umount", self.path)
- if rc:
- raise CommandError('umount', out, rc)
-
- if fs_is_mounted(self.path):
- panic("fs is still mounted:", self.path)
-
- self.mdc.cleanup()
- self.vosc.cleanup()
-
- def load_module(self):
- self.vosc.load_module()
- Module.load_module(self)
-
- def cleanup_module(self):
- Module.cleanup_module(self)
- self.vosc.cleanup_module()
-
-
-# ============================================================
-# misc query functions
-
-def get_ost_net(self, osd_uuid):
- srv_list = []
- if not osd_uuid:
- return srv_list
- osd = self.lookup(osd_uuid)
- node_uuid = osd.get_first_ref('node')
- node = self.lookup(node_uuid)
- if not node:
- panic("unable to find node for osd_uuid:", osd_uuid,
- " node_ref:", node_uuid)
- for net_uuid in node.get_networks():
- db = node.lookup(net_uuid)
- net = Network(db, node_uuid)
- srv_list.append(net)
- return srv_list
-
-
-# the order of iniitailization is based on level.
-def getServiceLevel(self):
- type = self.get_class()
- ret=0;
- if type in ('network',):
- ret = 5
- elif type in ('ldlm',):
- ret = 20
- elif type in ('osd', 'cobd'):
- ret = 30
- elif type in ('mdsdev',):
- ret = 40
- elif type in ('mountpoint', 'echoclient'):
- ret = 70
- else:
- panic("Unknown type: ", type)
-
- if ret < config.minlevel or ret > config.maxlevel:
- ret = 0
- return ret
-
-#
-# return list of services in a profile. list is a list of tuples
-# [(level, db_object),]
-def getServices(self):
- list = []
- for ref_class, ref_uuid in self.get_all_refs():
- servdb = self.lookup(ref_uuid)
- if servdb:
- level = getServiceLevel(servdb)
- if level > 0:
- list.append((level, servdb))
- else:
- panic('service not found: ' + ref_uuid)
-
- list.sort()
- return list
-
-
-############################################################
-# MDC UUID hack -
-# FIXME: clean this mess up!
-#
-# OSC is no longer in the xml, so we have to fake it.
-# this is getting ugly and begging for another refactoring
-def get_osc(ost_db, uuid, fs_name):
- osc = OSC(ost_db, uuid, fs_name)
- return osc
-
-def get_mdc(db, uuid, fs_name, mds_uuid):
- mds_db = db.lookup(mds_uuid);
- if not mds_db:
- panic("no mds:", mds_uuid)
- mdc = MDC(mds_db, uuid, fs_name)
- return mdc
-
-def get_active_target(db):
- target_uuid = db.getUUID()
- target_name = db.getName()
- node_name = get_select(target_name)
- if node_name:
- tgt_dev_uuid = db.get_node_tgt_dev(node_name, target_uuid)
- else:
- tgt_dev_uuid = db.get_first_ref('active')
- return tgt_dev_uuid
-
-def get_server_by_nid_uuid(db, nid_uuid):
- for n in db.lookup_class("network"):
- net = Network(n)
- if net.nid_uuid == nid_uuid:
- return net
-
-
-############################################################
-# lconf level logic
-# Start a service.
-def newService(db):
- type = db.get_class()
- debug('Service:', type, db.getName(), db.getUUID())
- n = None
- if type == 'ldlm':
- n = LDLM(db)
- elif type == 'lov':
- n = LOV(db, "YOU_SHOULD_NEVER_SEE_THIS_UUID")
- elif type == 'network':
- n = Network(db)
- elif type == 'osd':
- n = OSD(db)
- elif type == 'cobd':
- n = COBD(db)
- elif type == 'mdsdev':
- n = MDSDEV(db)
- elif type == 'mountpoint':
- n = Mountpoint(db)
- elif type == 'echoclient':
- n = ECHO_CLIENT(db)
- else:
- panic("unknown service type:", type)
- return n
-
-#
-# Prepare the system to run lustre using a particular profile
-# in a the configuration.
-# * load & the modules
-# * setup networking for the current node
-# * make sure partitions are in place and prepared
-# * initialize devices with lctl
-# Levels is important, and needs to be enforced.
-def for_each_profile(db, prof_list, operation):
- for prof_uuid in prof_list:
- prof_db = db.lookup(prof_uuid)
- if not prof_db:
- panic("profile:", prof_uuid, "not found.")
- services = getServices(prof_db)
- operation(services)
-
-def doWriteconf(services):
- if config.nosetup:
- return
- have_mds = 0
- for s in services:
- if s[1].get_class() == 'mdsdev':
- n = newService(s[1])
- n.write_conf()
- have_mds = 1
- if have_mds == 0:
- panic("Cannot find mds device, please run --write_conf on the mds node.")
-
-
-def doSetup(services):
- if config.nosetup:
- return
- for s in services:
- n = newService(s[1])
- n.prepare()
-
-def doModules(services):
- if config.nomod:
- return
- for s in services:
- n = newService(s[1])
- n.load_module()
-
-def doCleanup(services):
- if config.nosetup:
- return
- services.reverse()
- for s in services:
- n = newService(s[1])
- if n.safe_to_clean():
- n.cleanup()
-
-def doUnloadModules(services):
- if config.nomod:
- return
- services.reverse()
- for s in services:
- n = newService(s[1])
- if n.safe_to_clean_modules():
- n.cleanup_module()
-
-def doMakeServiceScript(services):
- if config.nosetup:
- return
- try:
- os.makedirs(config.service_scripts)
- except OSError, e:
- if e[0] != errno.EEXIST:
- panic("Couldn't create scripts dir " + config.service_scripts + ": " + e[1])
-
- for s in services:
- if s[1].get_class() != 'osd' and s[1].get_class() != 'mdsdev':
- continue
-
- target_uuid = s[1].get_first_ref('target')
- target = toplustreDB.lookup(target_uuid)
- target_symlink = config.service_scripts + "/" + target.getName()
- if config.force:
- try:
- try:
- os.unlink(target_symlink)
- if config.verbose:
- print "Removed " + target_symlink
- except OSError, e:
- if e[0] != errno.EISDIR:
- raise e
- os.rmdir(target_symlink)
- if config.verbose:
- print "Removed " + target_symlink
- except OSError, e:
- if e[0] != errno.ENOENT:
- panic("Error removing " + target_symlink + ": " + e[1])
-
- try:
- os.symlink("/etc/init.d/lustre", target_symlink)
- if config.verbose:
- print "Created service link " + target_symlink + " to /etc/init.d/lustre"
-
- except OSError, e:
- if e[0] == errno.EEXIST:
- extra_error = " (use --force option to remove existing files)"
- else:
- extra_error = ""
- panic("Error creating " + target_symlink + ": " + e[1] + extra_error)
-
-# Check mtime of config logs
-def doCheckMtime(lustreDB, hosts):
- for h in hosts:
- node_db = lustreDB.lookup_name(h, 'node')
- if node_db:
- break
- if not node_db:
- return
-
- mdsdb = 0
- prof_list = node_db.get_refs('profile')
- for prof_uuid in prof_list:
- prof_db = node_db.lookup(prof_uuid)
- if prof_db:
- services = getServices(prof_db)
- for s in services:
- if s[1].get_class() == 'mdsdev':
- mdsdb = s[1]
- break
-
- if mdsdb and lustreDB.get_mtime():
- debug("Checking XML modification time")
- devpath = mdsdb.get_val('devpath','')
- xmtime = string.atol(lustreDB.get_mtime())
- cmd = "debugfs -c -R 'stat /LOGS' %s 2>&1 | grep mtime" %devpath
- ret, kmtimes = runcmd(cmd)
- if ret:
- log("Can not get mtime info of MDS LOGS directory")
- else:
- kmtime = string.atoi(string.split(kmtimes[0])[1], 0)
- if xmtime > kmtime:
- debug('xmtime ', xmtime, '> kmtime', kmtime)
- if config.old_conf:
- log("Warning: MDS startup logs are older than config %s."
- " Please run --write_conf on stopped MDS to update."
- %CONFIG_FILE)
- else:
- panic("Error: MDS startup logs are older than config %s."
- " Please run --write_conf on stopped MDS to update."
- " Use '--old_conf' to start anyways." %CONFIG_FILE)
- return
-
-#
-# Load profile for
-def doHost(lustreDB, hosts):
- global local_node_name, tgt_select
- node_db = None
- for h in hosts:
- node_db = lustreDB.lookup_name(h, 'node')
- if node_db:
- if config.service:
- tgt_select[config.service] = h
- config.group = config.service
- break
- if not node_db:
- panic('No host entry found.')
-
- local_node_name = node_db.get_val('name', 0)
- lustre_upcall = node_db.get_val('lustreUpcall', '')
- portals_upcall = node_db.get_val('portalsUpcall', '')
- timeout = node_db.get_val_int('timeout', 0)
- ptldebug = node_db.get_val('ptldebug', '')
- subsystem = node_db.get_val('subsystem', '')
-
- # Two step process: (1) load modules, (2) setup lustre
- # if not cleaning, load modules first.
- prof_list = node_db.get_refs('profile')
-
- if config.make_service_scripts:
- for_each_profile(node_db, prof_list, doMakeServiceScript)
- return
-
- elif config.write_conf:
- for_each_profile(node_db, prof_list, doModules)
- for_each_profile(node_db, prof_list, doWriteconf)
- for_each_profile(node_db, prof_list, doUnloadModules)
- lustreDB.close()
-
- elif config.recover:
- if not (config.tgt_uuid and config.client_uuid and config.conn_uuid):
- raise Lustre.LconfError( "--recovery requires --tgt_uuid <UUID> " +
- "--client_uuid <UUID> --conn_uuid <UUID>")
- doRecovery(lustreDB, lctl, config.tgt_uuid, config.client_uuid,
- config.conn_uuid)
- elif config.cleanup:
- if not mod_loaded('lnet'):
- return
-
- # ugly hack, only need to run lctl commands for --dump
- if config.lctl_dump or config.record:
- for_each_profile(node_db, prof_list, doCleanup)
- return
-
- sys_set_ptldebug(ptldebug)
- sys_set_subsystem(subsystem)
- sys_set_lustre_upcall(lustre_upcall)
- sys_set_portals_upcall(portals_upcall)
-
- for_each_profile(node_db, prof_list, doCleanup)
- for_each_profile(node_db, prof_list, doUnloadModules)
- lustreDB.close()
-
- else:
- # ugly hack, only need to run lctl commands for --dump
- if config.lctl_dump or config.record:
- sys_set_timeout(timeout)
- sys_set_lustre_upcall(lustre_upcall)
- for_each_profile(node_db, prof_list, doSetup)
- return
-
- if PLATFORM == 'LINUX':
- sys_set_netmem_max('/proc/sys/net/core/rmem_max', MAXTCPBUF)
- sys_set_netmem_max('/proc/sys/net/core/wmem_max', MAXTCPBUF)
-
- for_each_profile(node_db, prof_list, doModules)
-
- if PLATFORM == 'LINUX':
- # XXX need to be fixed for Darwin
- sys_set_debug_path()
- sys_set_ptldebug(ptldebug)
- sys_set_subsystem(subsystem)
- script = config.gdb_script
- run(lctl.lctl, ' modules >', script)
- if config.gdb:
- log ("The GDB module script is in", script)
- # pause, so user has time to break and
- # load the script
- time.sleep(5)
- sys_set_timeout(timeout)
- sys_set_lustre_upcall(lustre_upcall)
- sys_set_portals_upcall(portals_upcall)
-
- for_each_profile(node_db, prof_list, doSetup)
- lustreDB.close()
-
-def add_clumanager_node(node_db, nodes, services):
- new_services = []
- node_name = node_db.getUUID()
- nodes[node_name] = []
-
- for prof_uuid in node_db.get_refs('profile'):
- prof_db = toplustreDB.lookup(prof_uuid)
- for ref_class, ref_uuid in prof_db.get_all_refs():
- if ref_class not in ('osd', 'mdsdev'):
- continue
- devdb = toplustreDB.lookup(ref_uuid)
- tgt_uuid = devdb.get_first_ref('target')
-
- nodes[node_name].append(ref_uuid)
-
- if not services.has_key(tgt_uuid):
- if config.verbose:
- print "New service: " + tgt_uuid + " (originally found on " + node_name + ")"
- new_services.append(tgt_uuid)
- services[tgt_uuid] = []
- services[tgt_uuid].append(ref_uuid)
-
- return new_services
-
-def add_clumanager_services(new_services, nodes, dev_list):
- new_nodes = []
- for devdb in dev_list:
- tgt_uuid = devdb.get_first_ref('target')
- if tgt_uuid in new_services:
- node_uuid = devdb.get_first_ref('node')
-
- if not (nodes.has_key(node_uuid) or node_uuid in new_nodes):
- if config.verbose:
- print "New node: " + node_uuid + " for service " + tgt_uuid
- new_nodes.append(node_uuid)
-
- return new_nodes
-
-def doClumanager(lustreDB, hosts):
- nodes = {}
- services = {}
-
- dev_list = []
-
- for dev_uuid in toplustreDB.get_refs('osd') + toplustreDB.get_refs('mdsdev'):
- dev_list.append(lustreDB.lookup(dev_uuid))
-
- node_db = None
- for h in hosts:
- node_db = lustreDB.lookup_name(h, 'node')
- if node_db:
- our_host = h
- new_services = add_clumanager_node(node_db, nodes, services)
- break
-
- if not node_db:
- panic('No host entry found.')
-
- while 1:
- if len(new_services) == 0:
- break
-
- new_nodes = add_clumanager_services(new_services, nodes, dev_list)
- if len(new_nodes) == 0:
- break
-
- if len(new_nodes) + len(nodes.keys()) > 8:
- panic("CluManager only supports 8 nodes per failover \"cluster.\"")
-
- new_services = []
- for node_uuid in new_nodes:
- node_db = lustreDB.lookup(node_uuid)
- if not node_db:
- panic("No node entry for " + node_uuid + " was found.")
-
- new_services.append(add_clumanager_node(node_db, nodes, services))
-
- nodenames = []
- for node in nodes.keys():
- nodedb = lustreDB.lookup(node)
- nodenames.append(nodedb.getName())
- nodenames.sort()
-
- print """<?xml version="1.0"?>
-<cluconfig version="3.0">
- <clumembd broadcast="no" interval="750000" loglevel="5" multicast="yes" multicast_ipaddress="225.0.0.11" thread="yes" tko_count="20"/>
- <cluquorumd loglevel="5" pinginterval="2"/>
- <clurmtabd loglevel="5" pollinterval="4"/>
- <clusvcmgrd loglevel="5"/>
- <clulockd loglevel="5"/>
- <cluster config_viewnumber="1" name="%s"/>
- <sharedstate driver="libsharedraw.so" rawprimary="%s" rawshadow="%s" type="raw"/>
- <members> """ % (string.join(nodenames), config.rawprimary, config.rawsecondary)
-
-
- i = 0
- for node in nodenames:
- print " <member id=\"%d\" name=\"%s\" watchdog=\"yes\"/>" % (i, node)
- i = i + 1
-
- print " </members>\n <failoverdomains>"
-
- servicekeys = services.keys()
- servicekeys.sort()
-
- i = 0
- for service in servicekeys:
- svcdb = lustreDB.lookup(service)
- print " <failoverdomain id=\"%d\" name=\"%s\" ordered=\"yes\" restricted=\"yes\">" % (i, svcdb.getName())
- i = i + 1
-
- j = 0
- active_uuid = get_active_target(svcdb)
- for svc_uuid in [active_uuid] + services[service]:
- if svc_uuid == active_uuid and j > 0:
- continue
- svcdb = lustreDB.lookup(svc_uuid)
-
- svc_node_uuid = svcdb.get_first_ref('node')
- svc_nodedb = lustreDB.lookup(svc_node_uuid)
-
- print " <failoverdomainnode id=\"%d\" name=\"%s\"/>" % (j, svc_nodedb.getName())
- j = j + 1
-
- print " </failoverdomain>"
-
- print " </failoverdomains>\n <services>"
-
- i = 0
- for service in servicekeys:
- svcdb = lustreDB.lookup(service)
- active_uuid = get_active_target(svcdb)
- activedb = lustreDB.lookup(active_uuid)
-
- svc_node_uuid = activedb.get_first_ref('node')
- svc_nodedb = lustreDB.lookup(svc_node_uuid)
-
- print " <service checkinterval=\"30\" failoverdomain=\"%s\" id=\"%d\" name=\"%s\" userscript=\"%s/%s\">" \
- % ( svcdb.getName(), i, svcdb.getName(), config.service_scripts, svcdb.getName())
- print " <service_ipaddresses/>\n </service>"
- i = i + 1
-
- print " </services>\n</cluconfig>"
-
-def doRecovery(lustreDB, lctl, tgt_uuid, client_uuid, nid_uuid):
- tgt = lustreDB.lookup(tgt_uuid)
- if not tgt:
- raise Lustre.LconfError("doRecovery: "+ tgt_uuid +" not found.")
- new_uuid = get_active_target(tgt)
- if not new_uuid:
- raise Lustre.LconfError("doRecovery: no active target found for: " +
- tgt_uuid)
- srv_list = find_local_servers(get_ost_net(lustreDB, new_uuid))
- if not srv_list[0]:
- raise Lustre.LconfError("Unable to find a connection to:" + new_uuid)
-
- oldsrv = get_server_by_nid_uuid(lustreDB, nid_uuid)
- lustreDB.close()
-
- for srv in srv_list:
- if oldsrv.net_type != srv.net_type:
- continue
-
- log("Reconnecting", tgt_uuid, "to", srv.nid_uuid)
-
- lctl.recover(client_uuid, srv.nid_uuid)
-
-
-def setupModulePath(cmd, portals_dir = PORTALS_DIR):
- base = os.path.dirname(cmd)
- if development_mode():
- if not config.lustre:
- debug('using objdir module paths')
- config.lustre = (os.path.join(base, ".."))
- # normalize the portals dir, using command line arg if set
- if config.portals:
- portals_dir = config.portals
- dir = os.path.join(config.lustre, portals_dir)
- config.portals = dir
- debug('config.portals', config.portals)
- elif config.lustre and config.portals:
- # production mode
- # if --lustre and --portals, normalize portals
- # can ignore POTRALS_DIR here, since it is probly useless here
- config.portals = os.path.join(config.lustre, config.portals)
- debug('config.portals B', config.portals)
-
-def sysctl(path, val):
- debug("+ sysctl", path, val)
- if config.noexec:
- return
- try:
- fp = open(os.path.join('/proc/sys', path), 'w')
- fp.write(str(val))
- fp.close()
- except IOError, e:
- panic(str(e))
-
-
-def sys_set_debug_path():
- sysctl('lnet/debug_path', config.debug_path)
-
-def validate_upcall(upcall):
- import os
- if upcall in ('DEFAULT','NONE'):
- pass
- elif os.path.exists(upcall):
- if not os.access(upcall, os.X_OK):
- print "WARNING upcall script not executable: %s" % upcall
- else:
- print "WARNING invalid upcall script specified: %s" % upcall
-
-def sys_set_lustre_upcall(upcall):
- # the command line overrides the value in the node config
- if config.lustre_upcall:
- upcall = config.lustre_upcall
- elif config.upcall:
- upcall = config.upcall
- if upcall:
- validate_upcall(upcall)
- lctl.set_lustre_upcall(upcall)
-
-def sys_set_portals_upcall(upcall):
- # the command line overrides the value in the node config
- if config.portals_upcall:
- upcall = config.portals_upcall
- elif config.upcall:
- upcall = config.upcall
- if upcall:
- validate_upcall(upcall)
- sysctl('lnet/upcall', upcall)
-
-def sys_set_group_upcall(mds, upcall):
- if config.noexec:
- return
- # the command line overrides the value in the MDS config
- if config.group_upcall:
- upcall = config.group_upcall
- if upcall:
- validate_upcall(upcall)
- debug("setting MDS", mds, "upcall to:", upcall)
- path = "/proc/fs/lustre/mds/" + mds + "/group_upcall"
- fp = open(path, 'w')
- fp.write(upcall)
- fp.close()
-
-def sys_set_timeout(timeout):
- # the command overrides the value in the node config
- if config.timeout and config.timeout > 0:
- timeout = config.timeout
- if timeout != None and timeout > 0:
- lctl.set_timeout(timeout)
-
-def sys_tweak_socknal ():
- if config.single_socket:
- sysctl("socknal/typed", 0)
-
-def sys_optimize_elan ():
- procfiles = ["/proc/elan/config/eventint_punt_loops",
- "/proc/qsnet/elan3/config/eventint_punt_loops",
- "/proc/qsnet/elan4/config/elan4_mainint_punt_loops"]
- for p in procfiles:
- if os.access(p, os.W_OK):
- run ("echo 1 > " + p)
-
-def sys_set_ptldebug(ptldebug):
- if config.ptldebug:
- ptldebug = config.ptldebug
- if ptldebug:
- try:
- val = eval(ptldebug, ptldebug_names)
- val = "0x%x" % (val)
- sysctl('lnet/debug', val)
- except NameError, e:
- panic(str(e))
-
-def sys_set_subsystem(subsystem):
- if config.subsystem:
- subsystem = config.subsystem
- if subsystem:
- try:
- val = eval(subsystem, subsystem_names)
- val = "0x%x" % (val)
- sysctl('lnet/subsystem_debug', val)
- except NameError, e:
- panic(str(e))
-
-def sys_set_netmem_max(path, max):
- debug("setting", path, "to at least", max)
- if config.noexec:
- return
- fp = open(path)
- str = fp.readline()
- fp.close()
- cur = int(str)
- if max > cur:
- fp = open(path, 'w')
- fp.write('%d\n' %(max))
- fp.close()
-
-
-# Add dir to the global PATH, if not already there.
-def add_to_path(new_dir):
- syspath = string.split(os.environ['PATH'], ':')
- if new_dir in syspath:
- return
- os.environ['PATH'] = os.environ['PATH'] + ':' + new_dir
-
-def default_debug_path():
- path = '/tmp/lustre-log'
- if os.path.isdir('/r'):
- return '/r' + path
- else:
- return path
-
-def default_gdb_script():
- script = '/tmp/ogdb'
- if os.path.isdir('/r'):
- return '/r' + script
- else:
- return script
-
-DEFAULT_PATH = ('/sbin', '/usr/sbin', '/bin', '/usr/bin')
-# ensure basic elements are in the system path
-def sanitise_path():
- for dir in DEFAULT_PATH:
- add_to_path(dir)
-
-# global hack for the --select handling
-tgt_select = {}
-def init_select(args):
- # args = [service=nodeA,service2=nodeB service3=nodeC]
- # --service <service> is analagous to:
- # --group <service> --select <service>=<node>
- # this is handled in doHost()
- global tgt_select
- for arg in args:
- list = string.split(arg, ',')
- for entry in list:
- srv, node = string.split(entry, '=')
- tgt_select[srv] = node
-
-def get_select(srv):
- if tgt_select.has_key(srv):
- return tgt_select[srv]
- return None
-
-
-FLAG = Lustre.Options.FLAG
-PARAM = Lustre.Options.PARAM
-INTPARAM = Lustre.Options.INTPARAM
-PARAMLIST = Lustre.Options.PARAMLIST
-lconf_options = [
- ('verbose,v', "Print system commands as they are run"),
- ('ldapurl',"LDAP server URL, eg. ldap://localhost", PARAM),
- ('config', "Cluster config name used for LDAP query", PARAM),
- ('select', "service=nodeA,service2=nodeB ", PARAMLIST),
- ('service', "shorthand for --group <service> --select <service>=<node>", PARAM),
- ('node', "Load config for <nodename>", PARAM),
- ('cleanup,d', "Cleans up config. (Shutdown)"),
- ('force,f', "Forced unmounting and/or obd detach during cleanup",
- FLAG, 0),
- ('single_socket', "socknal option: only use one socket instead of bundle",
- FLAG, 0),
- ('failover',"""Used to shut down without saving state.
- This will allow this node to "give up" a service to a
- another node for failover purposes. This will not
- be a clean shutdown.""",
- FLAG, 0),
- ('abort_recovery',"""Used to start a service when you know recovery
- will not succeed. This will skip the recovery
- timeout period."""),
- ('gdb', """Prints message after creating gdb module script
- and sleeps for 5 seconds."""),
- ('noexec,n', """Prints the commands and steps that will be run for a
- config without executing them. This can used to check if a
- config file is doing what it should be doing"""),
- ('nomod', "Skip load/unload module step."),
- ('nosetup', "Skip device setup/cleanup step."),
- ('reformat', "Reformat all devices (without question)"),
- ('mkfsoptions', "Additional options for the mk*fs command line", PARAM),
- ('mountfsoptions', "Additional options for mount fs command line", PARAM),
- ('clientoptions', "Additional options for Lustre", PARAM),
- ('dump', "Dump the kernel debug log to file before portals is unloaded",
- PARAM),
- ('write_conf', "Save all the client config information on mds."),
- ('old_conf', "Start up service even though config logs appear outdated."),
- ('record', "Write config information on mds."),
- ('record_log', "Name of config record log.", PARAM),
- ('record_device', "MDS device name that will record the config commands",
- PARAM),
- ('minlevel', "Minimum level of services to configure/cleanup",
- INTPARAM, 0),
- ('maxlevel', """Maximum level of services to configure/cleanup
- Levels are aproximatly like:
- 10 - network
- 20 - device, ldlm
- 30 - osd, mdd
- 40 - mds, ost
- 70 - mountpoint, echo_client, osc, mdc, lov""",
- INTPARAM, 100),
- ('lustre', """Base directory of lustre sources. This parameter will
- cause lconf to load modules from a source tree.""", PARAM),
- ('portals', """Portals source directory. If this is a relative path,
- then it is assumed to be relative to lustre. """, PARAM),
- ('timeout', "Set recovery timeout", INTPARAM),
- ('upcall', "Set both portals and lustre upcall script", PARAM),
- ('lustre_upcall', "Set lustre upcall script", PARAM),
- ('portals_upcall', "Set portals upcall script", PARAM),
- ('group_upcall', "Set supplementary group upcall program", PARAM),
- ('lctl_dump', "Save lctl ioctls to the dumpfile argument", PARAM),
- ('ptldebug', "Set the portals debug level", PARAM),
- ('subsystem', "Set the portals debug subsystem", PARAM),
- ('gdb_script', "Fullname of gdb debug script", PARAM, default_gdb_script()),
- ('debug_path', "Path to save debug dumps", PARAM, default_debug_path()),
- ('allow_unprivileged_port', "Allow connections from unprivileged ports"),
- ('clumanager', "Generate CluManager config file for this node's cluster"),
- ('rawprimary', "For clumanager, device of the primary quorum", PARAM, "/dev/raw/raw1"),
- ('rawsecondary', "For clumanager, device of the secondary quorum", PARAM, "/dev/raw/raw2"),
- ('service_scripts', "For clumanager, directory containing per-service scripts", PARAM, "/etc/lustre/services"),
- ('make_service_scripts', "Create per-service symlinks for use with clumanager"),
-# Client recovery options
- ('recover', "Recover a device"),
- ('group,g', "The group of devices to configure or cleanup", PARAM),
- ('tgt_uuid', "The failed target (required for recovery)", PARAM),
- ('client_uuid', "The failed client (required for recovery)", PARAM),
- ('conn_uuid', "The failed connection (required for recovery)", PARAM),
-
- ('inactive', """The name of an inactive service, to be ignored during
- mounting (currently OST-only). Can be repeated.""",
- PARAMLIST),
- ('user_xattr', """Enable user_xattr support on MDS""", FLAG, 0),
- ('acl', """Enable ACL support on MDS""", FLAG, 0),
- ('quota', "Enable quota support for client file system", PARAM),
- ]
-
-def main():
- global lctl, config, toplustreDB, CONFIG_FILE
-
- # in the upcall this is set to SIG_IGN
- signal.signal(signal.SIGCHLD, signal.SIG_DFL)
-
- cl = Lustre.Options("lconf", "config.xml", lconf_options)
- try:
- config, args = cl.parse(sys.argv[1:])
- except Lustre.OptionError, e:
- print e
- sys.exit(1)
-
- setupModulePath(sys.argv[0])
-
- host = socket.gethostname()
-
- # the PRNG is normally seeded with time(), which is not so good for starting
- # time-synchronized clusters
- input = open('/dev/urandom', 'r')
- if not input:
- print 'Unable to open /dev/urandom!'
- sys.exit(1)
- seed = input.read(32)
- input.close()
- random.seed(seed)
-
- sanitise_path()
-
- init_select(config.select)
-
- if len(args) > 0:
- # allow config to be fetched via HTTP, but only with python2
- if sys.version[0] != '1' and args[0].startswith('http://'):
- import urllib2
- try:
- config_file = urllib2.urlopen(args[0])
- except (urllib2.URLError, socket.error), err:
- if hasattr(err, 'args'):
- err = err.args[1]
- print "Could not access '%s': %s" %(args[0], err)
- sys.exit(1)
- elif not os.access(args[0], os.R_OK):
- print 'File not found or readable:', args[0]
- sys.exit(1)
- else:
- # regular file
- config_file = open(args[0], 'r')
- try:
- dom = xml.dom.minidom.parse(config_file)
- except Exception:
- panic("%s does not appear to be a config file." % (args[0]))
- sys.exit(1) # make sure to die here, even in debug mode.
- config_file.close()
- CONFIG_FILE = args[0]
- lustreDB = Lustre.LustreDB_XML(dom.documentElement, dom.documentElement)
- if not config.config:
- config.config = os.path.basename(args[0])# use full path?
- if config.config[-4:] == '.xml':
- config.config = config.config[:-4]
- elif config.ldapurl:
- if not config.config:
- panic("--ldapurl requires --config name")
- dn = "config=%s,fs=lustre" % (config.config)
- lustreDB = Lustre.LustreDB_LDAP('', {}, base=dn, url = config.ldapurl)
- elif config.ptldebug or config.subsystem:
- sys_set_ptldebug(None)
- sys_set_subsystem(None)
- sys.exit(0)
- else:
- print 'Missing config file or ldap URL.'
- print 'see lconf --help for command summary'
- sys.exit(1)
-
- if config.reformat and config.cleanup:
- panic("Options \"reformat\" and \"cleanup\" are incompatible. "+
- "Please specify only one.")
-
- toplustreDB = lustreDB
-
- ver = lustreDB.get_version()
- if not ver:
- panic("No version found in config data, please recreate.")
- if ver != Lustre.CONFIG_VERSION:
- panic("Config version", ver, "does not match lconf version",
- Lustre.CONFIG_VERSION)
-
- node_list = []
- if config.node:
- node_list.append(config.node)
- else:
- if len(host) > 0:
- node_list.append(host)
-# node_list.append('localhost')
-
- debug("configuring for host: ", node_list)
-
- if len(host) > 0:
- config.debug_path = config.debug_path + '-' + host
- config.gdb_script = config.gdb_script + '-' + host
-
- lctl = LCTLInterface('lctl')
-
- if config.lctl_dump:
- lctl.use_save_file(config.lctl_dump)
-
- if not (config.reformat or config.write_conf or config.cleanup):
- doCheckMtime(lustreDB, node_list)
-
- if config.record:
- if not (config.record_device and config.record_log):
- panic("When recording, both --record_log and --record_device must be specified.")
- lctl.clear_log(config.record_device, config.record_log)
- lctl.record(config.record_device, config.record_log)
-
- if config.clumanager:
- doClumanager(lustreDB, node_list)
- else:
- doHost(lustreDB, node_list)
-
- if config.record:
- lctl.end_record()
-
-if __name__ == "__main__":
- try:
- main()
- except Lustre.LconfError, e:
- print e
-# traceback.print_exc(file=sys.stdout)
- sys.exit(1)
- except CommandError, e:
- e.dump()
- rc = e.rc
- if rc == 0:
- rc = 1
- sys.exit(rc)
-
- if first_cleanup_error:
- sys.exit(first_cleanup_error)
git://git.whamcloud.com / fs / lustre-release.git / blobdiff