From: nfshp Date: Tue, 1 Apr 2003 16:23:37 +0000 (+0000) Subject: merge b_devel into b_cray X-Git-Tag: v1_7_100~1^38~21 X-Git-Url: https://git.whamcloud.com/?a=commitdiff_plain;h=1445ba0cad6db0f4ac1f97cc209c55babca1095b;p=fs%2Flustre-release.git merge b_devel into b_cray --- diff --git a/lustre/include/linux/lustre_export.h b/lustre/include/linux/lustre_export.h index 694bd3e..550a8d7 100644 --- a/lustre/include/linux/lustre_export.h +++ b/lustre/include/linux/lustre_export.h @@ -26,13 +26,14 @@ struct ec_export_data { /* echo client */ }; struct obd_export { - __u64 exp_cookie; + struct portals_handle exp_handle; + atomic_t exp_refcount; struct obd_uuid exp_client_uuid; struct list_head exp_obd_chain; - struct list_head exp_conn_chain; struct obd_device *exp_obd; struct ptlrpc_connection *exp_connection; struct ldlm_export_data exp_ldlm_data; + struct ptlrpc_request *exp_outstanding_reply; union { struct mds_export_data eu_mds_data; struct filter_export_data eu_filter_data; diff --git a/lustre/utils/lconf.in b/lustre/utils/lconf.in index 0c04d62..3c6ca73 100755 --- a/lustre/utils/lconf.in +++ b/lustre/utils/lconf.in @@ -26,7 +26,7 @@ import sys, getopt, types import string, os, stat, popen2, socket, time, random, fcntl, select -import re, exceptions +import re, exceptions, signal import xml.dom.minidom if sys.version[0] == '1': @@ -34,6 +34,19 @@ if sys.version[0] == '1': else: from fcntl import F_GETFL, F_SETFL +PYMOD_DIR = "/usr/lib/lustre/python" + +def development_mode(): + base = os.path.dirname(sys.argv[0]) + if os.access(base+"/Makefile.am", os.R_OK): + return 1 + return 0 + +if not development_mode(): + sys.path.append(PYMOD_DIR) + +import Lustre + # Global parameters MAXTCPBUF = 1048576 DEFAULT_TCPBUF = 1048576 @@ -49,194 +62,16 @@ def cleanup_error(rc): if not first_cleanup_error: first_cleanup_error = rc - -def usage(): - print """usage: lconf config.xml - -config.xml Lustre configuration in xml format. ---ldapurl LDAP server URL, eg. ldap://localhost ---config Cluster config name used for LDAP query ---node Load config for ---select service=nodeA,service2=nodeB U --d | --cleanup Cleans up config. (Shutdown) --f | --force Forced unmounting and/or obd detach during cleanup --v | --verbose Print system commands as they are run --h | --help Print this help ---gdb Prints message after creating gdb module script - and sleeps for 5 seconds. --n | --noexec Prints the commands and steps that will be run for a - config without executing them. This can used to check if a - config file is doing what it should be doing. (Implies -v) ---nomod Skip load/unload module step. ---nosetup Skip device setup/cleanup step. ---reformat Reformat all devices (without question) ---dump Dump the kernel debug log before portals is unloaded ---minlevel Specify the minimum level of services to configure/cleanup (default 0) ---maxlevel Specify the maximum level of services to configure/cleanup (default 100) - Levels are aproximatly like: - 10 - network - 20 - device, ldlm - 30 - osd, mdd - 40 - mds, ost - 50 - mdc, osc - 60 - lov - 70 - mountpoint, echo_client ---lustre=src_dir Base directory of lustre sources. This parameter will cause lconf - to load modules from a source tree. ---portals=src_dir Portals source directory. If this is a relative path, then it is - assumed to be relative to lustre. - -""" - TODO = """ ---ldap server LDAP server with lustre config database ---makeldiff Translate xml source to LDIFF -This are perhaps not needed: -""" - sys.exit() - -# ============================================================ -# Config parameters, encapsulated in a class -class Config: - def __init__(self): - # flags - self._noexec = 0 - self._verbose = 0 - self._reformat = 0 - self._cleanup = 0 - self._gdb = 0 - self._nomod = 0 - self._nosetup = 0 - self._force = 0 - # parameters - self._modules = None - self._node = None - self._url = None - self._gdb_script = '/tmp/ogdb' - self._debug_path = '/tmp/lustre-log' - self._dump_file = None - self._lustre_dir = '' - self._portals_dir = '' - self._minlevel = 0 - self._maxlevel = 100 - self._timeout = 0 - self._recovery_upcall = '' - self._ldapurl = '' - self._config_name = '' - self._select = {} - self._lctl_dump = '' - - def verbose(self, flag = None): - if flag: self._verbose = flag - return self._verbose - - def noexec(self, flag = None): - if flag: self._noexec = flag - return self._noexec - - def reformat(self, flag = None): - if flag: self._reformat = flag - return self._reformat - - def cleanup(self, flag = None): - if flag: self._cleanup = flag - return self._cleanup - - def gdb(self, flag = None): - if flag: self._gdb = flag - return self._gdb - - def nomod(self, flag = None): - if flag: self._nomod = flag - return self._nomod - - def nosetup(self, flag = None): - if flag: self._nosetup = flag - return self._nosetup - - def force(self, flag = None): - if flag: self._force = flag - return self._force - - def node(self, val = None): - if val: self._node = val - return self._node - - def gdb_script(self): - if os.path.isdir('/r'): - return '/r' + self._gdb_script - else: - return self._gdb_script - - def debug_path(self): - if os.path.isdir('/r'): - return '/r' + self._debug_path - else: - return self._debug_path - - def dump_file(self, val = None): - if val: self._dump_file = val - return self._dump_file - def minlevel(self, val = None): - if val: self._minlevel = int(val) - return self._minlevel - - def maxlevel(self, val = None): - if val: self._maxlevel = int(val) - return self._maxlevel - - def portals_dir(self, val = None): - if val: self._portals_dir = val - return self._portals_dir - - def lustre_dir(self, val = None): - if val: self._lustre_dir = val - return self._lustre_dir - - def timeout(self, val = None): - if val: self._timeout = val - return self._timeout - - def recovery_upcall(self, val = None): - if val: self._recovery_upcall = val - return self._recovery_upcall - - def ldapurl(self, val = None): - if val: self._ldapurl = val - return self._ldapurl - - def config_name(self, val = None): - if val: self._config_name = val - return self._config_name - - def init_select(self, arg): - # arg = "service=nodeA,service2=nodeB" - list = string.split(arg, ',') - for entry in list: - srv, node = string.split(entry, '=') - self._select[srv] = node - - def select(self, srv): - if self._select.has_key(srv): - return self._select[srv] - return None - - def lctl_dump(self, val = None): - if val: self._lctl_dump = val - return self._lctl_dump - - -config = Config() - # ============================================================ # debugging and error funcs def fixme(msg = "this feature"): - raise LconfError, msg + ' not implmemented yet.' + raise Lustre.LconfError, msg + ' not implmemented yet.' def panic(*args): msg = string.join(map(str,args)) - if not config.noexec(): - raise LconfError(msg) + if not config.noexec: + raise Lustre.LconfError(msg) else: print "! " + msg @@ -249,10 +84,11 @@ def logall(msgs): print string.strip(s) def debug(*args): - if config.verbose(): + if config.verbose: msg = string.join(map(str,args)) print msg + # ============================================================ # locally defined exceptions class CommandError (exceptions.Exception): @@ -278,10 +114,6 @@ class CommandError (exceptions.Exception): else: print self.cmd_err -class LconfError (exceptions.Exception): - def __init__(self, args): - self.args = args - # ============================================================ # handle daemons, like the acceptor @@ -395,7 +227,7 @@ class LCTLInterface: self.lctl = find_prog(cmd) self.save_file = '' if not self.lctl: - if config.noexec(): + if config.noexec: debug('! lctl not found') self.lctl = 'lctl' else: @@ -422,7 +254,7 @@ class LCTLInterface: cmds = '\n dump ' + self.save_file + cmds debug("+", cmd_line, cmds) - if config.noexec(): return (0, []) + if config.noexec: return (0, []) child = popen2.Popen3(cmd_line, 1) # Capture stdout and stderr from command child.tochild.write(cmds + "\n") @@ -486,7 +318,7 @@ class LCTLInterface: # create a new connection def connect(self, srv): cmds = "\n add_uuid %s %s %s" % (srv.uuid, srv.nid, srv.net_type) - if srv.net_type in ('tcp', 'toe') and not config.lctl_dump(): + if srv.net_type in ('tcp', 'toe') and not config.lctl_dump: flags = '' if srv.irq_affinity: flags = flags + 'i' @@ -503,6 +335,14 @@ class LCTLInterface: cmds = cmds + "\n quit" self.run(cmds) + + # Recover a device + def recover(self, dev_uuid, new_conn): + cmds = """ + device %%%s + probe + recover %s""" %(dev_uuid, new_conn) + self.run(cmds) # add a route to a range def add_route(self, net, gw, lo, hi): @@ -578,11 +418,12 @@ class LCTLInterface: device $%s cleanup %s detach - quit""" % (name, ('', 'force')[config.force()]) + quit""" % (name, ('', 'force')[config.force]) self.run(cmds) # create an lov - def lov_setconfig(self, uuid, mdsuuid, stripe_cnt, stripe_sz, stripe_off, pattern, devlist): + def lov_setconfig(self, uuid, mdsuuid, stripe_cnt, stripe_sz, stripe_off, + pattern, devlist): cmds = """ device $%s probe @@ -616,7 +457,7 @@ class LCTLInterface: # save it if necessary def runcmd(cmd): debug ("+", cmd) - if config.noexec(): return (0, []) + if config.noexec: return (0, []) f = os.popen(cmd + ' 2>&1') out = f.readlines() ret = f.close() @@ -634,7 +475,7 @@ def run(*args): def run_daemon(*args): cmd = string.join(map(str,args)) debug ("+", cmd) - if config.noexec(): return 0 + if config.noexec: return 0 f = os.popen(cmd + ' 2>&1') ret = f.close() if ret: @@ -649,8 +490,8 @@ def find_prog(cmd): syspath = string.split(os.environ['PATH'], ':') cmdpath = os.path.dirname(sys.argv[0]) syspath.insert(0, cmdpath); - if config.portals_dir(): - syspath.insert(0, os.path.join(config.portals_dir()+'/linux/utils/')) + if config.portals: + syspath.insert(0, os.path.join(config.portals, 'linux/utils/')) for d in syspath: prog = os.path.join(d,cmd) if os.access(prog, os.X_OK): @@ -690,20 +531,25 @@ def is_block(path): # build fs according to type # fixme: dangerous -def mkfs(dev, devsize, fstype): +def mkfs(dev, devsize, fstype,jsize): block_cnt = '' + jopt = '' if devsize: # devsize is in 1k, and fs block count is in 4k block_cnt = devsize/4 if(fstype in ('ext3', 'extN')): + # ext3 journal size is in megabytes + if jsize: jopt = "-J size=%d" %(jsize,) mkfs = 'mkfs.ext2 -j -b 4096 -F ' elif (fstype == 'reiserfs'): + # reiserfs journal size is in blocks + if jsize: jopt = "--journal_size %d" %(jsize,) mkfs = 'mkreiserfs -ff' else: print 'unsupported fs type: ', fstype - (ret, out) = run (mkfs, dev, block_cnt) + (ret, out) = run (mkfs, jopt, dev, block_cnt) if ret: panic("Unable to build fs:", dev) # enable hash tree indexing on fsswe @@ -745,7 +591,7 @@ def init_loop(file, size, fstype): if dev: print 'WARNING file:', file, 'already mapped to', dev return dev - if config.reformat() or not os.access(file, os.R_OK | os.W_OK): + if config.reformat or not os.access(file, os.R_OK | os.W_OK): if size < 8000: panic(file, "size must be larger than 8MB, currently set to:", size) (ret, out) = run("dd if=/dev/zero bs=1k count=0 seek=%d of=%s" %(size, @@ -783,12 +629,12 @@ def need_format(fstype, dev): return 0 # initialize a block device if needed -def block_dev(dev, size, fstype, format): - if config.noexec(): return dev +def block_dev(dev, size, fstype, format, journal_size): + if config.noexec: return dev if not is_block(dev): dev = init_loop(dev, size, fstype) - if config.reformat() or (need_format(fstype, dev) and format == 'yes'): - mkfs(dev, size, fstype) + if config.reformat or (need_format(fstype, dev) and format == 'yes'): + mkfs(dev, size, fstype, journal_size) # else: # panic("device:", dev, @@ -807,8 +653,7 @@ def if2addr(iface): return ip def get_local_nid(net_type, wildcard): - """Return the local nid. First look for an elan interface, - then use the local address. """ + """Return the local nid.""" local = "" if os.access('/proc/elan/device0/position', os.R_OK): local = get_local_address('elan', '*') @@ -848,11 +693,13 @@ def get_local_address(net_type, wildcard): def is_prepared(uuid): """Return true if a device exists for the uuid""" - # expect this format: - # 1 UP ldlm ldlm ldlm_UUID 2 - if config.lctl_dump(): + if config.lctl_dump: return 0 + if config.noexec and config.cleanup: + return 1 try: + # expect this format: + # 1 UP ldlm ldlm ldlm_UUID 2 out = lctl.device_list() for s in out: if uuid == string.split(s)[4]: @@ -862,19 +709,9 @@ def is_prepared(uuid): return 0 def is_network_prepared(): - """If the PTLRPC device exists, then assumet that all networking + """If the LDLM device exists, then assume that all networking has been configured""" - if config.lctl_dump(): - return 0 - try: - out = lctl.device_list() - for s in out: - if 'RPCDEV_UUID' == string.split(s)[4]: - return 1 - except CommandError, e: - e.dump() - return 0 - + return is_prepared('ldlm_UUID') def fs_is_mounted(path): """Return true if path is a mounted lustre filesystem""" @@ -923,11 +760,11 @@ class Module: def add_portals_module(self, dev_dir, modname): """Append a module to list of modules to load.""" - self.kmodule_list.append((config.portals_dir(), dev_dir, modname)) + self.kmodule_list.append((config.portals, dev_dir, modname)) def add_lustre_module(self, dev_dir, modname): """Append a module to list of modules to load.""" - self.kmodule_list.append((config.lustre_dir(), dev_dir, modname)) + self.kmodule_list.append((config.lustre, dev_dir, modname)) def mod_loaded(self, modname): """Check if a module is already loaded. Look in /proc/modules for it.""" @@ -943,7 +780,7 @@ class Module: """Load all the modules in the list in the order they appear.""" for src_dir, dev_dir, mod in self.kmodule_list: # (rc, out) = run ('/sbin/lsmod | grep -s', mod) - if self.mod_loaded(mod) and not config.noexec(): + if self.mod_loaded(mod) and not config.noexec: continue log ('loading module:', mod) if src_dir: @@ -963,14 +800,12 @@ class Module: rev = self.kmodule_list rev.reverse() for src_dir, dev_dir, mod in rev: - if not self.mod_loaded(mod): + if not self.mod_loaded(mod) and not config.noexec: continue # debug hack - if mod == 'portals' and config.dump_file(): - lctl.dump(config.dump_file()) + if mod == 'portals' and config.dump: + lctl.dump(config.dump) log('unloading module:', mod) - if config.noexec(): - continue (rc, out) = run('/sbin/rmmod', mod) if rc: log('! unable to unload module:', mod) @@ -988,7 +823,10 @@ class Network(Module): self.nid_exchange = self.db.get_val_int('nidexchange', 0) if '*' in self.nid: - self.nid = get_local_nid(self.net_type, self.nid) + if self.nid_exchange: + self.nid = get_local_nid(self.net_type, self.nid) + else: + self.nid = get_local_address(self.net_type, self.nid) if not self.nid: panic("unable to set nid for", self.net_type, self.nid) debug("nid:", self.nid) @@ -1013,6 +851,7 @@ class Network(Module): if self.net_type == 'gm': self.add_portals_module("/linux/gmnal", 'kgmnal') self.add_lustre_module('obdclass', 'obdclass') + self.add_lustre_module('ptlrpc', 'ptlrpc') def prepare(self): if is_network_prepared(): @@ -1082,19 +921,6 @@ class LDLM(Module): if is_prepared(self.uuid): Module.cleanup(self) -class PTLRPC(Module): - def __init__(self,db): - Module.__init__(self, 'PTLRPC', db) - self.add_lustre_module('ptlrpc', 'ptlrpc') - def prepare(self): - if is_prepared(self.uuid): - return - self.info() - lctl.newdev(attach="ptlrpc %s %s" % (self.name, self.uuid)) - def cleanup(self): - if is_prepared(self.uuid): - Module.cleanup(self) - class LOV(Module): def __init__(self,db): Module.__init__(self, 'LOV', db) @@ -1178,6 +1004,7 @@ class MDSDEV(Module): Module.__init__(self, 'MDSDEV', db) self.devpath = self.db.get_val('devpath','') self.size = self.db.get_val_int('devsize', 0) + self.journal_size = self.db.get_val_int('journalsize', 0) self.fstype = self.db.get_val('fstype', '') # overwrite the orignal MDSDEV name and uuid with the MDS name and uuid target_uuid = self.db.get_first_ref('target') @@ -1186,8 +1013,12 @@ class MDSDEV(Module): self.lovconfig_uuids = mds.get_refs('lovconfig') # FIXME: if fstype not set, then determine based on kernel version self.format = self.db.get_val('autoformat', "no") + if mds.get_val('failover', 0): + self.failover_mds = 'f' + else: + self.failover_mds = '' - active_uuid = mds.get_active_target() + active_uuid = get_active_target(mds) if not active_uuid: panic("No target device found:", target_uuid) if active_uuid == self.uuid: @@ -1215,7 +1046,8 @@ class MDSDEV(Module): return self.info(self.devpath, self.fstype, self.format) run_acceptors() - blkdev = block_dev(self.devpath, self.size, self.fstype, self.format) + blkdev = block_dev(self.devpath, self.size, self.fstype, self.format, + self.journal_size) if not is_prepared('MDT_UUID'): lctl.newdev(attach="mdt %s %s" % ('MDT', 'MDT_UUID'), setup ="") @@ -1244,6 +1076,7 @@ class OSD(Module): self.osdtype = self.db.get_val('osdtype') self.devpath = self.db.get_val('devpath', '') self.size = self.db.get_val_int('devsize', 0) + self.journal_size = self.db.get_val_int('journalsize', 0) self.fstype = self.db.get_val('fstype', '') target_uuid = self.db.get_first_ref('target') ost = self.db.lookup(target_uuid) @@ -1252,8 +1085,12 @@ class OSD(Module): self.format = self.db.get_val('autoformat', 'yes') if self.fstype == 'extN': self.add_lustre_module('extN', 'extN') + if ost.get_val('failover', 0): + self.failover_ost = 'f' + else: + self.failover_ost = '' - active_uuid = ost.get_active_target() + active_uuid = get_active_target(ost) if not active_uuid: panic("No target device found:", target_uuid) if active_uuid == self.uuid: @@ -1281,14 +1118,17 @@ class OSD(Module): if not self.active: debug(self.uuid, "not active") return - self.info(self.osdtype, self.devpath, self.size, self.fstype, self.format) + self.info(self.osdtype, self.devpath, self.size, self.fstype, + self.format, self.journal_size) run_acceptors() if self.osdtype == 'obdecho': blkdev = '' else: - blkdev = block_dev(self.devpath, self.size, self.fstype, self.format) + blkdev = block_dev(self.devpath, self.size, self.fstype, + self.format, self.journal_size) lctl.newdev(attach="%s %s %s" % (self.osdtype, self.name, self.uuid), - setup ="%s %s" %(blkdev, self.fstype)) + setup ="%s %s %s" %(blkdev, self.fstype, + self.failover_ost)) if not is_prepared('OSS_UUID'): lctl.newdev(attach="ost %s %s" % ('OSS', 'OSS_UUID'), setup ="") @@ -1313,7 +1153,7 @@ class Client(Module): self.target_uuid = tgtdb.getUUID() self.db = tgtdb - self.tgt_dev_uuid = tgtdb.get_active_target() + self.tgt_dev_uuid = get_active_target(tgtdb) if not self.tgt_dev_uuid: panic("No target device found for target:", self.target_name) @@ -1325,7 +1165,8 @@ class Client(Module): self.module_name = string.upper(module) self.name = '%s_%s_%s' % (self.module_name, owner, self.target_name) self.uuid = '%05x%05x_%.14s_%05x%05x' % (int(random.random() * 1048576), - int(random.random() * 1048576),self.name, + int(random.random() * 1048576), + self.name, int(random.random() * 1048576), int(random.random() * 1048576)) self.uuid = self.uuid[0:36] @@ -1334,7 +1175,7 @@ class Client(Module): def lookup_server(self, srv_uuid): """ Lookup a server's network information """ - self._server_nets = self.db.get_ost_net(srv_uuid) + self._server_nets = get_ost_net(self.db, srv_uuid) if len(self._server_nets) == 0: panic ("Unable to find a server for:", srv_uuid) @@ -1488,6 +1329,7 @@ class Mountpoint(Module): else: mdc_uuid = self.vosc.get_mdc_uuid() if not mdc_uuid: + self.vosc.cleanup() panic("Unable to determine MDC UUID. Probably need to cleanup before re-mounting.") self.info(self.path, self.mds_uuid, self.obd_uuid) if config.lctl_dump: @@ -1497,12 +1339,15 @@ class Mountpoint(Module): run("mkdir", self.path) ret, val = run(cmd) if ret: + self.vosc.cleanup() + if self.vosc.need_mdc(): + cleanup_mdc(self.db, self.name, self.mds_uuid) panic("mount failed:", self.path) def cleanup(self): self.info(self.path, self.mds_uuid,self.obd_uuid) if fs_is_mounted(self.path): - if config.force(): + if config.force: (rc, out) = run("umount", "-f", self.path) else: (rc, out) = run("umount", self.path) @@ -1527,424 +1372,64 @@ class Mountpoint(Module): # ============================================================ # XML processing and query -class LustreDB: - def lookup(self, uuid): - """ lookup returns a new LustreDB instance""" - return self._lookup_by_uuid(uuid) - - def lookup_name(self, name, class_name = ""): - """ lookup returns a new LustreDB instance""" - return self._lookup_by_name(name, class_name) - - def lookup_class(self, class_name): - """ lookup returns a new LustreDB instance""" - return self._lookup_by_class(class_name) - - def get_val(self, tag, default=None): - v = self._get_val(tag) - if v: - return v - if default != None: - return default - debug("LustreDB", self.getName(), " no value for:", tag) - return None - - def get_class(self): - return self._get_class() - - def get_val_int(self, tag, default=0): - str = self._get_val(tag) - try: - if str: - return int(str) - return default - except ValueError: - panic("text value is not integer:", str) - - def get_first_ref(self, tag): - """ Get the first uuidref of the type TAG. Only - one is expected. Returns the uuid.""" - uuids = self._get_refs(tag) - if len(uuids) > 0: - return uuids[0] - return None - - def get_refs(self, tag): - """ Get all the refs of type TAG. Returns list of uuids. """ - uuids = self._get_refs(tag) - return uuids - - def get_all_refs(self): - """ Get all the refs. Returns list of uuids. """ - uuids = self._get_all_refs() - return uuids - - def get_ost_net(self, osd_uuid): - srv_list = [] - if not osd_uuid: - return srv_list - osd = self.lookup(osd_uuid) - node_uuid = osd.get_first_ref('node') - node = self.lookup(node_uuid) - if not node: - panic("unable to find node for osd_uuid:", osd_uuid, - " node_ref:", node_uuid) - for net_uuid in node.get_networks(): - db = node.lookup(net_uuid) - srv_list.append(Network(db)) +def get_ost_net(self, osd_uuid): + srv_list = [] + if not osd_uuid: return srv_list + osd = self.lookup(osd_uuid) + node_uuid = osd.get_first_ref('node') + node = self.lookup(node_uuid) + if not node: + panic("unable to find node for osd_uuid:", osd_uuid, + " node_ref:", node_uuid) + for net_uuid in node.get_networks(): + db = node.lookup(net_uuid) + srv_list.append(Network(db)) + return srv_list + + +# the order of iniitailization is based on level. +def getServiceLevel(self): + type = self.get_class() + ret=0; + if type in ('network',): + ret = 5 + elif type in ('routetbl',): + ret = 6 + elif type in ('device', 'ldlm'): + ret = 20 + elif type in ('osd', 'mdd', 'cobd'): + ret = 30 + elif type in ('mdsdev','ost'): + ret = 40 + elif type in ('mdc','osc'): + ret = 50 + elif type in ('lov',): + ret = 60 + elif type in ('mountpoint', 'echoclient'): + ret = 70 + + if ret < config.minlevel or ret > config.maxlevel: + ret = 0 + return ret - def nid2server(self, nid, net_type): - netlist = self.lookup_class('network') - for net_db in netlist: - if net_db.get_val('nid') == nid and net_db.get_val('nettype') == net_type: - return net_db - return None - - # the tag name is the service type - # fixme: this should do some checks to make sure the dom_node is a service - # - # determine what "level" a particular node is at. - - # the order of iniitailization is based on level. - def getServiceLevel(self): - type = self.get_class() - ret=0; - if type in ('network',): - ret = 5 - elif type in ('routetbl',): - ret = 6 - elif type in ('ptlrpc',): - ret = 7 - elif type in ('device', 'ldlm'): - ret = 20 - elif type in ('osd', 'mdd', 'cobd'): - ret = 30 - elif type in ('mdsdev','ost'): - ret = 40 - elif type in ('mdc','osc'): - ret = 50 - elif type in ('lov',): - ret = 60 - elif type in ('mountpoint', 'echoclient'): - ret = 70 - - if ret < config.minlevel() or ret > config.maxlevel(): - ret = 0 - return ret - - # - # return list of services in a profile. list is a list of tuples - # [(level, db_object),] - def getServices(self): - list = [] - for ref_class, ref_uuid in self.get_all_refs(): - servdb = self.lookup(ref_uuid) - if servdb: - level = servdb.getServiceLevel() - if level > 0: - list.append((level, servdb)) - else: - panic('service not found: ' + ref_uuid) - - list.sort() - return list - - # Find the target_device for target on a node - # node->profiles->device_refs->target - def get_target_device(self, target_uuid, node_name): - node_db = self.lookup_name(node_name) - if not node_db: - return None - prof_list = node_db.get_refs('profile') - for prof_uuid in prof_list: - prof_db = node_db.lookup(prof_uuid) - ref_list = prof_db.get_all_refs() - for ref in ref_list: - dev = self.lookup(ref[1]) - if dev and dev.get_first_ref('target') == target_uuid: - return ref[1] - return None - - def get_active_target(self): - target_uuid = self.getUUID() - target_name = self.getName() - node_name = config.select(target_name) - if node_name: - tgt_dev_uuid = self.get_target_device(target_uuid, node_name) - else: - tgt_dev_uuid = self.get_first_ref('active') - return tgt_dev_uuid - - - # get all network uuids for this node - def get_networks(self): - ret = [] - prof_list = self.get_refs('profile') - for prof_uuid in prof_list: - prof_db = self.lookup(prof_uuid) - net_list = prof_db.get_refs('network') - #debug("get_networks():", prof_uuid, net_list) - for net_uuid in net_list: - ret.append(net_uuid) - return ret - -class LustreDB_XML(LustreDB): - def __init__(self, dom, root_node): - # init xmlfile - self.dom_node = dom - self.root_node = root_node - - def xmltext(self, dom_node, tag): - list = dom_node.getElementsByTagName(tag) - if len(list) > 0: - dom_node = list[0] - dom_node.normalize() - if dom_node.firstChild: - txt = string.strip(dom_node.firstChild.data) - if txt: - return txt - - def xmlattr(self, dom_node, attr): - return dom_node.getAttribute(attr) - - def _get_val(self, tag): - """a value could be an attribute of the current node - or the text value in a child node""" - ret = self.xmlattr(self.dom_node, tag) - if not ret: - ret = self.xmltext(self.dom_node, tag) - return ret - - def _get_class(self): - return self.dom_node.nodeName - - # - # [(ref_class, ref_uuid),] - def _get_all_refs(self): - list = [] - for n in self.dom_node.childNodes: - if n.nodeType == n.ELEMENT_NODE: - ref_uuid = self.xml_get_ref(n) - ref_class = n.nodeName - list.append((ref_class, ref_uuid)) - - list.sort() - return list - - def _get_refs(self, tag): - """ Get all the refs of type TAG. Returns list of uuids. """ - uuids = [] - refname = '%s_ref' % tag - reflist = self.dom_node.getElementsByTagName(refname) - for r in reflist: - uuids.append(self.xml_get_ref(r)) - return uuids - - def xmllookup_by_uuid(self, dom_node, uuid): - for n in dom_node.childNodes: - if n.nodeType == n.ELEMENT_NODE: - if self.xml_get_uuid(n) == uuid: - return n - else: - n = self.xmllookup_by_uuid(n, uuid) - if n: return n - return None - - def _lookup_by_uuid(self, uuid): - dom = self. xmllookup_by_uuid(self.root_node, uuid) - if dom: - return LustreDB_XML(dom, self.root_node) - - def xmllookup_by_name(self, dom_node, name): - for n in dom_node.childNodes: - if n.nodeType == n.ELEMENT_NODE: - if self.xml_get_name(n) == name: - return n - else: - n = self.xmllookup_by_name(n, name) - if n: return n - return None - - def _lookup_by_name(self, name, class_name): - dom = self.xmllookup_by_name(self.root_node, name) - if dom: - return LustreDB_XML(dom, self.root_node) - - def xmllookup_by_class(self, dom_node, class_name): - return dom_node.getElementsByTagName(class_name) - - def _lookup_by_class(self, class_name): - ret = [] - domlist = self.xmllookup_by_class(self.root_node, class_name) - for node in domlist: - ret.append(LustreDB_XML(node, self.root_node)) - return ret - - def xml_get_name(self, n): - return n.getAttribute('name') - - def getName(self): - return self.xml_get_name(self.dom_node) - - def xml_get_ref(self, n): - return n.getAttribute('uuidref') - - def xml_get_uuid(self, dom_node): - return dom_node.getAttribute('uuid') - - def getUUID(self): - return self.xml_get_uuid(self.dom_node) - - def get_routes(self, type, gw): - """ Return the routes as a list of tuples of the form: - [(type, gw, lo, hi),]""" - res = [] - tbl = self.dom_node.getElementsByTagName('routetbl') - for t in tbl: - routes = t.getElementsByTagName('route') - for r in routes: - net_type = self.xmlattr(r, 'type') - if type != net_type: - lo = self.xmlattr(r, 'lo') - hi = self.xmlattr(r, 'hi') - res.append((type, gw, lo, hi)) - return res - - def get_route_tbl(self): - ret = [] - for r in self.dom_node.getElementsByTagName('route'): - net_type = self.xmlattr(r, 'type') - gw = self.xmlattr(r, 'gw') - lo = self.xmlattr(r, 'lo') - hi = self.xmlattr(r, 'hi') - ret.append((net_type, gw, lo, hi)) - return ret - - -# ================================================================ -# LDAP Support -class LustreDB_LDAP(LustreDB): - def __init__(self, name, attrs, - base = "fs=lustre", - parent = None, - url = "ldap://localhost", - user = "cn=Manager, fs=lustre", - pw = "secret" - ): - self._name = name - self._attrs = attrs - self._base = base - self._parent = parent - self._url = url - self._user = user - self._pw = pw - if parent: - self.l = parent.l - self._base = parent._base - else: - self.open() - - def open(self): - import ldap - try: - self.l = ldap.initialize(self._url) - # Set LDAP protocol version used - self.l.protocol_version=ldap.VERSION3 - # user and pw only needed if modifying db - self.l.bind_s("", "", ldap.AUTH_SIMPLE); - except ldap.LDAPError, e: - panic(e) - # FIXME, do something useful here - - def close(self): - self.l.unbind_s() - - def ldap_search(self, filter): - """Return list of uuids matching the filter.""" - import ldap - dn = self._base - ret = [] - uuids = [] - try: - for name, attrs in self.l.search_s(dn, ldap.SCOPE_ONELEVEL, - filter, ["uuid"]): - for v in attrs['uuid']: - uuids.append(v) - except ldap.NO_SUCH_OBJECT, e: - pass - except ldap.LDAPError, e: - print e # FIXME: die here? - if len(uuids) > 0: - for uuid in uuids: - ret.append(self._lookup_by_uuid(uuid)) - return ret - - def _lookup_by_name(self, name, class_name): - list = self.ldap_search("lustreName=%s" %(name)) - if len(list) == 1: - return list[0] - return [] - - def _lookup_by_class(self, class_name): - return self.ldap_search("objectclass=%s" %(string.upper(class_name))) - - def _lookup_by_uuid(self, uuid): - import ldap - dn = "uuid=%s,%s" % (uuid, self._base) - ret = None - try: - for name, attrs in self.l.search_s(dn, ldap.SCOPE_BASE, - "objectclass=*"): - ret = LustreDB_LDAP(name, attrs, parent = self) - - except ldap.NO_SUCH_OBJECT, e: - debug("NO_SUCH_OBJECT:", uuid) - pass # just return empty list - except ldap.LDAPError, e: - print e # FIXME: die here? - return ret - - - def _get_val(self, k): - ret = None - if self._attrs.has_key(k): - v = self._attrs[k] - if type(v) == types.ListType: - ret = str(v[0]) +# +# return list of services in a profile. list is a list of tuples +# [(level, db_object),] +def getServices(self): + list = [] + for ref_class, ref_uuid in self.get_all_refs(): + servdb = self.lookup(ref_uuid) + if servdb: + level = getServiceLevel(servdb) + if level > 0: + list.append((level, servdb)) else: - ret = str(v) - return ret - - def _get_class(self): - return string.lower(self._attrs['objectClass'][0]) - - # - # [(ref_class, ref_uuid),] - def _get_all_refs(self): - list = [] - for k in self._attrs.keys(): - if re.search('.*Ref', k): - for uuid in self._attrs[k]: - list.append((k, uuid)) - return list + panic('service not found: ' + ref_uuid) - def _get_refs(self, tag): - """ Get all the refs of type TAG. Returns list of uuids. """ - uuids = [] - refname = '%sRef' % tag - if self._attrs.has_key(refname): - return self._attrs[refname] - return [] + list.sort() + return list - def getName(self): - return self._get_val('lustreName') - - def getUUID(self): - return self._get_val('uuid') - - def get_route_tbl(self): - return [] ############################################################ # MDC UUID hack - @@ -2054,6 +1539,16 @@ def find_route(srv_list): return srv, r return None,None +def get_active_target(db): + target_uuid = db.getUUID() + target_name = db.getName() + node_name = get_select(target_name) + if node_name: + tgt_dev_uuid = db.get_target_device(target_uuid, node_name) + else: + tgt_dev_uuid = db.get_first_ref('active') + return tgt_dev_uuid + ############################################################ # lconf level logic @@ -2064,8 +1559,6 @@ def newService(db): n = None if type == 'ldlm': n = LDLM(db) - elif type == 'ptlrpc': - n = PTLRPC(db) elif type == 'lov': n = LOV(db) elif type == 'network': @@ -2099,25 +1592,25 @@ def for_each_profile(db, prof_list, operation): prof_db = db.lookup(prof_uuid) if not prof_db: panic("profile:", profile, "not found.") - services = prof_db.getServices() + services = getServices(prof_db) operation(services) def doSetup(services): - if config.nosetup(): + if config.nosetup: return for s in services: n = newService(s[1]) n.prepare() def doModules(services): - if config.nomod(): + if config.nomod: return for s in services: n = newService(s[1]) n.load_module() def doCleanup(services): - if config.nosetup(): + if config.nosetup: return services.reverse() for s in services: @@ -2125,7 +1618,7 @@ def doCleanup(services): n.cleanup() def doUnloadModules(services): - if config.nomod(): + if config.nomod: return services.reverse() for s in services: @@ -2147,7 +1640,7 @@ def doHost(lustreDB, hosts): return router_flag = node_db.get_val_int('router', 0) - recovery_upcall = node_db.get_val('recovery_upcall', '') + recovery_upcall = node_db.get_val('recoveryUpcall', '') timeout = node_db.get_val_int('timeout', 0) add_local_interfaces(node_db) @@ -2158,16 +1651,22 @@ def doHost(lustreDB, hosts): # if not cleaning, load modules first. prof_list = node_db.get_refs('profile') - if config.cleanup(): - if config.force(): + if config.recover: + if not (config.tgt_uuid or config.client_uuid or config.conn_uuid): + panic( "--recovery requires the failed uuids") + doRecovery(lustreDB, lctl, config.tgt_uuid, config.client_uuid, + config.conn_uuid) + elif config.cleanup: + if config.force: # the command line can override this value timeout = 5 # ugly hack, only need to run lctl commands for --dump - if config.lctl_dump(): + if config.lctl_dump: for_each_profile(node_db, prof_list, doCleanup) return sys_set_timeout(timeout) + sys_set_ptldebug() sys_set_recovery_upcall(recovery_upcall) for_each_profile(node_db, prof_list, doCleanup) @@ -2175,16 +1674,20 @@ def doHost(lustreDB, hosts): else: # ugly hack, only need to run lctl commands for --dump - if config.lctl_dump(): + if config.lctl_dump: for_each_profile(node_db, prof_list, doSetup) return + sys_make_devices() + sys_set_netmem_max('/proc/sys/net/core/rmem_max', MAXTCPBUF) + sys_set_netmem_max('/proc/sys/net/core/wmem_max', MAXTCPBUF) + for_each_profile(node_db, prof_list, doModules) sys_set_debug_path() - script = config.gdb_script() + script = config.gdb_script run(lctl.lctl, ' modules >', script) - if config.gdb(): + if config.gdb: log ("The GDB module script is in", script) # pause, so user has time to break and # load the script @@ -2194,102 +1697,34 @@ def doHost(lustreDB, hosts): for_each_profile(node_db, prof_list, doSetup) -############################################################ -# Command line processing -# -def parse_cmdline(argv): - short_opts = "hdnvf" - long_opts = ["ldap", "reformat", "lustre=", "verbose", "gdb", - "portals=", "makeldiff", "cleanup", "noexec", - "help", "node=", "nomod", "nosetup", - "dump=", "force", "minlevel=", "maxlevel=", - "timeout=", "recovery_upcall=", - "ldapurl=", "config=", "select=", "lctl_dump="] - opts = [] - args = [] +def doRecovery(db, lctl, tgt_uuid, client_uuid, conn_uuid): + tgt = db.lookup(tgt_uuid) + new_uuid = get_active_target(tgt) + net = local_net(get_ost_net(db, new_uuid)) + if not net: + panic("Unable to find a connection to:", new_uuid) + lctl.connect(net) + lctl.recover(client_uuid, net.uuid) - try: - opts, args = getopt.getopt(argv, short_opts, long_opts) - except getopt.error: - print "invalid opt" - usage() - - for o, a in opts: - if o in ("-h", "--help"): - usage() - if o in ("-d","--cleanup"): - config.cleanup(1) - if o in ("-v", "--verbose"): - config.verbose(1) - if o in ("-n", "--noexec"): - config.noexec(1) - if o == "--portals": - config.portals_dir(a) - if o == "--lustre": - config.lustre_dir(a) - if o == "--reformat": - config.reformat(1) - if o == "--node": - config.node(a) - if o == "--gdb": - config.gdb(1) - if o == "--nomod": - config.nomod(1) - if o == "--nosetup": - config.nosetup(1) - if o == "--dump": - config.dump_file(a) - if o in ("-f", "--force"): - config.force(1) - if o == "--minlevel": - config.minlevel(a) - if o == "--maxlevel": - config.maxlevel(a) - if o == "--timeout": - config.timeout(a) - if o == "--recovery_upcall": - config.recovery_upcall(a) - if o == "--ldapurl": - config.ldapurl(a) - if o == "--config": - config.config_name(a) - if o == "--select": - config.init_select(a) - if o == "--lctl_dump": - config.lctl_dump(a) - - return args - -def fetch(url): - import urllib - data = "" - try: - s = urllib.urlopen(url) - data = s.read() - except: - usage() - return data def setupModulePath(cmd, portals_dir = PORTALS_DIR): base = os.path.dirname(cmd) - if os.access(base+"/Makefile", os.R_OK): - if not config.lustre_dir(): - config.lustre_dir(os.path.join(base, "..")) + if development_mode(): + if not config.lustre: + config.lustre = (os.path.join(base, "..")) # normalize the portals dir, using command line arg if set - if config.portals_dir(): - portals_dir = config.portals_dir() - dir = os.path.join(config.lustre_dir(), portals_dir) - config.portals_dir(dir) - elif config.lustre_dir() and config.portals_dir(): + if config.portals: + portals_dir = config.portals + dir = os.path.join(config.lustre, portals_dir) + config.portals = dir + elif config.lustre and config.portals: # production mode # if --lustre and --portals, normalize portals # can ignore POTRALS_DIR here, since it is probly useless here - dir = config.portals_dir() - dir = os.path.join(config.lustre_dir(), dir) - config.portals_dir(dir) + config.portals = os.path.join(config.lustre, config.portals) def sysctl(path, val): - if config.noexec(): + if config.noexec: return try: fp = open(os.path.join('/proc/sys', path), 'w') @@ -2300,34 +1735,34 @@ def sysctl(path, val): def sys_set_debug_path(): - debug("debug path: ", config.debug_path()) - sysctl('portals/debug_path', config.debug_path()) + debug("debug path: ", config.debug_path) + sysctl('portals/debug_path', config.debug_path) def sys_set_recovery_upcall(upcall): # the command overrides the value in the node config - if config.recovery_upcall(): - upcall = config.recovery_upcall() + if config.recovery_upcall: + upcall = config.recovery_upcall if upcall: debug("setting recovery_upcall:", upcall) sysctl('lustre/recovery_upcall', upcall) def sys_set_timeout(timeout): # the command overrides the value in the node config - if config.timeout() > 0: - timeout = config.timeout() + if config.timeout > 0: + timeout = config.timeout if timeout > 0: debug("setting timeout:", timeout) sysctl('lustre/timeout', timeout) -def sys_set_ptldebug(ptldebug): +def sys_set_ptldebug(): # the command overrides the value in the node config - if config.ptldebug(): - ptldebug = config.ptldebug() - sysctl('portals/debug', ptldebug) + if config.ptldebug != None: + ptldebug = config.ptldebug + sysctl('portals/debug', ptldebug) def sys_set_netmem_max(path, max): debug("setting", path, "to at least", max) - if config.noexec(): + if config.noexec: return fp = open(path) str = fp.readline() @@ -2353,6 +1788,20 @@ def add_to_path(new_dir): return os.environ['PATH'] = os.environ['PATH'] + ':' + new_dir +def default_debug_path(): + path = '/tmp/lustre-log' + if os.path.isdir('/r'): + return '/r' + path + else: + return path + +def default_gdb_script(): + script = '/tmp/ogdb' + if os.path.isdir('/r'): + return '/r' + script + else: + return script + DEFAULT_PATH = ('/sbin', '/usr/sbin', '/bin', '/usr/bin') # ensure basic elements are in the system path @@ -2360,13 +1809,89 @@ def sanitise_path(): for dir in DEFAULT_PATH: add_to_path(dir) -# Initialize or shutdown lustre according to a configuration file -# * prepare the system for lustre -# * configure devices with lctl -# Shutdown does steps in reverse -# +# global hack for the --select handling +tgt_select = {} +def init_select(arg): + # arg = "service=nodeA,service2=nodeB" + global tgt_select + list = string.split(arg, ',') + for entry in list: + srv, node = string.split(entry, '=') + tgt_select[srv] = node + +def get_select(srv): + if tgt_select.has_key(srv): + return tgt_select[srv] + return None + +lconf_options = [ + ('verbose,v', "Print system commands as they are run"), + ('ldapurl',"LDAP server URL, eg. ldap://localhost", Lustre.Options.PARAM), + ('config', "Cluster config name used for LDAP query", Lustre.Options.PARAM), + ('select', "service=nodeA,service2=nodeB ", Lustre.Options.PARAM), + ('node', "Load config for ", Lustre.Options.PARAM), + ('cleanup,d', "Cleans up config. (Shutdown)"), + ('force,f', "Forced unmounting and/or obd detach during cleanup", + Lustre.Options.FLAG, 0), + ('gdb', """Prints message after creating gdb module script + and sleeps for 5 seconds."""), + ('noexec,n', """Prints the commands and steps that will be run for a + config without executing them. This can used to check if a + config file is doing what it should be doing"""), + ('nomod', "Skip load/unload module step."), + ('nosetup', "Skip device setup/cleanup step."), + ('reformat', "Reformat all devices (without question)"), + ('dump', "Dump the kernel debug log to file before portals is unloaded", + Lustre.Options.PARAM), + ('minlevel', "Specify the minimum level of services to configure/cleanup", + Lustre.Options.PARAM, 0), + ('maxlevel', """Specify the maximum level of services to configure/cleanup + Levels are aproximatly like: + 10 - network + 20 - device, ldlm + 30 - osd, mdd + 40 - mds, ost + 70 - mountpoint, echo_client, osc, mdc, lov""", + Lustre.Options.PARAM, 100), + ('lustre', """Base directory of lustre sources. This parameter will + cause lconf to load modules from a source tree.""", + Lustre.Options.PARAM), + ('portals', """Portals source directory. If this is a relative path, + then it is assumed to be relative to lustre. """, + Lustre.Options.PARAM), + ('timeout', "Set recovery timeout", Lustre.Options.PARAM), + ('recovery_upcall', "Set recovery upcall script", Lustre.Options.PARAM), + ('lctl_dump', "Save lctl ioctls to the dumfile argument", + Lustre.Options.PARAM), + ('ptldebug', "Set the portals debug level", Lustre.Options.PARAM), + ('gdb_script', "Fullname of gdb debug script", Lustre.Options.PARAM, + default_gdb_script()), + ('debug_path', "Path to save debug dumps", Lustre.Options.PARAM, + default_debug_path()), +# Client recovery options + ('recover', "Recover a device"), + ('tgt_uuid', "The failed target (required for recovery)", + Lustre.Options.PARAM), + ('client_uuid', "The failed client (required for recovery)", + Lustre.Options.PARAM), + ('conn_uuid', "The failed connection (required for recovery)", + Lustre.Options.PARAM), + ] + def main(): - global lctl, MAXTCPBUF + global lctl, config + + # in the upcall this is set to SIG_IGN + signal.signal(signal.SIGCHLD, signal.SIG_DFL) + + cl = Lustre.Options("lconf", "config.xml", lconf_options) + try: + config, args = cl.parse(sys.argv[1:]) + except Lustre.OptionError, e: + print e + sys.exit(1) + + setupModulePath(sys.argv[0]) host = socket.gethostname() @@ -2382,7 +1907,6 @@ def main(): sanitise_path() - args = parse_cmdline(sys.argv[1:]) if len(args) > 0: if not os.access(args[0], os.R_OK): print 'File not found or readable:', args[0] @@ -2392,44 +1916,41 @@ def main(): except Exception: panic("%s does not appear to be a config file." % (args[0])) sys.exit(1) # make sure to die here, even in debug mode. - db = LustreDB_XML(dom.documentElement, dom.documentElement) - elif config.ldapurl(): - if not config.config_name(): + db = Lustre.LustreDB_XML(dom.documentElement, dom.documentElement) + elif config.ldapurl: + if not config.config: panic("--ldapurl requires --config name") - dn = "config=%s,fs=lustre" % (config.config_name()) - db = LustreDB_LDAP('', {}, base=dn, url = config.ldapurl()) + dn = "config=%s,fs=lustre" % (config.config) + db = Lustre.LustreDB_LDAP('', {}, base=dn, url = config.ldapurl) else: - usage() + cl.usage() + sys.exit(1) node_list = [] - if config.node(): - node_list.append(config.node()) + if config.node: + node_list.append(config.node) else: if len(host) > 0: node_list.append(host) node_list.append('localhost') + debug("configuring for host: ", node_list) if len(host) > 0: - config._debug_path = config._debug_path + '-' + host - config._gdb_script = config._gdb_script + '-' + host - - setupModulePath(sys.argv[0]) + config.debug_path = config.debug_path + '-' + host + config.gdb_script = config.gdb_script + '-' + host lctl = LCTLInterface('lctl') - if config.lctl_dump(): - lctl.use_save_file(config.lctl_dump()) - else: - sys_make_devices() - sys_set_netmem_max('/proc/sys/net/core/rmem_max', MAXTCPBUF) - sys_set_netmem_max('/proc/sys/net/core/wmem_max', MAXTCPBUF) + + if config.lctl_dump: + lctl.use_save_file(config.lctl_dump) doHost(db, node_list) if __name__ == "__main__": try: main() - except LconfError, e: + except Lustre.LconfError, e: print e except CommandError, e: e.dump() @@ -2437,4 +1958,3 @@ if __name__ == "__main__": if first_cleanup_error: sys.exit(first_cleanup_error) -